drivers/net/ethernet/netronome/nfp/abm/qdisc.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2018 Netronome Systems, Inc. */
   3
   4 #include <linux/rtnetlink.h>
   5 #include <net/pkt_cls.h>
   6 #include <net/pkt_sched.h>
   7 #include <net/red.h>
   8
   9 #include "../nfpcore/nfp_cpp.h"
  10 #include "../nfp_app.h"
  11 #include "../nfp_main.h"
  12 #include "../nfp_net.h"
  13 #include "../nfp_port.h"
  14 #include "main.h"
  15
  16 static bool nfp_abm_qdisc_is_red(struct nfp_qdisc *qdisc)
  17 {
  18         return qdisc->type == NFP_QDISC_RED || qdisc->type == NFP_QDISC_GRED;
  19 }
  20
  21 static bool nfp_abm_qdisc_child_valid(struct nfp_qdisc *qdisc, unsigned int id)
  22 {
  23         return qdisc->children[id] &&
  24                qdisc->children[id] != NFP_QDISC_UNTRACKED;
  25 }
  26
  27 static void *nfp_abm_qdisc_tree_deref_slot(void __rcu **slot)
  28 {
  29         return rtnl_dereference(*slot);
  30 }
  31
  32 static void
  33 nfp_abm_stats_propagate(struct nfp_alink_stats *parent,
  34                         struct nfp_alink_stats *child)
  35 {
  36         parent->tx_pkts         += child->tx_pkts;
  37         parent->tx_bytes        += child->tx_bytes;
  38         parent->backlog_pkts    += child->backlog_pkts;
  39         parent->backlog_bytes   += child->backlog_bytes;
  40         parent->overlimits      += child->overlimits;
  41         parent->drops           += child->drops;
  42 }
  43
  44 static void
  45 nfp_abm_stats_update_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc,
  46                          unsigned int queue)
  47 {
  48         struct nfp_cpp *cpp = alink->abm->app->cpp;
  49         unsigned int i;
  50         int err;
  51
  52         if (!qdisc->offloaded)
  53                 return;
  54
  55         for (i = 0; i < qdisc->red.num_bands; i++) {
  56                 err = nfp_abm_ctrl_read_q_stats(alink, i, queue,
  57                                                 &qdisc->red.band[i].stats);
  58                 if (err)
  59                         nfp_err(cpp, "RED stats (%d, %d) read failed with error %d\n",
  60                                 i, queue, err);
  61
  62                 err = nfp_abm_ctrl_read_q_xstats(alink, i, queue,
  63                                                  &qdisc->red.band[i].xstats);
  64                 if (err)
  65                         nfp_err(cpp, "RED xstats (%d, %d) read failed with error %d\n",
  66                                 i, queue, err);
  67         }
  68 }
  69
  70 static void
  71 nfp_abm_stats_update_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc)
  72 {
  73         unsigned int i;
  74
  75         if (qdisc->type != NFP_QDISC_MQ)
  76                 return;
  77
  78         for (i = 0; i < alink->total_queues; i++)
  79                 if (nfp_abm_qdisc_child_valid(qdisc, i))
  80                         nfp_abm_stats_update_red(alink, qdisc->children[i], i);
  81 }
  82
  83 static void __nfp_abm_stats_update(struct nfp_abm_link *alink, u64 time_now)
  84 {
  85         alink->last_stats_update = time_now;
  86         if (alink->root_qdisc)
  87                 nfp_abm_stats_update_mq(alink, alink->root_qdisc);
  88 }
  89
  90 static void nfp_abm_stats_update(struct nfp_abm_link *alink)
  91 {
  92         u64 now;
  93
  94         /* Limit the frequency of updates - stats of non-leaf qdiscs are a sum
  95          * of all their leafs, so we would read the same stat multiple times
  96          * for every dump.
  97          */
  98         now = ktime_get();
  99         if (now - alink->last_stats_update < NFP_ABM_STATS_REFRESH_IVAL)
 100                 return;
 101
 102         __nfp_abm_stats_update(alink, now);
 103 }
 104
 105 static void
 106 nfp_abm_qdisc_unlink_children(struct nfp_qdisc *qdisc,
 107                               unsigned int start, unsigned int end)
 108 {
 109         unsigned int i;
 110
 111         for (i = start; i < end; i++)
 112                 if (nfp_abm_qdisc_child_valid(qdisc, i)) {
 113                         qdisc->children[i]->use_cnt--;
 114                         qdisc->children[i] = NULL;
 115                 }
 116 }
 117
 118 static void
 119 nfp_abm_qdisc_offload_stop(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc)
 120 {
 121         unsigned int i;
 122
 123         /* Don't complain when qdisc is getting unlinked */
 124         if (qdisc->use_cnt)
 125                 nfp_warn(alink->abm->app->cpp, "Offload of '%08x' stopped\n",
 126                          qdisc->handle);
 127
 128         if (!nfp_abm_qdisc_is_red(qdisc))
 129                 return;
 130
 131         for (i = 0; i < qdisc->red.num_bands; i++) {
 132                 qdisc->red.band[i].stats.backlog_pkts = 0;
 133                 qdisc->red.band[i].stats.backlog_bytes = 0;
 134         }
 135 }
 136
 137 static int
 138 __nfp_abm_stats_init(struct nfp_abm_link *alink, unsigned int band,
 139                      unsigned int queue, struct nfp_alink_stats *prev_stats,
 140                      struct nfp_alink_xstats *prev_xstats)
 141 {
 142         u64 backlog_pkts, backlog_bytes;
 143         int err;
 144
 145         /* Don't touch the backlog, backlog can only be reset after it has
 146          * been reported back to the tc qdisc stats.
 147          */
 148         backlog_pkts = prev_stats->backlog_pkts;
 149         backlog_bytes = prev_stats->backlog_bytes;
 150
 151         err = nfp_abm_ctrl_read_q_stats(alink, band, queue, prev_stats);
 152         if (err) {
 153                 nfp_err(alink->abm->app->cpp,
 154                         "RED stats init (%d, %d) failed with error %d\n",
 155                         band, queue, err);
 156                 return err;
 157         }
 158
 159         err = nfp_abm_ctrl_read_q_xstats(alink, band, queue, prev_xstats);
 160         if (err) {
 161                 nfp_err(alink->abm->app->cpp,
 162                         "RED xstats init (%d, %d) failed with error %d\n",
 163                         band, queue, err);
 164                 return err;
 165         }
 166
 167         prev_stats->backlog_pkts = backlog_pkts;
 168         prev_stats->backlog_bytes = backlog_bytes;
 169         return 0;
 170 }
 171
 172 static int
 173 nfp_abm_stats_init(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc,
 174                    unsigned int queue)
 175 {
 176         unsigned int i;
 177         int err;
 178
 179         for (i = 0; i < qdisc->red.num_bands; i++) {
 180                 err = __nfp_abm_stats_init(alink, i, queue,
 181                                            &qdisc->red.band[i].prev_stats,
 182                                            &qdisc->red.band[i].prev_xstats);
 183                 if (err)
 184                         return err;
 185         }
 186
 187         return 0;
 188 }
 189
 190 static void
 191 nfp_abm_offload_compile_red(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc,
 192                             unsigned int queue)
 193 {
 194         bool good_red, good_gred;
 195         unsigned int i;
 196
 197         good_red = qdisc->type == NFP_QDISC_RED &&
 198                    qdisc->params_ok &&
 199                    qdisc->use_cnt == 1 &&
 200                    !alink->has_prio &&
 201                    !qdisc->children[0];
 202         good_gred = qdisc->type == NFP_QDISC_GRED &&
 203                     qdisc->params_ok &&
 204                     qdisc->use_cnt == 1;
 205         qdisc->offload_mark = good_red || good_gred;
 206
 207         /* If we are starting offload init prev_stats */
 208         if (qdisc->offload_mark && !qdisc->offloaded)
 209                 if (nfp_abm_stats_init(alink, qdisc, queue))
 210                         qdisc->offload_mark = false;
 211
 212         if (!qdisc->offload_mark)
 213                 return;
 214
 215         for (i = 0; i < alink->abm->num_bands; i++) {
 216                 enum nfp_abm_q_action act;
 217
 218                 nfp_abm_ctrl_set_q_lvl(alink, i, queue,
 219                                        qdisc->red.band[i].threshold);
 220                 act = qdisc->red.band[i].ecn ?
 221                         NFP_ABM_ACT_MARK_DROP : NFP_ABM_ACT_DROP;
 222                 nfp_abm_ctrl_set_q_act(alink, i, queue, act);
 223         }
 224 }
 225
 226 static void
 227 nfp_abm_offload_compile_mq(struct nfp_abm_link *alink, struct nfp_qdisc *qdisc)
 228 {
 229         unsigned int i;
 230
 231         qdisc->offload_mark = qdisc->type == NFP_QDISC_MQ;
 232         if (!qdisc->offload_mark)
 233                 return;
 234
 235         for (i = 0; i < alink->total_queues; i++) {
 236                 struct nfp_qdisc *child = qdisc->children[i];
 237
 238                 if (!nfp_abm_qdisc_child_valid(qdisc, i))
 239                         continue;
 240
 241                 nfp_abm_offload_compile_red(alink, child, i);
 242         }
 243 }
 244
 245 void nfp_abm_qdisc_offload_update(struct nfp_abm_link *alink)
 246 {
 247         struct nfp_abm *abm = alink->abm;
 248         struct radix_tree_iter iter;
 249         struct nfp_qdisc *qdisc;
 250         void __rcu **slot;
 251         size_t i;
 252
 253         /* Mark all thresholds as unconfigured */
 254         for (i = 0; i < abm->num_bands; i++)
 255                 __bitmap_set(abm->threshold_undef,
 256                              i * NFP_NET_MAX_RX_RINGS + alink->queue_base,
 257                              alink->total_queues);
 258
 259         /* Clear offload marks */
 260         radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) {
 261                 qdisc = nfp_abm_qdisc_tree_deref_slot(slot);
 262                 qdisc->offload_mark = false;
 263         }
 264
 265         if (alink->root_qdisc)
 266                 nfp_abm_offload_compile_mq(alink, alink->root_qdisc);
 267
 268         /* Refresh offload status */
 269         radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) {
 270                 qdisc = nfp_abm_qdisc_tree_deref_slot(slot);
 271                 if (!qdisc->offload_mark && qdisc->offloaded)
 272                         nfp_abm_qdisc_offload_stop(alink, qdisc);
 273                 qdisc->offloaded = qdisc->offload_mark;
 274         }
 275
 276         /* Reset the unconfigured thresholds */
 277         for (i = 0; i < abm->num_thresholds; i++)
 278                 if (test_bit(i, abm->threshold_undef))
 279                         __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY);
 280
 281         __nfp_abm_stats_update(alink, ktime_get());
 282 }
 283
 284 static void
 285 nfp_abm_qdisc_clear_mq(struct net_device *netdev, struct nfp_abm_link *alink,
 286                        struct nfp_qdisc *qdisc)
 287 {
 288         struct radix_tree_iter iter;
 289         unsigned int mq_refs = 0;
 290         void __rcu **slot;
 291
 292         if (!qdisc->use_cnt)
 293                 return;
 294         /* MQ doesn't notify well on destruction, we need special handling of
 295          * MQ's children.
 296          */
 297         if (qdisc->type == NFP_QDISC_MQ &&
 298             qdisc == alink->root_qdisc &&
 299             netdev->reg_state == NETREG_UNREGISTERING)
 300                 return;
 301
 302         /* Count refs held by MQ instances and clear pointers */
 303         radix_tree_for_each_slot(slot, &alink->qdiscs, &iter, 0) {
 304                 struct nfp_qdisc *mq = nfp_abm_qdisc_tree_deref_slot(slot);
 305                 unsigned int i;
 306
 307                 if (mq->type != NFP_QDISC_MQ || mq->netdev != netdev)
 308                         continue;
 309                 for (i = 0; i < mq->num_children; i++)
 310                         if (mq->children[i] == qdisc) {
 311                                 mq->children[i] = NULL;
 312                                 mq_refs++;
 313                         }
 314         }
 315
 316         WARN(qdisc->use_cnt != mq_refs, "non-zero qdisc use count: %d (- %d)\n",
 317              qdisc->use_cnt, mq_refs);
 318 }
 319
 320 static void
 321 nfp_abm_qdisc_free(struct net_device *netdev, struct nfp_abm_link *alink,
 322                    struct nfp_qdisc *qdisc)
 323 {
 324         struct nfp_port *port = nfp_port_from_netdev(netdev);
 325
 326         if (!qdisc)
 327                 return;
 328         nfp_abm_qdisc_clear_mq(netdev, alink, qdisc);
 329         WARN_ON(radix_tree_delete(&alink->qdiscs,
 330                                   TC_H_MAJ(qdisc->handle)) != qdisc);
 331
 332         kfree(qdisc->children);
 333         kfree(qdisc);
 334
 335         port->tc_offload_cnt--;
 336 }
 337
 338 static struct nfp_qdisc *
 339 nfp_abm_qdisc_alloc(struct net_device *netdev, struct nfp_abm_link *alink,
 340                     enum nfp_qdisc_type type, u32 parent_handle, u32 handle,
 341                     unsigned int children)
 342 {
 343         struct nfp_port *port = nfp_port_from_netdev(netdev);
 344         struct nfp_qdisc *qdisc;
 345         int err;
 346
 347         qdisc = kzalloc(sizeof(*qdisc), GFP_KERNEL);
 348         if (!qdisc)
 349                 return NULL;
 350
 351         if (children) {
 352                 qdisc->children = kcalloc(children, sizeof(void *), GFP_KERNEL);
 353                 if (!qdisc->children)
 354                         goto err_free_qdisc;
 355         }
 356
 357         qdisc->netdev = netdev;
 358         qdisc->type = type;
 359         qdisc->parent_handle = parent_handle;
 360         qdisc->handle = handle;
 361         qdisc->num_children = children;
 362
 363         err = radix_tree_insert(&alink->qdiscs, TC_H_MAJ(qdisc->handle), qdisc);
 364         if (err) {
 365                 nfp_err(alink->abm->app->cpp,
 366                         "Qdisc insertion into radix tree failed: %d\n", err);
 367                 goto err_free_child_tbl;
 368         }
 369
 370         port->tc_offload_cnt++;
 371         return qdisc;
 372
 373 err_free_child_tbl:
 374         kfree(qdisc->children);
 375 err_free_qdisc:
 376         kfree(qdisc);
 377         return NULL;
 378 }
 379
 380 static struct nfp_qdisc *
 381 nfp_abm_qdisc_find(struct nfp_abm_link *alink, u32 handle)
 382 {
 383         return radix_tree_lookup(&alink->qdiscs, TC_H_MAJ(handle));
 384 }
 385
 386 static int
 387 nfp_abm_qdisc_replace(struct net_device *netdev, struct nfp_abm_link *alink,
 388                       enum nfp_qdisc_type type, u32 parent_handle, u32 handle,
 389                       unsigned int children, struct nfp_qdisc **qdisc)
 390 {
 391         *qdisc = nfp_abm_qdisc_find(alink, handle);
 392         if (*qdisc) {
 393                 if (WARN_ON((*qdisc)->type != type))
 394                         return -EINVAL;
 395                 return 1;
 396         }
 397
 398         *qdisc = nfp_abm_qdisc_alloc(netdev, alink, type, parent_handle, handle,
 399                                      children);
 400         return *qdisc ? 0 : -ENOMEM;
 401 }
 402
 403 static void
 404 nfp_abm_qdisc_destroy(struct net_device *netdev, struct nfp_abm_link *alink,
 405                       u32 handle)
 406 {
 407         struct nfp_qdisc *qdisc;
 408
 409         qdisc = nfp_abm_qdisc_find(alink, handle);
 410         if (!qdisc)
 411                 return;
 412
 413         /* We don't get TC_SETUP_ROOT_QDISC w/ MQ when netdev is unregistered */
 414         if (alink->root_qdisc == qdisc)
 415                 qdisc->use_cnt--;
 416
 417         nfp_abm_qdisc_unlink_children(qdisc, 0, qdisc->num_children);
 418         nfp_abm_qdisc_free(netdev, alink, qdisc);
 419
 420         if (alink->root_qdisc == qdisc) {
 421                 alink->root_qdisc = NULL;
 422                 /* Only root change matters, other changes are acted upon on
 423                  * the graft notification.
 424                  */
 425                 nfp_abm_qdisc_offload_update(alink);
 426         }
 427 }
 428
 429 static int
 430 nfp_abm_qdisc_graft(struct nfp_abm_link *alink, u32 handle, u32 child_handle,
 431                     unsigned int id)
 432 {
 433         struct nfp_qdisc *parent, *child;
 434
 435         parent = nfp_abm_qdisc_find(alink, handle);
 436         if (!parent)
 437                 return 0;
 438
 439         if (WARN(id >= parent->num_children,
 440                  "graft child out of bound %d >= %d\n",
 441                  id, parent->num_children))
 442                 return -EINVAL;
 443
 444         nfp_abm_qdisc_unlink_children(parent, id, id + 1);
 445
 446         child = nfp_abm_qdisc_find(alink, child_handle);
 447         if (child)
 448                 child->use_cnt++;
 449         else
 450                 child = NFP_QDISC_UNTRACKED;
 451         parent->children[id] = child;
 452
 453         nfp_abm_qdisc_offload_update(alink);
 454
 455         return 0;
 456 }
 457
 458 static void
 459 nfp_abm_stats_calculate(struct nfp_alink_stats *new,
 460                         struct nfp_alink_stats *old,
 461                         struct gnet_stats_basic_packed *bstats,
 462                         struct gnet_stats_queue *qstats)
 463 {
 464         _bstats_update(bstats, new->tx_bytes - old->tx_bytes,
 465                        new->tx_pkts - old->tx_pkts);
 466         qstats->qlen += new->backlog_pkts - old->backlog_pkts;
 467         qstats->backlog += new->backlog_bytes - old->backlog_bytes;
 468         qstats->overlimits += new->overlimits - old->overlimits;
 469         qstats->drops += new->drops - old->drops;
 470 }
 471
 472 static void
 473 nfp_abm_stats_red_calculate(struct nfp_alink_xstats *new,
 474                             struct nfp_alink_xstats *old,
 475                             struct red_stats *stats)
 476 {
 477         stats->forced_mark += new->ecn_marked - old->ecn_marked;
 478         stats->pdrop += new->pdrop - old->pdrop;
 479 }
 480
 481 static int
 482 nfp_abm_gred_stats(struct nfp_abm_link *alink, u32 handle,
 483                    struct tc_gred_qopt_offload_stats *stats)
 484 {
 485         struct nfp_qdisc *qdisc;
 486         unsigned int i;
 487
 488         nfp_abm_stats_update(alink);
 489
 490         qdisc = nfp_abm_qdisc_find(alink, handle);
 491         if (!qdisc)
 492                 return -EOPNOTSUPP;
 493         /* If the qdisc offload has stopped we may need to adjust the backlog
 494          * counters back so carry on even if qdisc is not currently offloaded.
 495          */
 496
 497         for (i = 0; i < qdisc->red.num_bands; i++) {
 498                 if (!stats->xstats[i])
 499                         continue;
 500
 501                 nfp_abm_stats_calculate(&qdisc->red.band[i].stats,
 502                                         &qdisc->red.band[i].prev_stats,
 503                                         &stats->bstats[i], &stats->qstats[i]);
 504                 qdisc->red.band[i].prev_stats = qdisc->red.band[i].stats;
 505
 506                 nfp_abm_stats_red_calculate(&qdisc->red.band[i].xstats,
 507                                             &qdisc->red.band[i].prev_xstats,
 508                                             stats->xstats[i]);
 509                 qdisc->red.band[i].prev_xstats = qdisc->red.band[i].xstats;
 510         }
 511
 512         return qdisc->offloaded ? 0 : -EOPNOTSUPP;
 513 }
 514
 515 static bool
 516 nfp_abm_gred_check_params(struct nfp_abm_link *alink,
 517                           struct tc_gred_qopt_offload *opt)
 518 {
 519         struct nfp_cpp *cpp = alink->abm->app->cpp;
 520         struct nfp_abm *abm = alink->abm;
 521         unsigned int i;
 522
 523         if (opt->set.grio_on || opt->set.wred_on) {
 524                 nfp_warn(cpp, "GRED offload failed - GRIO and WRED not supported (p:%08x h:%08x)\n",
 525                          opt->parent, opt->handle);
 526                 return false;
 527         }
 528         if (opt->set.dp_def != alink->def_band) {
 529                 nfp_warn(cpp, "GRED offload failed - default band must be %d (p:%08x h:%08x)\n",
 530                          alink->def_band, opt->parent, opt->handle);
 531                 return false;
 532         }
 533         if (opt->set.dp_cnt != abm->num_bands) {
 534                 nfp_warn(cpp, "GRED offload failed - band count must be %d (p:%08x h:%08x)\n",
 535                          abm->num_bands, opt->parent, opt->handle);
 536                 return false;
 537         }
 538
 539         for (i = 0; i < abm->num_bands; i++) {
 540                 struct tc_gred_vq_qopt_offload_params *band = &opt->set.tab[i];
 541
 542                 if (!band->present)
 543                         return false;
 544                 if (!band->is_ecn && !nfp_abm_has_drop(abm)) {
 545                         nfp_warn(cpp, "GRED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x vq:%d)\n",
 546                                  opt->parent, opt->handle, i);
 547                         return false;
 548                 }
 549                 if (band->is_ecn && !nfp_abm_has_mark(abm)) {
 550                         nfp_warn(cpp, "GRED offload failed - ECN marking not supported (p:%08x h:%08x vq:%d)\n",
 551                                  opt->parent, opt->handle, i);
 552                         return false;
 553                 }
 554                 if (band->is_harddrop) {
 555                         nfp_warn(cpp, "GRED offload failed - harddrop is not supported (p:%08x h:%08x vq:%d)\n",
 556                                  opt->parent, opt->handle, i);
 557                         return false;
 558                 }
 559                 if (band->min != band->max) {
 560                         nfp_warn(cpp, "GRED offload failed - threshold mismatch (p:%08x h:%08x vq:%d)\n",
 561                                  opt->parent, opt->handle, i);
 562                         return false;
 563                 }
 564                 if (band->min > S32_MAX) {
 565                         nfp_warn(cpp, "GRED offload failed - threshold too large %d > %d (p:%08x h:%08x vq:%d)\n",
 566                                  band->min, S32_MAX, opt->parent, opt->handle,
 567                                  i);
 568                         return false;
 569                 }
 570         }
 571
 572         return true;
 573 }
 574
 575 static int
 576 nfp_abm_gred_replace(struct net_device *netdev, struct nfp_abm_link *alink,
 577                      struct tc_gred_qopt_offload *opt)
 578 {
 579         struct nfp_qdisc *qdisc;
 580         unsigned int i;
 581         int ret;
 582
 583         ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_GRED, opt->parent,
 584                                     opt->handle, 0, &qdisc);
 585         if (ret < 0)
 586                 return ret;
 587
 588         qdisc->params_ok = nfp_abm_gred_check_params(alink, opt);
 589         if (qdisc->params_ok) {
 590                 qdisc->red.num_bands = opt->set.dp_cnt;
 591                 for (i = 0; i < qdisc->red.num_bands; i++) {
 592                         qdisc->red.band[i].ecn = opt->set.tab[i].is_ecn;
 593                         qdisc->red.band[i].threshold = opt->set.tab[i].min;
 594                 }
 595         }
 596
 597         if (qdisc->use_cnt)
 598                 nfp_abm_qdisc_offload_update(alink);
 599
 600         return 0;
 601 }
 602
 603 int nfp_abm_setup_tc_gred(struct net_device *netdev, struct nfp_abm_link *alink,
 604                           struct tc_gred_qopt_offload *opt)
 605 {
 606         switch (opt->command) {
 607         case TC_GRED_REPLACE:
 608                 return nfp_abm_gred_replace(netdev, alink, opt);
 609         case TC_GRED_DESTROY:
 610                 nfp_abm_qdisc_destroy(netdev, alink, opt->handle);
 611                 return 0;
 612         case TC_GRED_STATS:
 613                 return nfp_abm_gred_stats(alink, opt->handle, &opt->stats);
 614         default:
 615                 return -EOPNOTSUPP;
 616         }
 617 }
 618
 619 static int
 620 nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt)
 621 {
 622         struct nfp_qdisc *qdisc;
 623
 624         nfp_abm_stats_update(alink);
 625
 626         qdisc = nfp_abm_qdisc_find(alink, opt->handle);
 627         if (!qdisc || !qdisc->offloaded)
 628                 return -EOPNOTSUPP;
 629
 630         nfp_abm_stats_red_calculate(&qdisc->red.band[0].xstats,
 631                                     &qdisc->red.band[0].prev_xstats,
 632                                     opt->xstats);
 633         qdisc->red.band[0].prev_xstats = qdisc->red.band[0].xstats;
 634         return 0;
 635 }
 636
 637 static int
 638 nfp_abm_red_stats(struct nfp_abm_link *alink, u32 handle,
 639                   struct tc_qopt_offload_stats *stats)
 640 {
 641         struct nfp_qdisc *qdisc;
 642
 643         nfp_abm_stats_update(alink);
 644
 645         qdisc = nfp_abm_qdisc_find(alink, handle);
 646         if (!qdisc)
 647                 return -EOPNOTSUPP;
 648         /* If the qdisc offload has stopped we may need to adjust the backlog
 649          * counters back so carry on even if qdisc is not currently offloaded.
 650          */
 651
 652         nfp_abm_stats_calculate(&qdisc->red.band[0].stats,
 653                                 &qdisc->red.band[0].prev_stats,
 654                                 stats->bstats, stats->qstats);
 655         qdisc->red.band[0].prev_stats = qdisc->red.band[0].stats;
 656
 657         return qdisc->offloaded ? 0 : -EOPNOTSUPP;
 658 }
 659
 660 static bool
 661 nfp_abm_red_check_params(struct nfp_abm_link *alink,
 662                          struct tc_red_qopt_offload *opt)
 663 {
 664         struct nfp_cpp *cpp = alink->abm->app->cpp;
 665         struct nfp_abm *abm = alink->abm;
 666
 667         if (!opt->set.is_ecn && !nfp_abm_has_drop(abm)) {
 668                 nfp_warn(cpp, "RED offload failed - drop is not supported (ECN option required) (p:%08x h:%08x)\n",
 669                          opt->parent, opt->handle);
 670                 return false;
 671         }
 672         if (opt->set.is_ecn && !nfp_abm_has_mark(abm)) {
 673                 nfp_warn(cpp, "RED offload failed - ECN marking not supported (p:%08x h:%08x)\n",
 674                          opt->parent, opt->handle);
 675                 return false;
 676         }
 677         if (opt->set.is_harddrop) {
 678                 nfp_warn(cpp, "RED offload failed - harddrop is not supported (p:%08x h:%08x)\n",
 679                          opt->parent, opt->handle);
 680                 return false;
 681         }
 682         if (opt->set.min != opt->set.max) {
 683                 nfp_warn(cpp, "RED offload failed - unsupported min/max parameters (p:%08x h:%08x)\n",
 684                          opt->parent, opt->handle);
 685                 return false;
 686         }
 687         if (opt->set.min > NFP_ABM_LVL_INFINITY) {
 688                 nfp_warn(cpp, "RED offload failed - threshold too large %d > %d (p:%08x h:%08x)\n",
 689                          opt->set.min, NFP_ABM_LVL_INFINITY, opt->parent,
 690                          opt->handle);
 691                 return false;
 692         }
 693
 694         return true;
 695 }
 696
 697 static int
 698 nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink,
 699                     struct tc_red_qopt_offload *opt)
 700 {
 701         struct nfp_qdisc *qdisc;
 702         int ret;
 703
 704         ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_RED, opt->parent,
 705                                     opt->handle, 1, &qdisc);
 706         if (ret < 0)
 707                 return ret;
 708
 709         /* If limit != 0 child gets reset */
 710         if (opt->set.limit) {
 711                 if (nfp_abm_qdisc_child_valid(qdisc, 0))
 712                         qdisc->children[0]->use_cnt--;
 713                 qdisc->children[0] = NULL;
 714         } else {
 715                 /* Qdisc was just allocated without a limit will use noop_qdisc,
 716                  * i.e. a block hole.
 717                  */
 718                 if (!ret)
 719                         qdisc->children[0] = NFP_QDISC_UNTRACKED;
 720         }
 721
 722         qdisc->params_ok = nfp_abm_red_check_params(alink, opt);
 723         if (qdisc->params_ok) {
 724                 qdisc->red.num_bands = 1;
 725                 qdisc->red.band[0].ecn = opt->set.is_ecn;
 726                 qdisc->red.band[0].threshold = opt->set.min;
 727         }
 728
 729         if (qdisc->use_cnt == 1)
 730                 nfp_abm_qdisc_offload_update(alink);
 731
 732         return 0;
 733 }
 734
 735 int nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink,
 736                          struct tc_red_qopt_offload *opt)
 737 {
 738         switch (opt->command) {
 739         case TC_RED_REPLACE:
 740                 return nfp_abm_red_replace(netdev, alink, opt);
 741         case TC_RED_DESTROY:
 742                 nfp_abm_qdisc_destroy(netdev, alink, opt->handle);
 743                 return 0;
 744         case TC_RED_STATS:
 745                 return nfp_abm_red_stats(alink, opt->handle, &opt->stats);
 746         case TC_RED_XSTATS:
 747                 return nfp_abm_red_xstats(alink, opt);
 748         case TC_RED_GRAFT:
 749                 return nfp_abm_qdisc_graft(alink, opt->handle,
 750                                            opt->child_handle, 0);
 751         default:
 752                 return -EOPNOTSUPP;
 753         }
 754 }
 755
 756 static int
 757 nfp_abm_mq_create(struct net_device *netdev, struct nfp_abm_link *alink,
 758                   struct tc_mq_qopt_offload *opt)
 759 {
 760         struct nfp_qdisc *qdisc;
 761         int ret;
 762
 763         ret = nfp_abm_qdisc_replace(netdev, alink, NFP_QDISC_MQ,
 764                                     TC_H_ROOT, opt->handle, alink->total_queues,
 765                                     &qdisc);
 766         if (ret < 0)
 767                 return ret;
 768
 769         qdisc->params_ok = true;
 770         qdisc->offloaded = true;
 771         nfp_abm_qdisc_offload_update(alink);
 772         return 0;
 773 }
 774
 775 static int
 776 nfp_abm_mq_stats(struct nfp_abm_link *alink, u32 handle,
 777                  struct tc_qopt_offload_stats *stats)
 778 {
 779         struct nfp_qdisc *qdisc, *red;
 780         unsigned int i, j;
 781
 782         qdisc = nfp_abm_qdisc_find(alink, handle);
 783         if (!qdisc)
 784                 return -EOPNOTSUPP;
 785
 786         nfp_abm_stats_update(alink);
 787
 788         /* MQ stats are summed over the children in the core, so we need
 789          * to add up the unreported child values.
 790          */
 791         memset(&qdisc->mq.stats, 0, sizeof(qdisc->mq.stats));
 792         memset(&qdisc->mq.prev_stats, 0, sizeof(qdisc->mq.prev_stats));
 793
 794         for (i = 0; i < qdisc->num_children; i++) {
 795                 if (!nfp_abm_qdisc_child_valid(qdisc, i))
 796                         continue;
 797
 798                 if (!nfp_abm_qdisc_is_red(qdisc->children[i]))
 799                         continue;
 800                 red = qdisc->children[i];
 801
 802                 for (j = 0; j < red->red.num_bands; j++) {
 803                         nfp_abm_stats_propagate(&qdisc->mq.stats,
 804                                                 &red->red.band[j].stats);
 805                         nfp_abm_stats_propagate(&qdisc->mq.prev_stats,
 806                                                 &red->red.band[j].prev_stats);
 807                 }
 808         }
 809
 810         nfp_abm_stats_calculate(&qdisc->mq.stats, &qdisc->mq.prev_stats,
 811                                 stats->bstats, stats->qstats);
 812
 813         return qdisc->offloaded ? 0 : -EOPNOTSUPP;
 814 }
 815
 816 int nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink,
 817                         struct tc_mq_qopt_offload *opt)
 818 {
 819         switch (opt->command) {
 820         case TC_MQ_CREATE:
 821                 return nfp_abm_mq_create(netdev, alink, opt);
 822         case TC_MQ_DESTROY:
 823                 nfp_abm_qdisc_destroy(netdev, alink, opt->handle);
 824                 return 0;
 825         case TC_MQ_STATS:
 826                 return nfp_abm_mq_stats(alink, opt->handle, &opt->stats);
 827         case TC_MQ_GRAFT:
 828                 return nfp_abm_qdisc_graft(alink, opt->handle,
 829                                            opt->graft_params.child_handle,
 830                                            opt->graft_params.queue);
 831         default:
 832                 return -EOPNOTSUPP;
 833         }
 834 }
 835
 836 int nfp_abm_setup_root(struct net_device *netdev, struct nfp_abm_link *alink,
 837                        struct tc_root_qopt_offload *opt)
 838 {
 839         if (opt->ingress)
 840                 return -EOPNOTSUPP;
 841         if (alink->root_qdisc)
 842                 alink->root_qdisc->use_cnt--;
 843         alink->root_qdisc = nfp_abm_qdisc_find(alink, opt->handle);
 844         if (alink->root_qdisc)
 845                 alink->root_qdisc->use_cnt++;
 846
 847         nfp_abm_qdisc_offload_update(alink);
 848
 849         return 0;
 850 }