drivers/net/team/team_mode_loadbalance.c

   1 /*
   2  * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team
   3  * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/types.h>
  13 #include <linux/module.h>
  14 #include <linux/init.h>
  15 #include <linux/errno.h>
  16 #include <linux/netdevice.h>
  17 #include <linux/etherdevice.h>
  18 #include <linux/filter.h>
  19 #include <linux/if_team.h>
  20
  21 static rx_handler_result_t lb_receive(struct team *team, struct team_port *port,
  22                                       struct sk_buff *skb)
  23 {
  24         if (unlikely(skb->protocol == htons(ETH_P_SLOW))) {
  25                 /* LACPDU packets should go to exact delivery */
  26                 const unsigned char *dest = eth_hdr(skb)->h_dest;
  27
  28                 if (is_link_local_ether_addr(dest) && dest[5] == 0x02)
  29                         return RX_HANDLER_EXACT;
  30         }
  31         return RX_HANDLER_ANOTHER;
  32 }
  33
  34 struct lb_priv;
  35
  36 typedef struct team_port *lb_select_tx_port_func_t(struct team *,
  37                                                    struct lb_priv *,
  38                                                    struct sk_buff *,
  39                                                    unsigned char);
  40
  41 #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */
  42
  43 struct lb_stats {
  44         u64 tx_bytes;
  45 };
  46
  47 struct lb_pcpu_stats {
  48         struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE];
  49         struct u64_stats_sync syncp;
  50 };
  51
  52 struct lb_stats_info {
  53         struct lb_stats stats;
  54         struct lb_stats last_stats;
  55         struct team_option_inst_info *opt_inst_info;
  56 };
  57
  58 struct lb_port_mapping {
  59         struct team_port __rcu *port;
  60         struct team_option_inst_info *opt_inst_info;
  61 };
  62
  63 struct lb_priv_ex {
  64         struct team *team;
  65         struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE];
  66         struct sock_fprog_kern *orig_fprog;
  67         struct {
  68                 unsigned int refresh_interval; /* in tenths of second */
  69                 struct delayed_work refresh_dw;
  70                 struct lb_stats_info info[LB_TX_HASHTABLE_SIZE];
  71         } stats;
  72 };
  73
  74 struct lb_priv {
  75         struct bpf_prog __rcu *fp;
  76         lb_select_tx_port_func_t __rcu *select_tx_port_func;
  77         struct lb_pcpu_stats __percpu *pcpu_stats;
  78         struct lb_priv_ex *ex; /* priv extension */
  79 };
  80
  81 static struct lb_priv *get_lb_priv(struct team *team)
  82 {
  83         return (struct lb_priv *) &team->mode_priv;
  84 }
  85
  86 struct lb_port_priv {
  87         struct lb_stats __percpu *pcpu_stats;
  88         struct lb_stats_info stats_info;
  89 };
  90
  91 static struct lb_port_priv *get_lb_port_priv(struct team_port *port)
  92 {
  93         return (struct lb_port_priv *) &port->mode_priv;
  94 }
  95
  96 #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \
  97         (lb_priv)->ex->tx_hash_to_port_mapping[hash].port
  98
  99 #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \
 100         (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info
 101
 102 static void lb_tx_hash_to_port_mapping_null_port(struct team *team,
 103                                                  struct team_port *port)
 104 {
 105         struct lb_priv *lb_priv = get_lb_priv(team);
 106         bool changed = false;
 107         int i;
 108
 109         for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) {
 110                 struct lb_port_mapping *pm;
 111
 112                 pm = &lb_priv->ex->tx_hash_to_port_mapping[i];
 113                 if (rcu_access_pointer(pm->port) == port) {
 114                         RCU_INIT_POINTER(pm->port, NULL);
 115                         team_option_inst_set_change(pm->opt_inst_info);
 116                         changed = true;
 117                 }
 118         }
 119         if (changed)
 120                 team_options_change_check(team);
 121 }
 122
 123 /* Basic tx selection based solely by hash */
 124 static struct team_port *lb_hash_select_tx_port(struct team *team,
 125                                                 struct lb_priv *lb_priv,
 126                                                 struct sk_buff *skb,
 127                                                 unsigned char hash)
 128 {
 129         int port_index = team_num_to_port_index(team, hash);
 130
 131         return team_get_port_by_index_rcu(team, port_index);
 132 }
 133
 134 /* Hash to port mapping select tx port */
 135 static struct team_port *lb_htpm_select_tx_port(struct team *team,
 136                                                 struct lb_priv *lb_priv,
 137                                                 struct sk_buff *skb,
 138                                                 unsigned char hash)
 139 {
 140         struct team_port *port;
 141
 142         port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash));
 143         if (likely(port))
 144                 return port;
 145         /* If no valid port in the table, fall back to simple hash */
 146         return lb_hash_select_tx_port(team, lb_priv, skb, hash);
 147 }
 148
 149 struct lb_select_tx_port {
 150         char *name;
 151         lb_select_tx_port_func_t *func;
 152 };
 153
 154 static const struct lb_select_tx_port lb_select_tx_port_list[] = {
 155         {
 156                 .name = "hash",
 157                 .func = lb_hash_select_tx_port,
 158         },
 159         {
 160                 .name = "hash_to_port_mapping",
 161                 .func = lb_htpm_select_tx_port,
 162         },
 163 };
 164 #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list)
 165
 166 static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func)
 167 {
 168         int i;
 169
 170         for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
 171                 const struct lb_select_tx_port *item;
 172
 173                 item = &lb_select_tx_port_list[i];
 174                 if (item->func == func)
 175                         return item->name;
 176         }
 177         return NULL;
 178 }
 179
 180 static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name)
 181 {
 182         int i;
 183
 184         for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
 185                 const struct lb_select_tx_port *item;
 186
 187                 item = &lb_select_tx_port_list[i];
 188                 if (!strcmp(item->name, name))
 189                         return item->func;
 190         }
 191         return NULL;
 192 }
 193
 194 static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv,
 195                                     struct sk_buff *skb)
 196 {
 197         struct bpf_prog *fp;
 198         uint32_t lhash;
 199         unsigned char *c;
 200
 201         fp = rcu_dereference_bh(lb_priv->fp);
 202         if (unlikely(!fp))
 203                 return 0;
 204         lhash = BPF_PROG_RUN(fp, skb);
 205         c = (char *) &lhash;
 206         return c[0] ^ c[1] ^ c[2] ^ c[3];
 207 }
 208
 209 static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv,
 210                                struct lb_port_priv *lb_port_priv,
 211                                unsigned char hash)
 212 {
 213         struct lb_pcpu_stats *pcpu_stats;
 214         struct lb_stats *port_stats;
 215         struct lb_stats *hash_stats;
 216
 217         pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats);
 218         port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats);
 219         hash_stats = &pcpu_stats->hash_stats[hash];
 220         u64_stats_update_begin(&pcpu_stats->syncp);
 221         port_stats->tx_bytes += tx_bytes;
 222         hash_stats->tx_bytes += tx_bytes;
 223         u64_stats_update_end(&pcpu_stats->syncp);
 224 }
 225
 226 static bool lb_transmit(struct team *team, struct sk_buff *skb)
 227 {
 228         struct lb_priv *lb_priv = get_lb_priv(team);
 229         lb_select_tx_port_func_t *select_tx_port_func;
 230         struct team_port *port;
 231         unsigned char hash;
 232         unsigned int tx_bytes = skb->len;
 233
 234         hash = lb_get_skb_hash(lb_priv, skb);
 235         select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func);
 236         port = select_tx_port_func(team, lb_priv, skb, hash);
 237         if (unlikely(!port))
 238                 goto drop;
 239         if (team_dev_queue_xmit(team, port, skb))
 240                 return false;
 241         lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash);
 242         return true;
 243
 244 drop:
 245         dev_kfree_skb_any(skb);
 246         return false;
 247 }
 248
 249 static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx)
 250 {
 251         struct lb_priv *lb_priv = get_lb_priv(team);
 252
 253         if (!lb_priv->ex->orig_fprog) {
 254                 ctx->data.bin_val.len = 0;
 255                 ctx->data.bin_val.ptr = NULL;
 256                 return 0;
 257         }
 258         ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len *
 259                                 sizeof(struct sock_filter);
 260         ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter;
 261         return 0;
 262 }
 263
 264 static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len,
 265                           const void *data)
 266 {
 267         struct sock_fprog_kern *fprog;
 268         struct sock_filter *filter = (struct sock_filter *) data;
 269
 270         if (data_len % sizeof(struct sock_filter))
 271                 return -EINVAL;
 272         fprog = kmalloc(sizeof(*fprog), GFP_KERNEL);
 273         if (!fprog)
 274                 return -ENOMEM;
 275         fprog->filter = kmemdup(filter, data_len, GFP_KERNEL);
 276         if (!fprog->filter) {
 277                 kfree(fprog);
 278                 return -ENOMEM;
 279         }
 280         fprog->len = data_len / sizeof(struct sock_filter);
 281         *pfprog = fprog;
 282         return 0;
 283 }
 284
 285 static void __fprog_destroy(struct sock_fprog_kern *fprog)
 286 {
 287         kfree(fprog->filter);
 288         kfree(fprog);
 289 }
 290
 291 static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx)
 292 {
 293         struct lb_priv *lb_priv = get_lb_priv(team);
 294         struct bpf_prog *fp = NULL;
 295         struct bpf_prog *orig_fp = NULL;
 296         struct sock_fprog_kern *fprog = NULL;
 297         int err;
 298
 299         if (ctx->data.bin_val.len) {
 300                 err = __fprog_create(&fprog, ctx->data.bin_val.len,
 301                                      ctx->data.bin_val.ptr);
 302                 if (err)
 303                         return err;
 304                 err = bpf_prog_create(&fp, fprog);
 305                 if (err) {
 306                         __fprog_destroy(fprog);
 307                         return err;
 308                 }
 309         }
 310
 311         if (lb_priv->ex->orig_fprog) {
 312                 /* Clear old filter data */
 313                 __fprog_destroy(lb_priv->ex->orig_fprog);
 314                 orig_fp = rcu_dereference_protected(lb_priv->fp,
 315                                                 lockdep_is_held(&team->lock));
 316         }
 317
 318         rcu_assign_pointer(lb_priv->fp, fp);
 319         lb_priv->ex->orig_fprog = fprog;
 320
 321         if (orig_fp) {
 322                 synchronize_rcu();
 323                 bpf_prog_destroy(orig_fp);
 324         }
 325         return 0;
 326 }
 327
 328 static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx)
 329 {
 330         struct lb_priv *lb_priv = get_lb_priv(team);
 331         lb_select_tx_port_func_t *func;
 332         char *name;
 333
 334         func = rcu_dereference_protected(lb_priv->select_tx_port_func,
 335                                          lockdep_is_held(&team->lock));
 336         name = lb_select_tx_port_get_name(func);
 337         BUG_ON(!name);
 338         ctx->data.str_val = name;
 339         return 0;
 340 }
 341
 342 static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx)
 343 {
 344         struct lb_priv *lb_priv = get_lb_priv(team);
 345         lb_select_tx_port_func_t *func;
 346
 347         func = lb_select_tx_port_get_func(ctx->data.str_val);
 348         if (!func)
 349                 return -EINVAL;
 350         rcu_assign_pointer(lb_priv->select_tx_port_func, func);
 351         return 0;
 352 }
 353
 354 static int lb_tx_hash_to_port_mapping_init(struct team *team,
 355                                            struct team_option_inst_info *info)
 356 {
 357         struct lb_priv *lb_priv = get_lb_priv(team);
 358         unsigned char hash = info->array_index;
 359
 360         LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info;
 361         return 0;
 362 }
 363
 364 static int lb_tx_hash_to_port_mapping_get(struct team *team,
 365                                           struct team_gsetter_ctx *ctx)
 366 {
 367         struct lb_priv *lb_priv = get_lb_priv(team);
 368         struct team_port *port;
 369         unsigned char hash = ctx->info->array_index;
 370
 371         port = LB_HTPM_PORT_BY_HASH(lb_priv, hash);
 372         ctx->data.u32_val = port ? port->dev->ifindex : 0;
 373         return 0;
 374 }
 375
 376 static int lb_tx_hash_to_port_mapping_set(struct team *team,
 377                                           struct team_gsetter_ctx *ctx)
 378 {
 379         struct lb_priv *lb_priv = get_lb_priv(team);
 380         struct team_port *port;
 381         unsigned char hash = ctx->info->array_index;
 382
 383         list_for_each_entry(port, &team->port_list, list) {
 384                 if (ctx->data.u32_val == port->dev->ifindex &&
 385                     team_port_enabled(port)) {
 386                         rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash),
 387                                            port);
 388                         return 0;
 389                 }
 390         }
 391         return -ENODEV;
 392 }
 393
 394 static int lb_hash_stats_init(struct team *team,
 395                               struct team_option_inst_info *info)
 396 {
 397         struct lb_priv *lb_priv = get_lb_priv(team);
 398         unsigned char hash = info->array_index;
 399
 400         lb_priv->ex->stats.info[hash].opt_inst_info = info;
 401         return 0;
 402 }
 403
 404 static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
 405 {
 406         struct lb_priv *lb_priv = get_lb_priv(team);
 407         unsigned char hash = ctx->info->array_index;
 408
 409         ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats;
 410         ctx->data.bin_val.len = sizeof(struct lb_stats);
 411         return 0;
 412 }
 413
 414 static int lb_port_stats_init(struct team *team,
 415                               struct team_option_inst_info *info)
 416 {
 417         struct team_port *port = info->port;
 418         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 419
 420         lb_port_priv->stats_info.opt_inst_info = info;
 421         return 0;
 422 }
 423
 424 static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
 425 {
 426         struct team_port *port = ctx->info->port;
 427         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 428
 429         ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats;
 430         ctx->data.bin_val.len = sizeof(struct lb_stats);
 431         return 0;
 432 }
 433
 434 static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info)
 435 {
 436         memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats));
 437         memset(&s_info->stats, 0, sizeof(struct lb_stats));
 438 }
 439
 440 static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info,
 441                                           struct team *team)
 442 {
 443         if (memcmp(&s_info->last_stats, &s_info->stats,
 444             sizeof(struct lb_stats))) {
 445                 team_option_inst_set_change(s_info->opt_inst_info);
 446                 return true;
 447         }
 448         return false;
 449 }
 450
 451 static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats,
 452                                    struct lb_stats *cpu_stats,
 453                                    struct u64_stats_sync *syncp)
 454 {
 455         unsigned int start;
 456         struct lb_stats tmp;
 457
 458         do {
 459                 start = u64_stats_fetch_begin_irq(syncp);
 460                 tmp.tx_bytes = cpu_stats->tx_bytes;
 461         } while (u64_stats_fetch_retry_irq(syncp, start));
 462         acc_stats->tx_bytes += tmp.tx_bytes;
 463 }
 464
 465 static void lb_stats_refresh(struct work_struct *work)
 466 {
 467         struct team *team;
 468         struct lb_priv *lb_priv;
 469         struct lb_priv_ex *lb_priv_ex;
 470         struct lb_pcpu_stats *pcpu_stats;
 471         struct lb_stats *stats;
 472         struct lb_stats_info *s_info;
 473         struct team_port *port;
 474         bool changed = false;
 475         int i;
 476         int j;
 477
 478         lb_priv_ex = container_of(work, struct lb_priv_ex,
 479                                   stats.refresh_dw.work);
 480
 481         team = lb_priv_ex->team;
 482         lb_priv = get_lb_priv(team);
 483
 484         if (!mutex_trylock(&team->lock)) {
 485                 schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0);
 486                 return;
 487         }
 488
 489         for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) {
 490                 s_info = &lb_priv->ex->stats.info[j];
 491                 __lb_stats_info_refresh_prepare(s_info);
 492                 for_each_possible_cpu(i) {
 493                         pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 494                         stats = &pcpu_stats->hash_stats[j];
 495                         __lb_one_cpu_stats_add(&s_info->stats, stats,
 496                                                &pcpu_stats->syncp);
 497                 }
 498                 changed |= __lb_stats_info_refresh_check(s_info, team);
 499         }
 500
 501         list_for_each_entry(port, &team->port_list, list) {
 502                 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 503
 504                 s_info = &lb_port_priv->stats_info;
 505                 __lb_stats_info_refresh_prepare(s_info);
 506                 for_each_possible_cpu(i) {
 507                         pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 508                         stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i);
 509                         __lb_one_cpu_stats_add(&s_info->stats, stats,
 510                                                &pcpu_stats->syncp);
 511                 }
 512                 changed |= __lb_stats_info_refresh_check(s_info, team);
 513         }
 514
 515         if (changed)
 516                 team_options_change_check(team);
 517
 518         schedule_delayed_work(&lb_priv_ex->stats.refresh_dw,
 519                               (lb_priv_ex->stats.refresh_interval * HZ) / 10);
 520
 521         mutex_unlock(&team->lock);
 522 }
 523
 524 static int lb_stats_refresh_interval_get(struct team *team,
 525                                          struct team_gsetter_ctx *ctx)
 526 {
 527         struct lb_priv *lb_priv = get_lb_priv(team);
 528
 529         ctx->data.u32_val = lb_priv->ex->stats.refresh_interval;
 530         return 0;
 531 }
 532
 533 static int lb_stats_refresh_interval_set(struct team *team,
 534                                          struct team_gsetter_ctx *ctx)
 535 {
 536         struct lb_priv *lb_priv = get_lb_priv(team);
 537         unsigned int interval;
 538
 539         interval = ctx->data.u32_val;
 540         if (lb_priv->ex->stats.refresh_interval == interval)
 541                 return 0;
 542         lb_priv->ex->stats.refresh_interval = interval;
 543         if (interval)
 544                 schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0);
 545         else
 546                 cancel_delayed_work(&lb_priv->ex->stats.refresh_dw);
 547         return 0;
 548 }
 549
 550 static const struct team_option lb_options[] = {
 551         {
 552                 .name = "bpf_hash_func",
 553                 .type = TEAM_OPTION_TYPE_BINARY,
 554                 .getter = lb_bpf_func_get,
 555                 .setter = lb_bpf_func_set,
 556         },
 557         {
 558                 .name = "lb_tx_method",
 559                 .type = TEAM_OPTION_TYPE_STRING,
 560                 .getter = lb_tx_method_get,
 561                 .setter = lb_tx_method_set,
 562         },
 563         {
 564                 .name = "lb_tx_hash_to_port_mapping",
 565                 .array_size = LB_TX_HASHTABLE_SIZE,
 566                 .type = TEAM_OPTION_TYPE_U32,
 567                 .init = lb_tx_hash_to_port_mapping_init,
 568                 .getter = lb_tx_hash_to_port_mapping_get,
 569                 .setter = lb_tx_hash_to_port_mapping_set,
 570         },
 571         {
 572                 .name = "lb_hash_stats",
 573                 .array_size = LB_TX_HASHTABLE_SIZE,
 574                 .type = TEAM_OPTION_TYPE_BINARY,
 575                 .init = lb_hash_stats_init,
 576                 .getter = lb_hash_stats_get,
 577         },
 578         {
 579                 .name = "lb_port_stats",
 580                 .per_port = true,
 581                 .type = TEAM_OPTION_TYPE_BINARY,
 582                 .init = lb_port_stats_init,
 583                 .getter = lb_port_stats_get,
 584         },
 585         {
 586                 .name = "lb_stats_refresh_interval",
 587                 .type = TEAM_OPTION_TYPE_U32,
 588                 .getter = lb_stats_refresh_interval_get,
 589                 .setter = lb_stats_refresh_interval_set,
 590         },
 591 };
 592
 593 static int lb_init(struct team *team)
 594 {
 595         struct lb_priv *lb_priv = get_lb_priv(team);
 596         lb_select_tx_port_func_t *func;
 597         int i, err;
 598
 599         /* set default tx port selector */
 600         func = lb_select_tx_port_get_func("hash");
 601         BUG_ON(!func);
 602         rcu_assign_pointer(lb_priv->select_tx_port_func, func);
 603
 604         lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL);
 605         if (!lb_priv->ex)
 606                 return -ENOMEM;
 607         lb_priv->ex->team = team;
 608
 609         lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats);
 610         if (!lb_priv->pcpu_stats) {
 611                 err = -ENOMEM;
 612                 goto err_alloc_pcpu_stats;
 613         }
 614
 615         for_each_possible_cpu(i) {
 616                 struct lb_pcpu_stats *team_lb_stats;
 617                 team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
 618                 u64_stats_init(&team_lb_stats->syncp);
 619         }
 620
 621
 622         INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh);
 623
 624         err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options));
 625         if (err)
 626                 goto err_options_register;
 627         return 0;
 628
 629 err_options_register:
 630         free_percpu(lb_priv->pcpu_stats);
 631 err_alloc_pcpu_stats:
 632         kfree(lb_priv->ex);
 633         return err;
 634 }
 635
 636 static void lb_exit(struct team *team)
 637 {
 638         struct lb_priv *lb_priv = get_lb_priv(team);
 639
 640         team_options_unregister(team, lb_options,
 641                                 ARRAY_SIZE(lb_options));
 642         cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw);
 643         free_percpu(lb_priv->pcpu_stats);
 644         kfree(lb_priv->ex);
 645 }
 646
 647 static int lb_port_enter(struct team *team, struct team_port *port)
 648 {
 649         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 650
 651         lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats);
 652         if (!lb_port_priv->pcpu_stats)
 653                 return -ENOMEM;
 654         return 0;
 655 }
 656
 657 static void lb_port_leave(struct team *team, struct team_port *port)
 658 {
 659         struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
 660
 661         free_percpu(lb_port_priv->pcpu_stats);
 662 }
 663
 664 static void lb_port_disabled(struct team *team, struct team_port *port)
 665 {
 666         lb_tx_hash_to_port_mapping_null_port(team, port);
 667 }
 668
 669 static const struct team_mode_ops lb_mode_ops = {
 670         .init                   = lb_init,
 671         .exit                   = lb_exit,
 672         .port_enter             = lb_port_enter,
 673         .port_leave             = lb_port_leave,
 674         .port_disabled          = lb_port_disabled,
 675         .receive                = lb_receive,
 676         .transmit               = lb_transmit,
 677 };
 678
 679 static const struct team_mode lb_mode = {
 680         .kind           = "loadbalance",
 681         .owner          = THIS_MODULE,
 682         .priv_size      = sizeof(struct lb_priv),
 683         .port_priv_size = sizeof(struct lb_port_priv),
 684         .ops            = &lb_mode_ops,
 685         .lag_tx_type    = NETDEV_LAG_TX_TYPE_HASH,
 686 };
 687
 688 static int __init lb_init_module(void)
 689 {
 690         return team_mode_register(&lb_mode);
 691 }
 692
 693 static void __exit lb_cleanup_module(void)
 694 {
 695         team_mode_unregister(&lb_mode);
 696 }
 697
 698 module_init(lb_init_module);
 699 module_exit(lb_cleanup_module);
 700
 701 MODULE_LICENSE("GPL v2");
 702 MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>");
 703 MODULE_DESCRIPTION("Load-balancing mode for team");
 704 MODULE_ALIAS_TEAM_MODE("loadbalance");