usr/src/uts/common/io/aggr/aggr_send.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25
  26 /*
  27  * IEEE 802.3ad Link Aggregation - Send code.
  28  *
  29  * Implements the Distributor function.
  30  */
  31
  32 #include <sys/conf.h>
  33 #include <sys/modctl.h>
  34 #include <sys/sunddi.h>
  35 #include <sys/callb.h>
  36 #include <sys/vlan.h>
  37 #include <sys/strsun.h>
  38 #include <sys/strsubr.h>
  39 #include <sys/dlpi.h>
  40
  41 #include <inet/common.h>
  42 #include <inet/led.h>
  43 #include <inet/ip.h>
  44 #include <inet/ip6.h>
  45 #include <inet/tcp.h>
  46 #include <netinet/udp.h>
  47
  48 #include <sys/aggr.h>
  49 #include <sys/aggr_impl.h>
  50
  51 /*
  52  * Update the TX load balancing policy of the specified group.
  53  */
  54 void
  55 aggr_send_update_policy(aggr_grp_t *grp, uint32_t policy)
  56 {
  57         uint8_t mac_policy = 0;
  58
  59         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
  60
  61         if ((policy & AGGR_POLICY_L2) != 0)
  62                 mac_policy |= MAC_PKT_HASH_L2;
  63         if ((policy & AGGR_POLICY_L3) != 0)
  64                 mac_policy |= MAC_PKT_HASH_L3;
  65         if ((policy & AGGR_POLICY_L4) != 0)
  66                 mac_policy |= MAC_PKT_HASH_L4;
  67
  68         grp->lg_tx_policy = policy;
  69         grp->lg_mac_tx_policy = mac_policy;
  70 }
  71
  72 #define HASH_HINT(hint) \
  73         ((hint) ^ ((hint) >> 24) ^ ((hint) >> 16) ^ ((hint) >> 8))
  74
  75 /*
  76  * Function invoked by mac layer to find a specific TX ring on a port
  77  * to send data.
  78  */
  79 mblk_t *
  80 aggr_find_tx_ring(void *arg, mblk_t *mp, uintptr_t hint, mac_ring_handle_t *rh)
  81 {
  82         aggr_grp_t *grp = arg;
  83         aggr_port_t *port;
  84         uint64_t hash;
  85
  86         rw_enter(&grp->lg_tx_lock, RW_READER);
  87         if (grp->lg_ntx_ports == 0) {
  88                 /*
  89                  * We could have returned from aggr_m_start() before
  90                  * the ports were actually attached. Drop the chain.
  91                  */
  92                 rw_exit(&grp->lg_tx_lock);
  93                 freemsgchain(mp);
  94                 return (NULL);
  95         }
  96         hash = mac_pkt_hash(DL_ETHER, mp, grp->lg_mac_tx_policy, B_TRUE);
  97         port = grp->lg_tx_ports[hash % grp->lg_ntx_ports];
  98
  99         /*
 100          * Use hash as the hint so to direct traffic to
 101          * different TX rings. Note below bit operation
 102          * is needed in case hint is 0 to get the most
 103          * benefit from HASH_HINT() algorithm.
 104          */
 105         if (port->lp_tx_ring_cnt > 1) {
 106                 if (hint == 0) {
 107                         hash = (hash << 24 | hash << 16 | hash);
 108                         hash = (hash << 32 | hash);
 109                 } else {
 110                         hash = hint;
 111                 }
 112                 hash = HASH_HINT(hash);
 113                 *rh = port->lp_pseudo_tx_rings[hash % port->lp_tx_ring_cnt];
 114         } else {
 115                 *rh = port->lp_pseudo_tx_rings[0];
 116         }
 117         rw_exit(&grp->lg_tx_lock);
 118
 119         return (mp);
 120 }
 121
 122 /*
 123  * aggr_tx_notify_thread:
 124  *
 125  * aggr_tx_ring_update() callback function wakes up this thread when
 126  * it gets called. This thread will call mac_tx_ring_update() to
 127  * notify upper mac of flow control getting relieved. Note that
 128  * aggr_tx_ring_update() cannot call mac_tx_ring_update() directly
 129  * because aggr_tx_ring_update() is called from lower mac with
 130  * mi_rw_lock held.
 131  */
 132 void
 133 aggr_tx_notify_thread(void *arg)
 134 {
 135         callb_cpr_t     cprinfo;
 136         aggr_grp_t      *grp = (aggr_grp_t *)arg;
 137         mac_ring_handle_t       pseudo_mrh;
 138
 139         CALLB_CPR_INIT(&cprinfo, &grp->lg_tx_flowctl_lock, callb_generic_cpr,
 140             "aggr_tx_notify_thread");
 141
 142         mutex_enter(&grp->lg_tx_flowctl_lock);
 143         while (!grp->lg_tx_notify_done) {
 144                 if ((grp->lg_tx_blocked_cnt) == 0) {
 145                         CALLB_CPR_SAFE_BEGIN(&cprinfo);
 146                         cv_wait(&grp->lg_tx_flowctl_cv,
 147                             &grp->lg_tx_flowctl_lock);
 148                         CALLB_CPR_SAFE_END(&cprinfo, &grp->lg_tx_flowctl_lock);
 149                         continue;
 150                 }
 151                 while (grp->lg_tx_blocked_cnt != 0) {
 152                         grp->lg_tx_blocked_cnt--;
 153                         pseudo_mrh =
 154                             grp->lg_tx_blocked_rings[grp->lg_tx_blocked_cnt];
 155                         mutex_exit(&grp->lg_tx_flowctl_lock);
 156                         mac_tx_ring_update(grp->lg_mh, pseudo_mrh);
 157                         mutex_enter(&grp->lg_tx_flowctl_lock);
 158                 }
 159         }
 160         /*
 161          * The grp is being destroyed, exit the thread.
 162          */
 163         grp->lg_tx_notify_thread = NULL;
 164         CALLB_CPR_EXIT(&cprinfo);
 165         thread_exit();
 166 }
 167
 168 /*
 169  * Callback function registered with lower mac to receive wakeups from
 170  * drivers when flow control is relieved (i.e. Tx descriptors are
 171  * available).
 172  */
 173 void
 174 aggr_tx_ring_update(void *arg1, uintptr_t arg2)
 175 {
 176         aggr_port_t *port = (aggr_port_t *)arg1;
 177         mac_ring_handle_t mrh = (mac_ring_handle_t)arg2;
 178         mac_ring_handle_t pseudo_mrh;
 179         aggr_grp_t *grp = port->lp_grp;
 180         int i = 0;
 181
 182         if (mrh == NULL) {
 183                 /*
 184                  * If the underlying NIC does not expose TX rings,
 185                  * still as pseudo TX ring is presented to the
 186                  * aggr mac.
 187                  */
 188                 pseudo_mrh = port->lp_pseudo_tx_rings[0];
 189         } else {
 190                 for (i = 0; i < port->lp_tx_ring_cnt; i++) {
 191                         if (port->lp_tx_rings[i] == mrh)
 192                                 break;
 193                 }
 194                 ASSERT(i < port->lp_tx_ring_cnt);
 195                 pseudo_mrh = port->lp_pseudo_tx_rings[i];
 196         }
 197         mutex_enter(&grp->lg_tx_flowctl_lock);
 198         /*
 199          * It could be possible that some (broken?) device driver
 200          * could send more than one wakeup on the same ring. In
 201          * such a case, multiple instances of the same pseudo TX
 202          * ring should not be saved in lg_tx_blocked_rings[]
 203          * array. So first check if woken up ring (pseudo_mrh) is
 204          * already in the lg_tx_blocked_rings[] array.
 205          */
 206         for (i = 0; i < grp->lg_tx_blocked_cnt; i++) {
 207                 if (grp->lg_tx_blocked_rings[i] == pseudo_mrh) {
 208                         mutex_exit(&grp->lg_tx_flowctl_lock);
 209                         return;
 210                 }
 211         }
 212         /* A distinct mac_ring_handle. Save and increment count */
 213         grp->lg_tx_blocked_rings[grp->lg_tx_blocked_cnt] = pseudo_mrh;
 214         grp->lg_tx_blocked_cnt++;
 215         cv_signal(&grp->lg_tx_flowctl_cv);
 216         mutex_exit(&grp->lg_tx_flowctl_lock);
 217 }
 218
 219 /*
 220  * Send function invoked by the MAC service module.
 221  */
 222 mblk_t *
 223 aggr_ring_tx(void *arg, mblk_t *mp)
 224 {
 225         aggr_pseudo_tx_ring_t *pseudo_ring = (aggr_pseudo_tx_ring_t *)arg;
 226         aggr_port_t *port = pseudo_ring->atr_port;
 227
 228         return (mac_hwring_send_priv(port->lp_mch, pseudo_ring->atr_hw_rh, mp));
 229 }
 230
 231 /*
 232  * Enable sending on the specified port.
 233  */
 234 void
 235 aggr_send_port_enable(aggr_port_t *port)
 236 {
 237         aggr_grp_t *grp = port->lp_grp;
 238
 239         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
 240
 241         if (port->lp_tx_enabled || (port->lp_state !=
 242             AGGR_PORT_STATE_ATTACHED)) {
 243                 /* already enabled or port not yet attached */
 244                 return;
 245         }
 246
 247         /*
 248          * Add to group's array of tx ports.
 249          */
 250         rw_enter(&grp->lg_tx_lock, RW_WRITER);
 251         if (grp->lg_tx_ports_size < grp->lg_ntx_ports+1) {
 252                 /* current array too small */
 253                 aggr_port_t **new_ports;
 254                 uint_t new_size;
 255
 256                 new_size = grp->lg_ntx_ports+1;
 257                 new_ports = kmem_zalloc(new_size * sizeof (aggr_port_t *),
 258                     KM_SLEEP);
 259
 260                 if (grp->lg_tx_ports_size > 0) {
 261                         ASSERT(grp->lg_tx_ports != NULL);
 262                         bcopy(grp->lg_tx_ports, new_ports,
 263                             grp->lg_ntx_ports * sizeof (aggr_port_t *));
 264                         kmem_free(grp->lg_tx_ports,
 265                             grp->lg_tx_ports_size * sizeof (aggr_port_t *));
 266                 }
 267
 268                 grp->lg_tx_ports = new_ports;
 269                 grp->lg_tx_ports_size = new_size;
 270         }
 271
 272         grp->lg_tx_ports[grp->lg_ntx_ports++] = port;
 273         port->lp_tx_idx = grp->lg_ntx_ports-1;
 274         rw_exit(&grp->lg_tx_lock);
 275
 276         port->lp_tx_enabled = B_TRUE;
 277
 278         aggr_grp_update_default(grp);
 279 }
 280
 281 /*
 282  * Disable sending from the specified port.
 283  */
 284 void
 285 aggr_send_port_disable(aggr_port_t *port)
 286 {
 287         uint_t idx, ntx;
 288         aggr_grp_t *grp = port->lp_grp;
 289
 290         ASSERT(MAC_PERIM_HELD(grp->lg_mh));
 291         ASSERT(MAC_PERIM_HELD(port->lp_mh));
 292
 293         if (!port->lp_tx_enabled) {
 294                 /* not yet enabled */
 295                 return;
 296         }
 297
 298         rw_enter(&grp->lg_tx_lock, RW_WRITER);
 299         idx = port->lp_tx_idx;
 300         ntx = grp->lg_ntx_ports;
 301         ASSERT(idx < ntx);
 302
 303         /* remove from array of attached ports */
 304         if (idx == (ntx - 1)) {
 305                 grp->lg_tx_ports[idx] = NULL;
 306         } else {
 307                 /* not the last entry, replace with last one */
 308                 aggr_port_t *victim;
 309
 310                 victim = grp->lg_tx_ports[ntx - 1];
 311                 grp->lg_tx_ports[ntx - 1] = NULL;
 312                 victim->lp_tx_idx = idx;
 313                 grp->lg_tx_ports[idx] = victim;
 314         }
 315
 316         port->lp_tx_idx = 0;
 317         grp->lg_ntx_ports--;
 318         rw_exit(&grp->lg_tx_lock);
 319
 320         port->lp_tx_enabled = B_FALSE;
 321
 322         aggr_grp_update_default(grp);
 323 }