include/sys/mac_soft_ring.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26
  27 #ifndef _SYS_MAC_SOFT_RING_H
  28 #define _SYS_MAC_SOFT_RING_H
  29
  30 #ifdef  __cplusplus
  31 extern "C" {
  32 #endif
  33
  34 #include <sys/types.h>
  35 #include <sys/cpuvar.h>
  36 #include <sys/cpupart.h>
  37 #include <sys/processor.h>
  38 #include <sys/stream.h>
  39 #include <sys/squeue.h>
  40 #include <sys/dlpi.h>
  41 #include <sys/mac_impl.h>
  42 #include <sys/mac_stat.h>
  43
  44 #define S_RING_NAMELEN 64
  45
  46 #define MAX_SR_FANOUT   24
  47
  48 extern boolean_t mac_soft_ring_enable;
  49 extern boolean_t mac_latency_optimize;
  50
  51 typedef struct mac_soft_ring_s mac_soft_ring_t;
  52 typedef struct mac_soft_ring_set_s mac_soft_ring_set_t;
  53
  54 typedef void (*mac_soft_ring_drain_func_t)(mac_soft_ring_t *);
  55 typedef mac_tx_cookie_t (*mac_tx_func_t)(mac_soft_ring_set_t *, mblk_t *,
  56     uintptr_t, uint16_t, mblk_t **);
  57
  58
  59 /* Tx notify callback */
  60 typedef struct mac_tx_notify_cb_s {
  61         mac_cb_t                mtnf_link;      /* Linked list of callbacks */
  62         mac_tx_notify_t         mtnf_fn;        /* The callback function */
  63         void                    *mtnf_arg;      /* Callback function argument */
  64 } mac_tx_notify_cb_t;
  65
  66 struct mac_soft_ring_s {
  67         /* Keep the most used members 64bytes cache aligned */
  68         kmutex_t        s_ring_lock;    /* lock before using any member */
  69         uint16_t        s_ring_type;    /* processing model of the sq */
  70         uint16_t        s_ring_state;   /* state flags and message count */
  71         int             s_ring_count;   /* # of mblocks in mac_soft_ring */
  72         size_t          s_ring_size;    /* Size of data queued */
  73         mblk_t          *s_ring_first;  /* first mblk chain or NULL */
  74         mblk_t          *s_ring_last;   /* last mblk chain or NULL */
  75
  76         mac_direct_rx_t s_ring_rx_func;
  77         void            *s_ring_rx_arg1;
  78         mac_resource_handle_t  s_ring_rx_arg2;
  79
  80         /*
  81          * Threshold after which packets get dropped.
  82          * Is always greater than s_ring_tx_hiwat
  83          */
  84         int             s_ring_tx_max_q_cnt;
  85         /* # of mblocks after which to apply flow control */
  86         int             s_ring_tx_hiwat;
  87         /* # of mblocks after which to relieve flow control */
  88         int             s_ring_tx_lowat;
  89         boolean_t       s_ring_tx_woken_up;
  90         uint32_t        s_ring_hiwat_cnt;       /* times blocked for Tx descs */
  91
  92         void            *s_ring_tx_arg1;
  93         void            *s_ring_tx_arg2;
  94
  95         /* Tx notify callback */
  96         mac_cb_info_t   s_ring_notify_cb_info;          /* cb list info */
  97         mac_cb_t        *s_ring_notify_cb_list;         /* The cb list */
  98
  99         clock_t         s_ring_awaken;  /* time async thread was awakened */
 100
 101         kthread_t       *s_ring_run;    /* Current thread processing sq */
 102         processorid_t   s_ring_cpuid;   /* processor to bind to */
 103         processorid_t   s_ring_cpuid_save;      /* saved cpuid during offline */
 104         kcondvar_t      s_ring_async;   /* async thread blocks on */
 105         clock_t         s_ring_wait;    /* lbolts to wait after a fill() */
 106         timeout_id_t    s_ring_tid;     /* timer id of pending timeout() */
 107         kthread_t       *s_ring_worker; /* kernel thread id */
 108         char            s_ring_name[S_RING_NAMELEN + 1];
 109         uint32_t        s_ring_total_inpkt;
 110         uint32_t        s_ring_total_rbytes;
 111         uint32_t        s_ring_drops;
 112         struct mac_client_impl_s *s_ring_mcip;
 113         kstat_t         *s_ring_ksp;
 114
 115         /* Teardown, poll disable control ops */
 116         kcondvar_t      s_ring_client_cv; /* Client wait for control op */
 117
 118         mac_soft_ring_set_t *s_ring_set;   /* The SRS this ring belongs to */
 119         mac_soft_ring_t *s_ring_next;
 120         mac_soft_ring_t *s_ring_prev;
 121         mac_soft_ring_drain_func_t s_ring_drain_func;
 122
 123         mac_tx_stats_t  s_st_stat;
 124 };
 125
 126 typedef void (*mac_srs_drain_proc_t)(mac_soft_ring_set_t *, uint_t);
 127
 128 /* Transmit side Soft Ring Set */
 129 typedef struct mac_srs_tx_s {
 130         /* Members for Tx size processing */
 131         uint32_t        st_mode;
 132         mac_tx_func_t   st_func;
 133         void            *st_arg1;
 134         void            *st_arg2;
 135         mac_group_t     *st_group;      /* TX group for share */
 136         boolean_t       st_woken_up;
 137
 138         /*
 139          * st_max_q_cnt is the queue depth threshold to limit
 140          * outstanding packets on the Tx SRS. Once the limit
 141          * is reached, Tx SRS will drop packets until the
 142          * limit goes below the threshold.
 143          */
 144         uint32_t        st_max_q_cnt;   /* max. outstanding packets */
 145         /*
 146          * st_hiwat is used Tx serializer and bandwidth mode.
 147          * This is the queue depth threshold upto which
 148          * packets will get buffered with no flow-control
 149          * back pressure applied to the caller. Once this
 150          * threshold is reached, back pressure will be
 151          * applied to the caller of mac_tx() (mac_tx() starts
 152          * returning a cookie to indicate a blocked SRS).
 153          * st_hiwat should always be lesser than or equal to
 154          * st_max_q_cnt.
 155          */
 156         uint32_t        st_hiwat;       /* mblk cnt to apply flow control */
 157         uint32_t        st_lowat;       /* mblk cnt to relieve flow control */
 158         uint32_t        st_hiwat_cnt; /* times blocked for Tx descs */
 159         mac_tx_stats_t  st_stat;
 160         mac_capab_aggr_t        st_capab_aggr;
 161         /*
 162          * st_soft_rings is used as an array to store aggr Tx soft
 163          * rings. When aggr_find_tx_ring() returns a pseudo ring,
 164          * the associated soft ring has to be found. st_soft_rings
 165          * array stores the soft ring associated with a pseudo Tx
 166          * ring and it can be accessed using the pseudo ring
 167          * index (mr_index). Note that the ring index is unique
 168          * for each ring in a group.
 169          */
 170         mac_soft_ring_t **st_soft_rings;
 171 } mac_srs_tx_t;
 172
 173 /* Receive side Soft Ring Set */
 174 typedef struct mac_srs_rx_s {
 175         /*
 176          * Upcall Function for fanout, Rx processing etc. Perhaps
 177          * the same 3 members below can be used for Tx
 178          * processing, but looking around, mac_rx_func_t has
 179          * proliferated too much into various files at different
 180          * places. I am leaving the consolidation battle for
 181          * another day.
 182          */
 183         mac_direct_rx_t         sr_func;        /* srs_lock */
 184         void                    *sr_arg1;       /* srs_lock */
 185         mac_resource_handle_t   sr_arg2;        /* srs_lock */
 186         mac_rx_func_t           sr_lower_proc;  /* Atomically changed */
 187         uint32_t                sr_poll_pkt_cnt;
 188         uint32_t                sr_poll_thres;
 189
 190         /* mblk cnt to apply flow control */
 191         uint32_t                sr_hiwat;
 192         /* mblk cnt to relieve flow control */
 193         uint32_t                sr_lowat;
 194         mac_rx_stats_t          sr_stat;
 195
 196         /* Times polling was enabled */
 197         uint32_t                sr_poll_on;
 198         /* Times polling was enabled by worker thread */
 199         uint32_t                sr_worker_poll_on;
 200         /* Times polling was disabled */
 201         uint32_t                sr_poll_off;
 202         /* Poll thread signalled count */
 203         uint32_t                sr_poll_thr_sig;
 204         /* Poll thread busy */
 205         uint32_t                sr_poll_thr_busy;
 206         /* SRS drains, stays in poll mode but doesn't poll */
 207         uint32_t                sr_poll_drain_no_poll;
 208         /*
 209          * SRS has nothing to do and no packets in H/W but
 210          * there is a backlog in softrings. SRS stays in
 211          * poll mode but doesn't do polling.
 212          */
 213         uint32_t                sr_poll_no_poll;
 214         /* Active polling restarted */
 215         uint32_t                sr_below_hiwat;
 216         /* Found packets in last poll so try and poll again */
 217         uint32_t                sr_poll_again;
 218         /*
 219          * Packets in queue but poll thread not allowed to process so
 220          * signal the worker thread.
 221          */
 222         uint32_t                sr_poll_sig_worker;
 223         /*
 224          * Poll thread has nothing to do and H/W has nothing so
 225          * reenable the interrupts.
 226          */
 227         uint32_t                sr_poll_intr_enable;
 228         /*
 229          * Poll thread has nothing to do and worker thread was already
 230          * running so it can decide to reenable interrupt or poll again.
 231          */
 232         uint32_t                sr_poll_goto_sleep;
 233         /* Worker thread goes back to draining the queue */
 234         uint32_t                sr_drain_again;
 235         /* More Packets in queue so signal the poll thread to drain */
 236         uint32_t                sr_drain_poll_sig;
 237         /* More Packets in queue so signal the worker thread to drain */
 238         uint32_t                sr_drain_worker_sig;
 239         /* Poll thread is already running so worker has nothing to do */
 240         uint32_t                sr_drain_poll_running;
 241         /* We have packets already queued so keep polling */
 242         uint32_t                sr_drain_keep_polling;
 243         /* Drain is done and interrupts are reenabled */
 244         uint32_t                sr_drain_finish_intr;
 245         /* Polling thread needs to schedule worker wakeup */
 246         uint32_t                sr_poll_worker_wakeup;
 247 } mac_srs_rx_t;
 248
 249 /*
 250  * mac_soft_ring_set_s:
 251  * This is used both for Tx and Rx side. The srs_type identifies Rx or
 252  * Tx type.
 253  *
 254  * Note that the structure is carefully crafted, with Rx elements coming
 255  * first followed by Tx specific members. Future additions to this
 256  * structure should follow the same guidelines.
 257  *
 258  * Rx-side notes:
 259  * mac_rx_classify_flow_add() always creates a mac_soft_ring_set_t and fn_flow
 260  * points to info from it (func = srs_lower_proc, arg = soft_ring_set). On
 261  * interrupt path, srs_lower_proc does B/W adjustment and switch to polling mode
 262  * (if poll capable) and feeds the packets to soft_ring_list via choosen
 263  * fanout type (specified by srs_type). In poll mode, the poll thread which is
 264  * also a pointer can pick up the packets and feed them to various
 265  * soft_ring_list.
 266  *
 267  * The srs_type can either be protocol based or fanout based where fanout itelf
 268  * can be various types
 269  *
 270  * The polling works by turning off interrupts as soon as a packets
 271  * are queued on the soft ring set. Once the backlog is clear and poll
 272  * thread return empty handed i.e. Rx ring doesn't have anything, the
 273  * interrupt is turned back on. For this purpose we keep a separate
 274  * srs_poll_pkt_cnt counter which tracks the packets queued between SRS
 275  * and the soft rings as well. The counter is incremented when packets
 276  * are queued and decremented when SRS processes them (in case it has
 277  * no soft rings) or the soft ring process them. Its important that
 278  * in case SRS has softrings, the decrement doesn't happen till the
 279  * packet is processed by the soft rings since it takes very little time
 280  * for SRS to queue packet from SRS to soft rings and it will keep
 281  * bringing more packets in the system faster than soft rings can
 282  * process them.
 283  *
 284  * Tx side notes:
 285  * The srs structure acts as a serializer with a worker thread. The
 286  * default behavior of srs though is to act as a pass-thru. The queues
 287  * (srs_first, srs_last, srs_count) get used when Tx ring runs out of Tx
 288  * descriptors or to enforce bandwidth limits.
 289  *
 290  * When multiple Tx rings are present, the SRS state will be set to
 291  * SRS_FANOUT_OTH. Outgoing packets coming into mac_tx_srs_process()
 292  * function will be fanned out to one of the Tx side soft rings based on
 293  * a hint passed in mac_tx_srs_process(). Each soft ring, in turn, will
 294  * be associated with a distinct h/w Tx ring.
 295  */
 296
 297 struct mac_soft_ring_set_s {
 298         /*
 299          * Common elements, common to both Rx and Tx SRS type.
 300          * The following block of fields are protected by srs_lock
 301          */
 302         kmutex_t        srs_lock;
 303         uint32_t        srs_type;
 304         uint32_t        srs_state;      /* state flags */
 305         uint32_t        srs_count;
 306         mblk_t          *srs_first;     /* first mblk chain or NULL */
 307         mblk_t          *srs_last;      /* last mblk chain or NULL */
 308         kcondvar_t      srs_async;      /* cv for worker thread */
 309         kcondvar_t      srs_cv;         /* cv for poll thread */
 310         kcondvar_t      srs_quiesce_done_cv;    /* cv for removal */
 311         timeout_id_t    srs_tid;        /* timeout id for pending timeout */
 312
 313         /*
 314          * List of soft rings & processing function.
 315          * The following block is protected by Rx quiescence.
 316          * i.e. they can be changed only after quiescing the SRS
 317          * Protected by srs_lock.
 318          */
 319         mac_soft_ring_t *srs_soft_ring_head;
 320         mac_soft_ring_t *srs_soft_ring_tail;
 321         int             srs_soft_ring_count;
 322         int             srs_soft_ring_quiesced_count;
 323         int             srs_soft_ring_condemned_count;
 324         mac_soft_ring_t **srs_tcp_soft_rings;
 325         int             srs_tcp_ring_count;
 326         mac_soft_ring_t **srs_udp_soft_rings;
 327         int             srs_udp_ring_count;
 328         mac_soft_ring_t **srs_oth_soft_rings;
 329         int             srs_oth_ring_count;
 330         /*
 331          * srs_tx_soft_rings is used by tx_srs in
 332          * when operating in multi tx ring mode.
 333          */
 334         mac_soft_ring_t **srs_tx_soft_rings;
 335         int             srs_tx_ring_count;
 336
 337         /*
 338          * Bandwidth control related members.
 339          * They are common to both Rx- and Tx-side.
 340          * Following protected by srs_lock
 341          */
 342         mac_bw_ctl_t    *srs_bw;
 343         size_t          srs_size;       /* Size of packets queued in bytes */
 344         pri_t           srs_pri;
 345
 346         mac_soft_ring_set_t     *srs_next;      /* mac_srs_g_lock */
 347         mac_soft_ring_set_t     *srs_prev;      /* mac_srs_g_lock */
 348
 349         /* Attribute specific drain func (BW ctl vs non-BW ctl) */
 350         mac_srs_drain_proc_t    srs_drain_func; /* Write once (WO) */
 351
 352         /*
 353          * If the associated ring is exclusively used by a mac client, e.g.,
 354          * an aggregation, this fields is used to keep a reference to the
 355          * MAC client's pseudo ring.
 356          */
 357         mac_resource_handle_t   srs_mrh;
 358         /*
 359          * The following blocks are write once (WO) and valid for the life
 360          * of the SRS
 361          */
 362         struct mac_client_impl_s *srs_mcip;     /* back ptr to mac client */
 363         void                    *srs_flent;     /* back ptr to flent */
 364         mac_ring_t              *srs_ring;      /*  Ring Descriptor */
 365
 366         /* Teardown, disable control ops */
 367         kcondvar_t      srs_client_cv;  /* Client wait for the control op */
 368
 369         kthread_t       *srs_worker;    /* WO, worker thread */
 370         kthread_t       *srs_poll_thr;  /* WO, poll thread */
 371
 372         uint_t          srs_ind;        /* Round Robin indx for picking up SR */
 373         processorid_t   srs_worker_cpuid;       /* processor to bind to */
 374         processorid_t   srs_worker_cpuid_save;  /* saved cpuid during offline */
 375         processorid_t   srs_poll_cpuid;         /* processor to bind to */
 376         processorid_t   srs_poll_cpuid_save;    /* saved cpuid during offline */
 377         uint_t          srs_fanout_state;
 378         mac_cpus_t      srs_cpu;
 379
 380         mac_srs_rx_t    srs_rx;
 381         mac_srs_tx_t    srs_tx;
 382         kstat_t         *srs_ksp;
 383 };
 384
 385 /*
 386  * type flags - combination allowed to process and drain the queue
 387  */
 388 #define ST_RING_WORKER_ONLY     0x0001  /* Worker thread only */
 389 #define ST_RING_ANY             0x0002  /* Any thread can process the queue */
 390 #define ST_RING_TCP             0x0004
 391 #define ST_RING_UDP             0x0008
 392 #define ST_RING_OTH             0x0010
 393
 394 #define ST_RING_BW_CTL          0x0020
 395 #define ST_RING_TX              0x0040
 396
 397 /*
 398  * State flags.
 399  */
 400 #define S_RING_PROC             0x0001  /* being processed */
 401 #define S_RING_BOUND            0x0002  /* Worker thread is bound to a cpu */
 402 #define S_RING_BLOCK            0x0004  /* No Tx descs */
 403 #define S_RING_TX_HIWAT         0x0008  /* Tx high watermark reached */
 404
 405 #define S_RING_WAKEUP_CLIENT    0x0010  /* flow ctrl, client wakeup needed */
 406 #define S_RING_BLANK            0x0020  /* Has been put into polling mode */
 407 #define S_RING_CLIENT_WAIT      0x0040  /* Client waiting for control op */
 408
 409 #define S_RING_CONDEMNED        0x0100  /* Being torn down */
 410 #define S_RING_CONDEMNED_DONE   0x0200  /* Being torn down */
 411 #define S_RING_QUIESCE          0x0400  /* No traffic flow, transient flag */
 412 #define S_RING_QUIESCE_DONE     0x0800  /* No traffic flow, transient flag */
 413
 414 #define S_RING_RESTART          0x1000  /* Go back to normal traffic flow */
 415 #define S_RING_ENQUEUED         0x2000  /* Pkts enqueued in Tx soft ring */
 416
 417 /*
 418  * arguments for processors to bind to
 419  */
 420 #define S_RING_BIND_NONE        -1
 421
 422 /*
 423  * defines for srs_type - identifies a link or a sub-flow
 424  * and other static characteristics of a SRS like a tx
 425  * srs, tcp only srs, etc.
 426  */
 427 #define SRST_LINK               0x00000001
 428 #define SRST_FLOW               0x00000002
 429 #define SRST_NO_SOFT_RINGS      0x00000004
 430 #define SRST_TCP_ONLY           0x00000008
 431
 432 #define SRST_FANOUT_PROTO       0x00000010
 433 #define SRST_FANOUT_SRC_IP      0x00000020
 434 #define SRST_FANOUT_OTH         0x00000040
 435 #define SRST_DEFAULT_GRP        0x00000080
 436
 437 #define SRST_TX                 0x00000100
 438 #define SRST_BW_CONTROL         0x00000200
 439 #define SRST_DIRECT_POLL        0x00000400
 440
 441 #define SRST_DLS_BYPASS         0x00001000
 442 #define SRST_CLIENT_POLL_ENABLED 0x00002000
 443
 444 /*
 445  * soft ring set flags. These bits are dynamic in nature and get
 446  * applied to srs_state. They reflect the state of SRS at any
 447  * point of time
 448  */
 449 #define SRS_BLANK               0x00000001
 450 #define SRS_WORKER_BOUND        0x00000002
 451 #define SRS_POLL_BOUND          0x00000004
 452 #define SRS_POLLING_CAPAB       0x00000008
 453
 454 #define SRS_PROC                0x00000010
 455 #define SRS_GET_PKTS            0x00000020
 456 #define SRS_POLLING             0x00000040
 457 #define SRS_BW_ENFORCED         0x00000080
 458
 459 #define SRS_WORKER              0x00000100
 460 #define SRS_ENQUEUED            0x00000200
 461 #define SRS_ANY_PROCESS         0x00000400
 462 #define SRS_PROC_FAST           0x00000800
 463
 464 #define SRS_POLL_PROC           0x00001000
 465 #define SRS_TX_BLOCKED          0x00002000      /* out of Tx descs */
 466 #define SRS_TX_HIWAT            0x00004000      /* Tx count exceeds hiwat */
 467 #define SRS_TX_WAKEUP_CLIENT    0x00008000      /* Flow-ctl: wakeup client */
 468
 469 #define SRS_CLIENT_PROC         0x00010000
 470 #define SRS_CLIENT_WAIT         0x00020000
 471 #define SRS_QUIESCE             0x00040000
 472 #define SRS_QUIESCE_DONE        0x00080000
 473
 474 #define SRS_CONDEMNED           0x00100000
 475 #define SRS_CONDEMNED_DONE      0x00200000
 476 #define SRS_POLL_THR_QUIESCED   0x00400000
 477 #define SRS_RESTART             0x00800000
 478
 479 #define SRS_RESTART_DONE        0x01000000
 480 #define SRS_POLL_THR_RESTART    0x02000000
 481 #define SRS_IN_GLIST            0x04000000
 482 #define SRS_POLL_THR_EXITED     0x08000000
 483
 484 #define SRS_QUIESCE_PERM        0x10000000
 485 #define SRS_LATENCY_OPT         0x20000000
 486 #define SRS_SOFTRING_QUEUE      0x40000000
 487
 488 #define SRS_QUIESCED(srs)       (srs->srs_state & SRS_QUIESCE_DONE)
 489
 490 /*
 491  * If the SRS_QUIESCE_PERM flag is set, the SRS worker thread will not be
 492  * able to be restarted.
 493  */
 494 #define SRS_QUIESCED_PERMANENT(srs)     (srs->srs_state & SRS_QUIESCE_PERM)
 495
 496 /*
 497  * soft ring set (SRS) Tx modes
 498  */
 499 typedef enum {
 500         SRS_TX_DEFAULT = 0,
 501         SRS_TX_SERIALIZE,
 502         SRS_TX_FANOUT,
 503         SRS_TX_BW,
 504         SRS_TX_BW_FANOUT,
 505         SRS_TX_AGGR,
 506         SRS_TX_BW_AGGR
 507 } mac_tx_srs_mode_t;
 508
 509 /*
 510  * SRS fanout states
 511  */
 512 typedef enum {
 513         SRS_FANOUT_UNINIT = 0,
 514         SRS_FANOUT_INIT,
 515         SRS_FANOUT_REINIT
 516 } mac_srs_fanout_state_t;
 517
 518 /*
 519  * Structure for dls statistics
 520  */
 521 struct dls_kstats {
 522         kstat_named_t   dlss_soft_ring_pkt_drop;
 523 };
 524
 525 extern struct dls_kstats dls_kstat;
 526
 527 #define DLS_BUMP_STAT(x, y)     (dls_kstat.x.value.ui32 += y)
 528
 529 /* Turn dynamic polling off */
 530 #define MAC_SRS_POLLING_OFF(mac_srs) {                                  \
 531         ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));                       \
 532         if (((mac_srs)->srs_state & (SRS_POLLING_CAPAB|SRS_POLLING)) == \
 533             (SRS_POLLING_CAPAB|SRS_POLLING)) {                          \
 534                 (mac_srs)->srs_state &= ~SRS_POLLING;                   \
 535                 (void) mac_hwring_enable_intr((mac_ring_handle_t)       \
 536                     (mac_srs)->srs_ring);                               \
 537                 (mac_srs)->srs_rx.sr_poll_off++;                        \
 538         }                                                               \
 539 }
 540
 541 #define MAC_COUNT_CHAIN(mac_srs, head, tail, cnt, sz)   {       \
 542         mblk_t          *tmp;                                   \
 543         boolean_t       bw_ctl = B_FALSE;                       \
 544                                                                 \
 545         ASSERT((head) != NULL);                                 \
 546         cnt = 0;                                                \
 547         sz = 0;                                                 \
 548         if ((mac_srs)->srs_type & SRST_BW_CONTROL)              \
 549                 bw_ctl = B_TRUE;                                \
 550         tmp = tail = (head);                                    \
 551         if ((head)->b_next == NULL) {                           \
 552                 cnt = 1;                                        \
 553                 if (bw_ctl)                                     \
 554                         sz += msgdsize(head);                   \
 555         } else {                                                \
 556                 while (tmp != NULL) {                           \
 557                         tail = tmp;                             \
 558                         cnt++;                                  \
 559                         if (bw_ctl)                             \
 560                                 sz += msgdsize(tmp);            \
 561                         tmp = tmp->b_next;                      \
 562                 }                                               \
 563         }                                                       \
 564 }
 565
 566 /*
 567  * Decrement the cumulative packet count in SRS and its
 568  * soft rings. If the srs_poll_pkt_cnt goes below lowat, then check
 569  * if if the interface was left in a polling mode and no one
 570  * is really processing the queue (to get the interface out
 571  * of poll mode). If no one is processing the queue, then
 572  * acquire the PROC and signal the poll thread to check the
 573  * interface for packets and get the interface back to interrupt
 574  * mode if nothing is found.
 575  */
 576 #define MAC_UPDATE_SRS_COUNT_LOCKED(mac_srs, cnt) {                     \
 577         mac_srs_rx_t    *srs_rx = &(mac_srs)->srs_rx;                   \
 578         ASSERT(MUTEX_HELD(&(mac_srs)->srs_lock));                       \
 579                                                                         \
 580         srs_rx->sr_poll_pkt_cnt -= cnt;                                 \
 581         if ((srs_rx->sr_poll_pkt_cnt <= srs_rx->sr_poll_thres) &&       \
 582                 (((mac_srs)->srs_state &                                \
 583                 (SRS_POLLING|SRS_PROC|SRS_GET_PKTS)) == SRS_POLLING))   \
 584         {                                                               \
 585                 (mac_srs)->srs_state |= (SRS_PROC|SRS_GET_PKTS);        \
 586                 cv_signal(&(mac_srs)->srs_cv);                          \
 587                 srs_rx->sr_below_hiwat++;                               \
 588         }                                                               \
 589 }
 590
 591 /*
 592  * The following two macros are used to update the inbound packet and byte.
 593  * count. The packet and byte count reflect the packets and bytes that are
 594  * taken out of the SRS's queue, i.e. indicating they are being delivered.
 595  * The srs_count and srs_size are updated in different locations as the
 596  * srs_size is also used to take into account any bandwidth limits. The
 597  * srs_size is updated only when a soft ring, if any, sends a packet up,
 598  * as opposed to updating it when the SRS sends a packet to the SR, i.e.
 599  * the srs_size reflects the packets in the SRS and SRs. These
 600  * macros decrement the srs_size and srs_count and also increment the
 601  * ipackets and ibytes stats resp.
 602  *
 603  * xxx-venu These are done under srs_lock, for now we still update
 604  * mci_stat_ibytes/mci_stat_ipackets atomically, need to check if
 605  * just updating them would be accurate enough.
 606  *
 607  * If we are updating these for a sub-flow SRS, then we need to also
 608  * updated it's MAC client bandwidth info, if the MAC client is also
 609  * bandwidth regulated.
 610  */
 611 #define MAC_UPDATE_SRS_SIZE_LOCKED(srs, sz) {                           \
 612         if ((srs)->srs_type & SRST_BW_CONTROL) {                        \
 613                 mutex_enter(&(srs)->srs_bw->mac_bw_lock);               \
 614                 (srs)->srs_bw->mac_bw_sz -= (sz);                       \
 615                 (srs)->srs_bw->mac_bw_used += (sz);                     \
 616                 mutex_exit(&(srs)->srs_bw->mac_bw_lock);                \
 617         }                                                               \
 618 }
 619
 620 #define MAC_TX_UPDATE_BW_INFO(srs, sz) {                                \
 621         (srs)->srs_bw->mac_bw_sz -= (sz);                               \
 622         (srs)->srs_bw->mac_bw_used += (sz);                             \
 623 }
 624
 625 #define MAC_TX_SOFT_RINGS(mac_srs) ((mac_srs)->srs_tx_ring_count >= 1)
 626
 627 /* Soft ring flags for teardown */
 628 #define SRS_POLL_THR_OWNER      (SRS_PROC | SRS_POLLING | SRS_GET_PKTS)
 629 #define SRS_PAUSE               (SRS_CONDEMNED | SRS_QUIESCE)
 630 #define S_RING_PAUSE            (S_RING_CONDEMNED | S_RING_QUIESCE)
 631
 632 /* Soft rings */
 633 extern void mac_soft_ring_init(void);
 634 extern void mac_soft_ring_finish(void);
 635 extern void mac_fanout_setup(mac_client_impl_t *, flow_entry_t *,
 636     mac_resource_props_t *, mac_direct_rx_t, void *, mac_resource_handle_t,
 637     cpupart_t *);
 638
 639 extern void mac_soft_ring_worker_wakeup(mac_soft_ring_t *);
 640 extern void mac_soft_ring_blank(void *, time_t, uint_t, int);
 641 extern mblk_t *mac_soft_ring_poll(mac_soft_ring_t *, int);
 642 extern void mac_soft_ring_destroy(mac_soft_ring_t *);
 643 extern void mac_soft_ring_dls_bypass(void *, mac_direct_rx_t, void *);
 644
 645 /* Rx SRS */
 646 extern mac_soft_ring_set_t *mac_srs_create(struct mac_client_impl_s *,
 647     flow_entry_t *, uint32_t, mac_direct_rx_t, void *, mac_resource_handle_t,
 648     mac_ring_t *);
 649 extern void mac_srs_free(mac_soft_ring_set_t *);
 650 extern void mac_srs_signal(mac_soft_ring_set_t *, uint_t);
 651 extern cpu_t *mac_srs_bind(mac_soft_ring_set_t *, processorid_t);
 652 extern void mac_rx_srs_retarget_intr(mac_soft_ring_set_t *, processorid_t);
 653 extern void mac_tx_srs_retarget_intr(mac_soft_ring_set_t *);
 654
 655 extern void mac_srs_change_upcall(void *, mac_direct_rx_t, void *);
 656 extern void mac_srs_quiesce_initiate(mac_soft_ring_set_t *);
 657 extern void mac_srs_client_poll_enable(struct mac_client_impl_s *,
 658     mac_soft_ring_set_t *);
 659 extern void mac_srs_client_poll_disable(struct mac_client_impl_s *,
 660     mac_soft_ring_set_t *);
 661 extern void mac_srs_client_poll_quiesce(struct mac_client_impl_s *,
 662     mac_soft_ring_set_t *);
 663 extern void mac_srs_client_poll_restart(struct mac_client_impl_s *,
 664     mac_soft_ring_set_t *);
 665 extern void mac_rx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
 666 extern void mac_rx_srs_restart(mac_soft_ring_set_t *);
 667 extern void mac_rx_srs_subflow_process(void *, mac_resource_handle_t, mblk_t *,
 668     boolean_t);
 669 extern void mac_tx_srs_quiesce(mac_soft_ring_set_t *, uint_t);
 670
 671 /* Tx SRS, Tx softring */
 672 extern void mac_tx_srs_wakeup(mac_soft_ring_set_t *, mac_ring_handle_t);
 673 extern void mac_tx_srs_setup(struct mac_client_impl_s *, flow_entry_t *);
 674 extern mac_tx_func_t mac_tx_get_func(uint32_t);
 675 extern mblk_t *mac_tx_send(mac_client_handle_t, mac_ring_handle_t, mblk_t *,
 676     mac_tx_stats_t *);
 677 extern boolean_t mac_tx_srs_ring_present(mac_soft_ring_set_t *, mac_ring_t *);
 678 extern mac_soft_ring_t *mac_tx_srs_get_soft_ring(mac_soft_ring_set_t *,
 679     mac_ring_t *);
 680 extern void mac_tx_srs_add_ring(mac_soft_ring_set_t *, mac_ring_t *);
 681 extern void mac_tx_srs_del_ring(mac_soft_ring_set_t *, mac_ring_t *);
 682 extern mac_tx_cookie_t mac_tx_srs_no_desc(mac_soft_ring_set_t *, mblk_t *,
 683     uint16_t, mblk_t **);
 684
 685 /* Subflow specific stuff */
 686 extern int mac_srs_flow_create(struct mac_client_impl_s *, flow_entry_t *,
 687     mac_resource_props_t *, int, int, mac_direct_rx_t);
 688 extern void mac_srs_update_bwlimit(flow_entry_t *, mac_resource_props_t *);
 689 extern void mac_srs_adjust_subflow_bwlimit(struct mac_client_impl_s *);
 690 extern void mac_srs_update_drv(struct mac_client_impl_s *);
 691 extern void mac_update_srs_priority(mac_soft_ring_set_t *, pri_t);
 692 extern void mac_client_update_classifier(mac_client_impl_t *, boolean_t);
 693
 694 extern void mac_soft_ring_intr_enable(void *);
 695 extern boolean_t mac_soft_ring_intr_disable(void *);
 696 extern mac_soft_ring_t *mac_soft_ring_create(int, clock_t, uint16_t,
 697     pri_t, mac_client_impl_t *, mac_soft_ring_set_t *,
 698     processorid_t, mac_direct_rx_t, void *, mac_resource_handle_t);
 699 extern cpu_t *mac_soft_ring_bind(mac_soft_ring_t *, processorid_t);
 700         extern void mac_soft_ring_unbind(mac_soft_ring_t *);
 701 extern void mac_soft_ring_free(mac_soft_ring_t *);
 702 extern void mac_soft_ring_signal(mac_soft_ring_t *, uint_t);
 703 extern void mac_rx_soft_ring_process(mac_client_impl_t *, mac_soft_ring_t *,
 704     mblk_t *, mblk_t *, int, size_t);
 705 extern mac_tx_cookie_t mac_tx_soft_ring_process(mac_soft_ring_t *,
 706     mblk_t *, uint16_t, mblk_t **);
 707 extern void mac_srs_worker_quiesce(mac_soft_ring_set_t *);
 708 extern void mac_srs_worker_restart(mac_soft_ring_set_t *);
 709 extern void mac_rx_attach_flow_srs(mac_impl_t *, flow_entry_t *,
 710     mac_soft_ring_set_t *, mac_ring_t *, mac_classify_type_t);
 711
 712 extern void mac_rx_srs_drain_bw(mac_soft_ring_set_t *, uint_t);
 713 extern void mac_rx_srs_drain(mac_soft_ring_set_t *, uint_t);
 714 extern void mac_rx_srs_process(void *, mac_resource_handle_t, mblk_t *,
 715     boolean_t);
 716 extern void mac_srs_worker(mac_soft_ring_set_t *);
 717 extern void mac_rx_srs_poll_ring(mac_soft_ring_set_t *);
 718 extern void mac_tx_srs_drain(mac_soft_ring_set_t *, uint_t);
 719
 720 extern void mac_tx_srs_restart(mac_soft_ring_set_t *);
 721 extern void mac_rx_srs_remove(mac_soft_ring_set_t *);
 722
 723 #ifdef  __cplusplus
 724 }
 725 #endif
 726
 727 #endif  /* _SYS_MAC_SOFT_RING_H */