usr/src/uts/intel/io/vmxnet3s/vmxnet3_rx.c

   1 /*
   2  * Copyright (C) 2007 VMware, Inc. All rights reserved.
   3  *
   4  * The contents of this file are subject to the terms of the Common
   5  * Development and Distribution License (the "License") version 1.0
   6  * and no later version.  You may not use this file except in
   7  * compliance with the License.
   8  *
   9  * You can obtain a copy of the License at
  10  *         http://www.opensource.org/licenses/cddl1.php
  11  *
  12  * See the License for the specific language governing permissions
  13  * and limitations under the License.
  14  */
  15 /*
  16  * Copyright (c) 2013, 2016 by Delphix. All rights reserved.
  17  */
  18
  19 #include <vmxnet3.h>
  20
  21 static void vmxnet3_put_rxbuf(vmxnet3_rxbuf_t *);
  22
  23 /*
  24  * Allocate a new rxBuf from memory. All its fields are set except
  25  * for its associated mblk which has to be allocated later.
  26  *
  27  * Returns:
  28  *      A new rxBuf or NULL.
  29  */
  30 static vmxnet3_rxbuf_t *
  31 vmxnet3_alloc_rxbuf(vmxnet3_softc_t *dp, boolean_t canSleep)
  32 {
  33         vmxnet3_rxbuf_t *rxBuf;
  34         int flag = canSleep ? KM_SLEEP : KM_NOSLEEP;
  35         int err;
  36
  37         rxBuf = kmem_zalloc(sizeof (vmxnet3_rxbuf_t), flag);
  38         if (!rxBuf) {
  39                 atomic_inc_32(&dp->rx_alloc_failed);
  40                 return (NULL);
  41         }
  42
  43         if ((err = vmxnet3_alloc_dma_mem_1(dp, &rxBuf->dma, (dp->cur_mtu + 18),
  44             canSleep)) != 0) {
  45                 VMXNET3_DEBUG(dp, 0, "Failed to allocate %d bytes for rx buf, "
  46                     "err:%d\n", (dp->cur_mtu + 18), err);
  47                 kmem_free(rxBuf, sizeof (vmxnet3_rxbuf_t));
  48                 atomic_inc_32(&dp->rx_alloc_failed);
  49                 return (NULL);
  50         }
  51
  52         rxBuf->freeCB.free_func = vmxnet3_put_rxbuf;
  53         rxBuf->freeCB.free_arg = (caddr_t)rxBuf;
  54         rxBuf->dp = dp;
  55
  56         atomic_inc_32(&dp->rx_num_bufs);
  57         atomic_inc_32(&dp->rx_alloc_buf);
  58         return (rxBuf);
  59 }
  60
  61 static void
  62 vmxnet3_free_rxbuf(vmxnet3_softc_t *dp, vmxnet3_rxbuf_t *rxBuf)
  63 {
  64         vmxnet3_free_dma_mem(&rxBuf->dma);
  65         kmem_free(rxBuf, sizeof (vmxnet3_rxbuf_t));
  66
  67 #ifndef DEBUG
  68         atomic_dec_32(&dp->rx_num_bufs);
  69 #else
  70         {
  71                 uint32_t nv = atomic_dec_32_nv(&dp->rx_num_bufs);
  72                 ASSERT(nv != (uint32_t)-1);
  73         }
  74 #endif
  75 }
  76
  77 /*
  78  * Return a rxBuf to the pool. The init argument, when B_TRUE, indicates
  79  * that we're being called for the purpose of pool initialization, and
  80  * therefore, we should place the buffer in the pool even if the device
  81  * isn't enabled.
  82  *
  83  * Returns:
  84  *      B_TRUE if the buffer was returned to the pool, or B_FALSE if it
  85  *      wasn't (e.g. if the device is stopped).
  86  */
  87 static boolean_t
  88 vmxnet3_put_rxpool_buf(vmxnet3_softc_t *dp, vmxnet3_rxbuf_t *rxBuf,
  89     boolean_t init)
  90 {
  91         vmxnet3_rxpool_t *rxPool = &dp->rxPool;
  92         boolean_t returned = B_FALSE;
  93
  94         mutex_enter(&dp->rxPoolLock);
  95         ASSERT(rxPool->nBufs <= rxPool->nBufsLimit);
  96         if ((dp->devEnabled || init) && rxPool->nBufs < rxPool->nBufsLimit) {
  97                 ASSERT((rxPool->listHead == NULL && rxPool->nBufs == 0) ||
  98                     (rxPool->listHead != NULL && rxPool->nBufs != 0));
  99                 rxBuf->next = rxPool->listHead;
 100                 rxPool->listHead = rxBuf;
 101                 rxPool->nBufs++;
 102                 returned = B_TRUE;
 103         }
 104         mutex_exit(&dp->rxPoolLock);
 105         return (returned);
 106 }
 107
 108 /*
 109  * Return a rxBuf to the pool or free it.
 110  */
 111 static void
 112 vmxnet3_put_rxbuf(vmxnet3_rxbuf_t *rxBuf)
 113 {
 114         vmxnet3_softc_t *dp = rxBuf->dp;
 115
 116         if (!vmxnet3_put_rxpool_buf(dp, rxBuf, B_FALSE))
 117                 vmxnet3_free_rxbuf(dp, rxBuf);
 118 }
 119
 120 /*
 121  * Get an unused rxBuf from the pool.
 122  *
 123  * Returns:
 124  *      A rxBuf or NULL if there are no buffers in the pool.
 125  */
 126 static vmxnet3_rxbuf_t *
 127 vmxnet3_get_rxpool_buf(vmxnet3_softc_t *dp)
 128 {
 129         vmxnet3_rxpool_t *rxPool = &dp->rxPool;
 130         vmxnet3_rxbuf_t *rxBuf = NULL;
 131
 132         mutex_enter(&dp->rxPoolLock);
 133         if (rxPool->listHead != NULL) {
 134                 rxBuf = rxPool->listHead;
 135                 rxPool->listHead = rxBuf->next;
 136                 rxPool->nBufs--;
 137                 ASSERT((rxPool->listHead == NULL && rxPool->nBufs == 0) ||
 138                     (rxPool->listHead != NULL && rxPool->nBufs != 0));
 139         }
 140         mutex_exit(&dp->rxPoolLock);
 141         return (rxBuf);
 142 }
 143
 144 /*
 145  * Fill a rxPool by allocating the maximum number of buffers.
 146  *
 147  * Returns:
 148  *      0 on success, non-zero on failure.
 149  */
 150 static int
 151 vmxnet3_rxpool_init(vmxnet3_softc_t *dp)
 152 {
 153         int err = 0;
 154         vmxnet3_rxbuf_t *rxBuf;
 155
 156         ASSERT(dp->rxPool.nBufsLimit > 0);
 157         while (dp->rxPool.nBufs < dp->rxPool.nBufsLimit) {
 158                 if ((rxBuf = vmxnet3_alloc_rxbuf(dp, B_FALSE)) == NULL) {
 159                         err = ENOMEM;
 160                         break;
 161                 }
 162                 VERIFY(vmxnet3_put_rxpool_buf(dp, rxBuf, B_TRUE));
 163         }
 164
 165         if (err != 0) {
 166                 while ((rxBuf = vmxnet3_get_rxpool_buf(dp)) != NULL) {
 167                         vmxnet3_free_rxbuf(dp, rxBuf);
 168                 }
 169         }
 170
 171         return (err);
 172 }
 173
 174 /*
 175  * Populate a Rx descriptor with a new rxBuf. If the pool argument is B_TRUE,
 176  * then try to take a buffer from rxPool. If the pool is empty and the
 177  * dp->alloc_ok is true, then fall back to dynamic allocation. If pool is
 178  * B_FALSE, then always allocate a new buffer (this is only used when
 179  * populating the initial set of buffers in the receive queue during start).
 180  *
 181  * Returns:
 182  *      0 on success, non-zero on failure.
 183  */
 184 static int
 185 vmxnet3_rx_populate(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq, uint16_t idx,
 186     boolean_t canSleep, boolean_t pool)
 187 {
 188         vmxnet3_rxbuf_t *rxBuf = NULL;
 189
 190         if (pool && (rxBuf = vmxnet3_get_rxpool_buf(dp)) == NULL) {
 191                 /* The maximum number of pool buffers have been allocated. */
 192                 atomic_inc_32(&dp->rx_pool_empty);
 193                 if (!dp->alloc_ok) {
 194                         atomic_inc_32(&dp->rx_alloc_failed);
 195                 }
 196         }
 197
 198         if (rxBuf == NULL && (!pool || dp->alloc_ok)) {
 199                 rxBuf = vmxnet3_alloc_rxbuf(dp, canSleep);
 200         }
 201
 202         if (rxBuf != NULL) {
 203                 rxBuf->mblk = desballoc((uchar_t *)rxBuf->dma.buf,
 204                     rxBuf->dma.bufLen, BPRI_MED, &rxBuf->freeCB);
 205                 if (rxBuf->mblk == NULL) {
 206                         if (pool) {
 207                                 VERIFY(vmxnet3_put_rxpool_buf(dp, rxBuf,
 208                                     B_FALSE));
 209                         } else {
 210                                 vmxnet3_free_rxbuf(dp, rxBuf);
 211                         }
 212                         atomic_inc_32(&dp->rx_alloc_failed);
 213                         return (ENOMEM);
 214                 }
 215
 216                 vmxnet3_cmdring_t *cmdRing = &rxq->cmdRing;
 217                 Vmxnet3_GenericDesc *rxDesc = VMXNET3_GET_DESC(cmdRing, idx);
 218
 219                 rxq->bufRing[idx].rxBuf = rxBuf;
 220                 rxDesc->rxd.addr = rxBuf->dma.bufPA;
 221                 rxDesc->rxd.len = rxBuf->dma.bufLen;
 222                 /* rxDesc->rxd.btype = 0; */
 223                 membar_producer();
 224                 rxDesc->rxd.gen = cmdRing->gen;
 225         } else {
 226                 return (ENOMEM);
 227         }
 228
 229         return (0);
 230 }
 231
 232 /*
 233  * Initialize a RxQueue by populating the whole Rx ring with rxBufs.
 234  *
 235  * Returns:
 236  *      0 on success, non-zero on failure.
 237  */
 238 int
 239 vmxnet3_rxqueue_init(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
 240 {
 241         vmxnet3_cmdring_t *cmdRing = &rxq->cmdRing;
 242         int err;
 243
 244         dp->rxPool.nBufsLimit = vmxnet3_getprop(dp, "RxBufPoolLimit", 0,
 245             cmdRing->size * 10, cmdRing->size * 2);
 246
 247         do {
 248                 if ((err = vmxnet3_rx_populate(dp, rxq, cmdRing->next2fill,
 249                     B_TRUE, B_FALSE)) != 0) {
 250                         goto error;
 251                 }
 252                 VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill);
 253         } while (cmdRing->next2fill);
 254
 255         /*
 256          * Pre-allocate rxPool buffers so that we never have to allocate
 257          * new buffers from interrupt context when we need to replace a buffer
 258          * in the rxqueue.
 259          */
 260         if ((err = vmxnet3_rxpool_init(dp)) != 0) {
 261                 goto error;
 262         }
 263
 264         return (0);
 265
 266 error:
 267         while (cmdRing->next2fill) {
 268                 VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill);
 269                 vmxnet3_free_rxbuf(dp, rxq->bufRing[cmdRing->next2fill].rxBuf);
 270         }
 271
 272         return (err);
 273 }
 274
 275 /*
 276  * Finish a RxQueue by freeing all the related rxBufs.
 277  */
 278 void
 279 vmxnet3_rxqueue_fini(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
 280 {
 281         vmxnet3_rxbuf_t *rxBuf;
 282         unsigned int i;
 283
 284         ASSERT(!dp->devEnabled);
 285
 286         /* First the rxPool */
 287         while ((rxBuf = vmxnet3_get_rxpool_buf(dp)))
 288                 vmxnet3_free_rxbuf(dp, rxBuf);
 289
 290         /* Then the ring */
 291         for (i = 0; i < rxq->cmdRing.size; i++) {
 292                 rxBuf = rxq->bufRing[i].rxBuf;
 293                 ASSERT(rxBuf);
 294                 ASSERT(rxBuf->mblk);
 295                 /*
 296                  * Here, freemsg() will trigger a call to vmxnet3_put_rxbuf()
 297                  * which will then call vmxnet3_free_rxbuf() because the
 298                  * underlying device is disabled.
 299                  */
 300                 freemsg(rxBuf->mblk);
 301         }
 302 }
 303
 304 /*
 305  * Determine if a received packet was checksummed by the Vmxnet3
 306  * device and tag the mp appropriately.
 307  */
 308 static void
 309 vmxnet3_rx_hwcksum(vmxnet3_softc_t *dp, mblk_t *mp,
 310     Vmxnet3_GenericDesc *compDesc)
 311 {
 312         uint32_t flags = 0;
 313
 314         if (!compDesc->rcd.cnc) {
 315                 if (compDesc->rcd.v4 && compDesc->rcd.ipc) {
 316                         flags |= HCK_IPV4_HDRCKSUM;
 317                         if ((compDesc->rcd.tcp || compDesc->rcd.udp) &&
 318                             compDesc->rcd.tuc) {
 319                                 flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
 320                         }
 321                 }
 322
 323                 VMXNET3_DEBUG(dp, 3, "rx cksum flags = 0x%x\n", flags);
 324
 325                 (void) hcksum_assoc(mp, NULL, NULL, 0, 0, 0, 0, flags, 0);
 326         }
 327 }
 328
 329 /*
 330  * Interrupt handler for Rx. Look if there are any pending Rx and
 331  * put them in mplist.
 332  *
 333  * Returns:
 334  *      A list of messages to pass to the MAC subystem.
 335  */
 336 mblk_t *
 337 vmxnet3_rx_intr(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
 338 {
 339         vmxnet3_compring_t *compRing = &rxq->compRing;
 340         vmxnet3_cmdring_t *cmdRing = &rxq->cmdRing;
 341         Vmxnet3_RxQueueCtrl *rxqCtrl = rxq->sharedCtrl;
 342         Vmxnet3_GenericDesc *compDesc;
 343         mblk_t *mplist = NULL, **mplistTail = &mplist;
 344
 345         ASSERT(mutex_owned(&dp->intrLock));
 346
 347         compDesc = VMXNET3_GET_DESC(compRing, compRing->next2comp);
 348         while (compDesc->rcd.gen == compRing->gen) {
 349                 mblk_t *mp = NULL, **mpTail = &mp;
 350                 boolean_t mpValid = B_TRUE;
 351                 boolean_t eop;
 352
 353                 ASSERT(compDesc->rcd.sop);
 354
 355                 do {
 356                         uint16_t rxdIdx = compDesc->rcd.rxdIdx;
 357                         vmxnet3_rxbuf_t *rxBuf = rxq->bufRing[rxdIdx].rxBuf;
 358                         mblk_t *mblk = rxBuf->mblk;
 359                         Vmxnet3_GenericDesc *rxDesc;
 360
 361                         while (compDesc->rcd.gen != compRing->gen) {
 362                                 /*
 363                                  * H/W may be still be in the middle of
 364                                  * generating this entry, so hold on until
 365                                  * the gen bit is flipped.
 366                                  */
 367                                 membar_consumer();
 368                         }
 369                         ASSERT(compDesc->rcd.gen == compRing->gen);
 370                         ASSERT(rxBuf);
 371                         ASSERT(mblk);
 372
 373                         /* Some Rx descriptors may have been skipped */
 374                         while (cmdRing->next2fill != rxdIdx) {
 375                                 rxDesc = VMXNET3_GET_DESC(cmdRing,
 376                                     cmdRing->next2fill);
 377                                 rxDesc->rxd.gen = cmdRing->gen;
 378                                 VMXNET3_INC_RING_IDX(cmdRing,
 379                                     cmdRing->next2fill);
 380                         }
 381
 382                         eop = compDesc->rcd.eop;
 383
 384                         /*
 385                          * Now we have a piece of the packet in the rxdIdx
 386                          * descriptor. Grab it only if we achieve to replace
 387                          * it with a fresh buffer.
 388                          */
 389                         if (vmxnet3_rx_populate(dp, rxq, rxdIdx, B_FALSE,
 390                             B_TRUE) == 0) {
 391                                 /* Success, we can chain the mblk with the mp */
 392                                 mblk->b_wptr = mblk->b_rptr + compDesc->rcd.len;
 393                                 *mpTail = mblk;
 394                                 mpTail = &mblk->b_cont;
 395                                 ASSERT(*mpTail == NULL);
 396
 397                                 VMXNET3_DEBUG(dp, 3, "rx 0x%p on [%u]\n", mblk,
 398                                     rxdIdx);
 399
 400                                 if (eop) {
 401                                         if (!compDesc->rcd.err) {
 402                                                 /*
 403                                                  * Tag the mp if it was
 404                                                  * checksummed by the H/W
 405                                                  */
 406                                                 vmxnet3_rx_hwcksum(dp, mp,
 407                                                     compDesc);
 408                                         } else {
 409                                                 mpValid = B_FALSE;
 410                                         }
 411                                 }
 412                         } else {
 413                                 /*
 414                                  * Keep the same buffer, we still need
 415                                  * to flip the gen bit
 416                                  */
 417                                 rxDesc = VMXNET3_GET_DESC(cmdRing, rxdIdx);
 418                                 rxDesc->rxd.gen = cmdRing->gen;
 419                                 mpValid = B_FALSE;
 420                         }
 421
 422                         VMXNET3_INC_RING_IDX(compRing, compRing->next2comp);
 423                         VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill);
 424                         compDesc = VMXNET3_GET_DESC(compRing,
 425                             compRing->next2comp);
 426                 } while (!eop);
 427
 428                 if (mp) {
 429                         if (mpValid) {
 430                                 *mplistTail = mp;
 431                                 mplistTail = &mp->b_next;
 432                                 ASSERT(*mplistTail == NULL);
 433                         } else {
 434                                 /* This message got holes, drop it */
 435                                 freemsg(mp);
 436                         }
 437                 }
 438         }
 439
 440         if (rxqCtrl->updateRxProd) {
 441                 uint32_t rxprod;
 442
 443                 /*
 444                  * All buffers are actually available, but we can't tell that to
 445                  * the device because it may interpret that as an empty ring.
 446                  * So skip one buffer.
 447                  */
 448                 if (cmdRing->next2fill) {
 449                         rxprod = cmdRing->next2fill - 1;
 450                 } else {
 451                         rxprod = cmdRing->size - 1;
 452                 }
 453                 VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_RXPROD, rxprod);
 454         }
 455
 456         return (mplist);
 457 }