?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
fix a kmap leak in virtio_console
[linux/fpc-iii.git] / drivers / net / vmxnet3 / vmxnet3_drv.c
blob3be786faaaec222f0226b8285dde01e88b147e59
1 /*
2 * Linux driver for VMware's vmxnet3 ethernet NIC.
3 *
4 * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License and no later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more
14 * details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * The full GNU General Public License is included in this distribution in
21 * the file called "COPYING".
22 *
23 * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
24 *
25 */
26
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
29
30 #include "vmxnet3_int.h"
31
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
34
35 /*
36 * PCI Device ID Table
37 * Last entry must be all 0s
38 */
39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
41 {0}
42 };
43
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
45
46 static int enable_mq = 1;
47
48 static void
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
50
51 /*
52 * Enable/Disable the given intr
53 */
54 static void
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
56 {
57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
58 }
59
60
61 static void
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
63 {
64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
65 }
66
67
68 /*
69 * Enable/Disable all intrs used by the device
70 */
71 static void
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
73 {
74 int i;
75
76 for (i = 0; i < adapter->intr.num_intrs; i++)
77 vmxnet3_enable_intr(adapter, i);
78 adapter->shared->devRead.intrConf.intrCtrl &=
79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
80 }
81
82
83 static void
84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
85 {
86 int i;
87
88 adapter->shared->devRead.intrConf.intrCtrl |=
89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
90 for (i = 0; i < adapter->intr.num_intrs; i++)
91 vmxnet3_disable_intr(adapter, i);
92 }
93
94
95 static void
96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
97 {
98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
99 }
100
101
102 static bool
103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
104 {
105 return tq->stopped;
106 }
107
108
109 static void
110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111 {
112 tq->stopped = false;
113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
114 }
115
116
117 static void
118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119 {
120 tq->stopped = false;
121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
122 }
123
124
125 static void
126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
127 {
128 tq->stopped = true;
129 tq->num_stop++;
130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
131 }
132
133
134 /*
135 * Check the link state. This may start or stop the tx queue.
136 */
137 static void
138 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
139 {
140 u32 ret;
141 int i;
142 unsigned long flags;
143
144 spin_lock_irqsave(&adapter->cmd_lock, flags);
145 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
146 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
147 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
148
149 adapter->link_speed = ret >> 16;
150 if (ret & 1) { /* Link is up. */
151 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
152 adapter->link_speed);
153 netif_carrier_on(adapter->netdev);
154
155 if (affectTxQueue) {
156 for (i = 0; i < adapter->num_tx_queues; i++)
157 vmxnet3_tq_start(&adapter->tx_queue[i],
158 adapter);
159 }
160 } else {
161 netdev_info(adapter->netdev, "NIC Link is Down\n");
162 netif_carrier_off(adapter->netdev);
163
164 if (affectTxQueue) {
165 for (i = 0; i < adapter->num_tx_queues; i++)
166 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
167 }
168 }
169 }
170
171 static void
172 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
173 {
174 int i;
175 unsigned long flags;
176 u32 events = le32_to_cpu(adapter->shared->ecr);
177 if (!events)
178 return;
179
180 vmxnet3_ack_events(adapter, events);
181
182 /* Check if link state has changed */
183 if (events & VMXNET3_ECR_LINK)
184 vmxnet3_check_link(adapter, true);
185
186 /* Check if there is an error on xmit/recv queues */
187 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
188 spin_lock_irqsave(&adapter->cmd_lock, flags);
189 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
190 VMXNET3_CMD_GET_QUEUE_STATUS);
191 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
192
193 for (i = 0; i < adapter->num_tx_queues; i++)
194 if (adapter->tqd_start[i].status.stopped)
195 dev_err(&adapter->netdev->dev,
196 "%s: tq[%d] error 0x%x\n",
197 adapter->netdev->name, i, le32_to_cpu(
198 adapter->tqd_start[i].status.error));
199 for (i = 0; i < adapter->num_rx_queues; i++)
200 if (adapter->rqd_start[i].status.stopped)
201 dev_err(&adapter->netdev->dev,
202 "%s: rq[%d] error 0x%x\n",
203 adapter->netdev->name, i,
204 adapter->rqd_start[i].status.error);
205
206 schedule_work(&adapter->work);
207 }
208 }
209
210 #ifdef __BIG_ENDIAN_BITFIELD
211 /*
212 * The device expects the bitfields in shared structures to be written in
213 * little endian. When CPU is big endian, the following routines are used to
214 * correctly read and write into ABI.
215 * The general technique used here is : double word bitfields are defined in
216 * opposite order for big endian architecture. Then before reading them in
217 * driver the complete double word is translated using le32_to_cpu. Similarly
218 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
219 * double words into required format.
220 * In order to avoid touching bits in shared structure more than once, temporary
221 * descriptors are used. These are passed as srcDesc to following functions.
222 */
223 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
224 struct Vmxnet3_RxDesc *dstDesc)
225 {
226 u32 *src = (u32 *)srcDesc + 2;
227 u32 *dst = (u32 *)dstDesc + 2;
228 dstDesc->addr = le64_to_cpu(srcDesc->addr);
229 *dst = le32_to_cpu(*src);
230 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
231 }
232
233 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
234 struct Vmxnet3_TxDesc *dstDesc)
235 {
236 int i;
237 u32 *src = (u32 *)(srcDesc + 1);
238 u32 *dst = (u32 *)(dstDesc + 1);
239
240 /* Working backwards so that the gen bit is set at the end. */
241 for (i = 2; i > 0; i--) {
242 src--;
243 dst--;
244 *dst = cpu_to_le32(*src);
245 }
246 }
247
248
249 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
250 struct Vmxnet3_RxCompDesc *dstDesc)
251 {
252 int i = 0;
253 u32 *src = (u32 *)srcDesc;
254 u32 *dst = (u32 *)dstDesc;
255 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
256 *dst = le32_to_cpu(*src);
257 src++;
258 dst++;
259 }
260 }
261
262
263 /* Used to read bitfield values from double words. */
264 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
265 {
266 u32 temp = le32_to_cpu(*bitfield);
267 u32 mask = ((1 << size) - 1) << pos;
268 temp &= mask;
269 temp >>= pos;
270 return temp;
271 }
272
273
274
275 #endif /* __BIG_ENDIAN_BITFIELD */
276
277 #ifdef __BIG_ENDIAN_BITFIELD
278
279 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
280 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
281 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
282 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
283 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
284 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
285 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
286 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
287 VMXNET3_TCD_GEN_SIZE)
288 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
289 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
290 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
291 (dstrcd) = (tmp); \
292 vmxnet3_RxCompToCPU((rcd), (tmp)); \
293 } while (0)
294 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
295 (dstrxd) = (tmp); \
296 vmxnet3_RxDescToCPU((rxd), (tmp)); \
297 } while (0)
298
299 #else
300
301 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
302 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
303 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
304 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
305 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
306 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
307
308 #endif /* __BIG_ENDIAN_BITFIELD */
309
310
311 static void
312 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
313 struct pci_dev *pdev)
314 {
315 if (tbi->map_type == VMXNET3_MAP_SINGLE)
316 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
317 PCI_DMA_TODEVICE);
318 else if (tbi->map_type == VMXNET3_MAP_PAGE)
319 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
320 PCI_DMA_TODEVICE);
321 else
322 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
323
324 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
325 }
326
327
328 static int
329 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
330 struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
331 {
332 struct sk_buff *skb;
333 int entries = 0;
334
335 /* no out of order completion */
336 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
337 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
338
339 skb = tq->buf_info[eop_idx].skb;
340 BUG_ON(skb == NULL);
341 tq->buf_info[eop_idx].skb = NULL;
342
343 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
344
345 while (tq->tx_ring.next2comp != eop_idx) {
346 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
347 pdev);
348
349 /* update next2comp w/o tx_lock. Since we are marking more,
350 * instead of less, tx ring entries avail, the worst case is
351 * that the tx routine incorrectly re-queues a pkt due to
352 * insufficient tx ring entries.
353 */
354 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
355 entries++;
356 }
357
358 dev_kfree_skb_any(skb);
359 return entries;
360 }
361
362
363 static int
364 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
365 struct vmxnet3_adapter *adapter)
366 {
367 int completed = 0;
368 union Vmxnet3_GenericDesc *gdesc;
369
370 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
371 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
372 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
373 &gdesc->tcd), tq, adapter->pdev,
374 adapter);
375
376 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
377 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
378 }
379
380 if (completed) {
381 spin_lock(&tq->tx_lock);
382 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
383 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
384 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
385 netif_carrier_ok(adapter->netdev))) {
386 vmxnet3_tq_wake(tq, adapter);
387 }
388 spin_unlock(&tq->tx_lock);
389 }
390 return completed;
391 }
392
393
394 static void
395 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
396 struct vmxnet3_adapter *adapter)
397 {
398 int i;
399
400 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
401 struct vmxnet3_tx_buf_info *tbi;
402
403 tbi = tq->buf_info + tq->tx_ring.next2comp;
404
405 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
406 if (tbi->skb) {
407 dev_kfree_skb_any(tbi->skb);
408 tbi->skb = NULL;
409 }
410 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
411 }
412
413 /* sanity check, verify all buffers are indeed unmapped and freed */
414 for (i = 0; i < tq->tx_ring.size; i++) {
415 BUG_ON(tq->buf_info[i].skb != NULL ||
416 tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
417 }
418
419 tq->tx_ring.gen = VMXNET3_INIT_GEN;
420 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
421
422 tq->comp_ring.gen = VMXNET3_INIT_GEN;
423 tq->comp_ring.next2proc = 0;
424 }
425
426
427 static void
428 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
429 struct vmxnet3_adapter *adapter)
430 {
431 if (tq->tx_ring.base) {
432 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
433 sizeof(struct Vmxnet3_TxDesc),
434 tq->tx_ring.base, tq->tx_ring.basePA);
435 tq->tx_ring.base = NULL;
436 }
437 if (tq->data_ring.base) {
438 dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
439 sizeof(struct Vmxnet3_TxDataDesc),
440 tq->data_ring.base, tq->data_ring.basePA);
441 tq->data_ring.base = NULL;
442 }
443 if (tq->comp_ring.base) {
444 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
445 sizeof(struct Vmxnet3_TxCompDesc),
446 tq->comp_ring.base, tq->comp_ring.basePA);
447 tq->comp_ring.base = NULL;
448 }
449 if (tq->buf_info) {
450 dma_free_coherent(&adapter->pdev->dev,
451 tq->tx_ring.size * sizeof(tq->buf_info[0]),
452 tq->buf_info, tq->buf_info_pa);
453 tq->buf_info = NULL;
454 }
455 }
456
457
458 /* Destroy all tx queues */
459 void
460 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
461 {
462 int i;
463
464 for (i = 0; i < adapter->num_tx_queues; i++)
465 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
466 }
467
468
469 static void
470 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
471 struct vmxnet3_adapter *adapter)
472 {
473 int i;
474
475 /* reset the tx ring contents to 0 and reset the tx ring states */
476 memset(tq->tx_ring.base, 0, tq->tx_ring.size *
477 sizeof(struct Vmxnet3_TxDesc));
478 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
479 tq->tx_ring.gen = VMXNET3_INIT_GEN;
480
481 memset(tq->data_ring.base, 0, tq->data_ring.size *
482 sizeof(struct Vmxnet3_TxDataDesc));
483
484 /* reset the tx comp ring contents to 0 and reset comp ring states */
485 memset(tq->comp_ring.base, 0, tq->comp_ring.size *
486 sizeof(struct Vmxnet3_TxCompDesc));
487 tq->comp_ring.next2proc = 0;
488 tq->comp_ring.gen = VMXNET3_INIT_GEN;
489
490 /* reset the bookkeeping data */
491 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
492 for (i = 0; i < tq->tx_ring.size; i++)
493 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
494
495 /* stats are not reset */
496 }
497
498
499 static int
500 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
501 struct vmxnet3_adapter *adapter)
502 {
503 size_t sz;
504
505 BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
506 tq->comp_ring.base || tq->buf_info);
507
508 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
509 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
510 &tq->tx_ring.basePA, GFP_KERNEL);
511 if (!tq->tx_ring.base) {
512 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
513 goto err;
514 }
515
516 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
517 tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
518 &tq->data_ring.basePA, GFP_KERNEL);
519 if (!tq->data_ring.base) {
520 netdev_err(adapter->netdev, "failed to allocate data ring\n");
521 goto err;
522 }
523
524 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
525 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
526 &tq->comp_ring.basePA, GFP_KERNEL);
527 if (!tq->comp_ring.base) {
528 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
529 goto err;
530 }
531
532 sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
533 tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
534 &tq->buf_info_pa, GFP_KERNEL);
535 if (!tq->buf_info)
536 goto err;
537
538 return 0;
539
540 err:
541 vmxnet3_tq_destroy(tq, adapter);
542 return -ENOMEM;
543 }
544
545 static void
546 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
547 {
548 int i;
549
550 for (i = 0; i < adapter->num_tx_queues; i++)
551 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
552 }
553
554 /*
555 * starting from ring->next2fill, allocate rx buffers for the given ring
556 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers
557 * are allocated or allocation fails
558 */
559
560 static int
561 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
562 int num_to_alloc, struct vmxnet3_adapter *adapter)
563 {
564 int num_allocated = 0;
565 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
566 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
567 u32 val;
568
569 while (num_allocated <= num_to_alloc) {
570 struct vmxnet3_rx_buf_info *rbi;
571 union Vmxnet3_GenericDesc *gd;
572
573 rbi = rbi_base + ring->next2fill;
574 gd = ring->base + ring->next2fill;
575
576 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
577 if (rbi->skb == NULL) {
578 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
579 rbi->len,
580 GFP_KERNEL);
581 if (unlikely(rbi->skb == NULL)) {
582 rq->stats.rx_buf_alloc_failure++;
583 break;
584 }
585
586 rbi->dma_addr = dma_map_single(
587 &adapter->pdev->dev,
588 rbi->skb->data, rbi->len,
589 PCI_DMA_FROMDEVICE);
590 } else {
591 /* rx buffer skipped by the device */
592 }
593 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
594 } else {
595 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
596 rbi->len != PAGE_SIZE);
597
598 if (rbi->page == NULL) {
599 rbi->page = alloc_page(GFP_ATOMIC);
600 if (unlikely(rbi->page == NULL)) {
601 rq->stats.rx_buf_alloc_failure++;
602 break;
603 }
604 rbi->dma_addr = dma_map_page(
605 &adapter->pdev->dev,
606 rbi->page, 0, PAGE_SIZE,
607 PCI_DMA_FROMDEVICE);
608 } else {
609 /* rx buffers skipped by the device */
610 }
611 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
612 }
613
614 BUG_ON(rbi->dma_addr == 0);
615 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
616 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
617 | val | rbi->len);
618
619 /* Fill the last buffer but dont mark it ready, or else the
620 * device will think that the queue is full */
621 if (num_allocated == num_to_alloc)
622 break;
623
624 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
625 num_allocated++;
626 vmxnet3_cmd_ring_adv_next2fill(ring);
627 }
628
629 netdev_dbg(adapter->netdev,
630 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
631 num_allocated, ring->next2fill, ring->next2comp);
632
633 /* so that the device can distinguish a full ring and an empty ring */
634 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
635
636 return num_allocated;
637 }
638
639
640 static void
641 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
642 struct vmxnet3_rx_buf_info *rbi)
643 {
644 struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
645 skb_shinfo(skb)->nr_frags;
646
647 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
648
649 __skb_frag_set_page(frag, rbi->page);
650 frag->page_offset = 0;
651 skb_frag_size_set(frag, rcd->len);
652 skb->data_len += rcd->len;
653 skb->truesize += PAGE_SIZE;
654 skb_shinfo(skb)->nr_frags++;
655 }
656
657
658 static void
659 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
660 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
661 struct vmxnet3_adapter *adapter)
662 {
663 u32 dw2, len;
664 unsigned long buf_offset;
665 int i;
666 union Vmxnet3_GenericDesc *gdesc;
667 struct vmxnet3_tx_buf_info *tbi = NULL;
668
669 BUG_ON(ctx->copy_size > skb_headlen(skb));
670
671 /* use the previous gen bit for the SOP desc */
672 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
673
674 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
675 gdesc = ctx->sop_txd; /* both loops below can be skipped */
676
677 /* no need to map the buffer if headers are copied */
678 if (ctx->copy_size) {
679 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
680 tq->tx_ring.next2fill *
681 sizeof(struct Vmxnet3_TxDataDesc));
682 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
683 ctx->sop_txd->dword[3] = 0;
684
685 tbi = tq->buf_info + tq->tx_ring.next2fill;
686 tbi->map_type = VMXNET3_MAP_NONE;
687
688 netdev_dbg(adapter->netdev,
689 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
690 tq->tx_ring.next2fill,
691 le64_to_cpu(ctx->sop_txd->txd.addr),
692 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
693 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
694
695 /* use the right gen for non-SOP desc */
696 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
697 }
698
699 /* linear part can use multiple tx desc if it's big */
700 len = skb_headlen(skb) - ctx->copy_size;
701 buf_offset = ctx->copy_size;
702 while (len) {
703 u32 buf_size;
704
705 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
706 buf_size = len;
707 dw2 |= len;
708 } else {
709 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
710 /* spec says that for TxDesc.len, 0 == 2^14 */
711 }
712
713 tbi = tq->buf_info + tq->tx_ring.next2fill;
714 tbi->map_type = VMXNET3_MAP_SINGLE;
715 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
716 skb->data + buf_offset, buf_size,
717 PCI_DMA_TODEVICE);
718
719 tbi->len = buf_size;
720
721 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
722 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
723
724 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
725 gdesc->dword[2] = cpu_to_le32(dw2);
726 gdesc->dword[3] = 0;
727
728 netdev_dbg(adapter->netdev,
729 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
730 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
731 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
732 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
733 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
734
735 len -= buf_size;
736 buf_offset += buf_size;
737 }
738
739 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
740 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
741 u32 buf_size;
742
743 buf_offset = 0;
744 len = skb_frag_size(frag);
745 while (len) {
746 tbi = tq->buf_info + tq->tx_ring.next2fill;
747 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
748 buf_size = len;
749 dw2 |= len;
750 } else {
751 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
752 /* spec says that for TxDesc.len, 0 == 2^14 */
753 }
754 tbi->map_type = VMXNET3_MAP_PAGE;
755 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
756 buf_offset, buf_size,
757 DMA_TO_DEVICE);
758
759 tbi->len = buf_size;
760
761 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
762 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
763
764 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
765 gdesc->dword[2] = cpu_to_le32(dw2);
766 gdesc->dword[3] = 0;
767
768 netdev_dbg(adapter->netdev,
769 "txd[%u]: 0x%llu %u %u\n",
770 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
771 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
772 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
773 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
774
775 len -= buf_size;
776 buf_offset += buf_size;
777 }
778 }
779
780 ctx->eop_txd = gdesc;
781
782 /* set the last buf_info for the pkt */
783 tbi->skb = skb;
784 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
785 }
786
787
788 /* Init all tx queues */
789 static void
790 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
791 {
792 int i;
793
794 for (i = 0; i < adapter->num_tx_queues; i++)
795 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
796 }
797
798
799 /*
800 * parse and copy relevant protocol headers:
801 * For a tso pkt, relevant headers are L2/3/4 including options
802 * For a pkt requesting csum offloading, they are L2/3 and may include L4
803 * if it's a TCP/UDP pkt
804 *
805 * Returns:
806 * -1: error happens during parsing
807 * 0: protocol headers parsed, but too big to be copied
808 * 1: protocol headers parsed and copied
809 *
810 * Other effects:
811 * 1. related *ctx fields are updated.
812 * 2. ctx->copy_size is # of bytes copied
813 * 3. the portion copied is guaranteed to be in the linear part
814 *
815 */
816 static int
817 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
818 struct vmxnet3_tx_ctx *ctx,
819 struct vmxnet3_adapter *adapter)
820 {
821 struct Vmxnet3_TxDataDesc *tdd;
822
823 if (ctx->mss) { /* TSO */
824 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
825 ctx->l4_hdr_size = tcp_hdrlen(skb);
826 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
827 } else {
828 if (skb->ip_summed == CHECKSUM_PARTIAL) {
829 ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
830
831 if (ctx->ipv4) {
832 const struct iphdr *iph = ip_hdr(skb);
833
834 if (iph->protocol == IPPROTO_TCP)
835 ctx->l4_hdr_size = tcp_hdrlen(skb);
836 else if (iph->protocol == IPPROTO_UDP)
837 ctx->l4_hdr_size = sizeof(struct udphdr);
838 else
839 ctx->l4_hdr_size = 0;
840 } else {
841 /* for simplicity, don't copy L4 headers */
842 ctx->l4_hdr_size = 0;
843 }
844 ctx->copy_size = min(ctx->eth_ip_hdr_size +
845 ctx->l4_hdr_size, skb->len);
846 } else {
847 ctx->eth_ip_hdr_size = 0;
848 ctx->l4_hdr_size = 0;
849 /* copy as much as allowed */
850 ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
851 , skb_headlen(skb));
852 }
853
854 /* make sure headers are accessible directly */
855 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
856 goto err;
857 }
858
859 if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
860 tq->stats.oversized_hdr++;
861 ctx->copy_size = 0;
862 return 0;
863 }
864
865 tdd = tq->data_ring.base + tq->tx_ring.next2fill;
866
867 memcpy(tdd->data, skb->data, ctx->copy_size);
868 netdev_dbg(adapter->netdev,
869 "copy %u bytes to dataRing[%u]\n",
870 ctx->copy_size, tq->tx_ring.next2fill);
871 return 1;
872
873 err:
874 return -1;
875 }
876
877
878 static void
879 vmxnet3_prepare_tso(struct sk_buff *skb,
880 struct vmxnet3_tx_ctx *ctx)
881 {
882 struct tcphdr *tcph = tcp_hdr(skb);
883
884 if (ctx->ipv4) {
885 struct iphdr *iph = ip_hdr(skb);
886
887 iph->check = 0;
888 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
889 IPPROTO_TCP, 0);
890 } else {
891 struct ipv6hdr *iph = ipv6_hdr(skb);
892
893 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
894 IPPROTO_TCP, 0);
895 }
896 }
897
898 static int txd_estimate(const struct sk_buff *skb)
899 {
900 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
901 int i;
902
903 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
904 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
905
906 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
907 }
908 return count;
909 }
910
911 /*
912 * Transmits a pkt thru a given tq
913 * Returns:
914 * NETDEV_TX_OK: descriptors are setup successfully
915 * NETDEV_TX_OK: error occurred, the pkt is dropped
916 * NETDEV_TX_BUSY: tx ring is full, queue is stopped
917 *
918 * Side-effects:
919 * 1. tx ring may be changed
920 * 2. tq stats may be updated accordingly
921 * 3. shared->txNumDeferred may be updated
922 */
923
924 static int
925 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926 struct vmxnet3_adapter *adapter, struct net_device *netdev)
927 {
928 int ret;
929 u32 count;
930 unsigned long flags;
931 struct vmxnet3_tx_ctx ctx;
932 union Vmxnet3_GenericDesc *gdesc;
933 #ifdef __BIG_ENDIAN_BITFIELD
934 /* Use temporary descriptor to avoid touching bits multiple times */
935 union Vmxnet3_GenericDesc tempTxDesc;
936 #endif
937
938 count = txd_estimate(skb);
939
940 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
941
942 ctx.mss = skb_shinfo(skb)->gso_size;
943 if (ctx.mss) {
944 if (skb_header_cloned(skb)) {
945 if (unlikely(pskb_expand_head(skb, 0, 0,
946 GFP_ATOMIC) != 0)) {
947 tq->stats.drop_tso++;
948 goto drop_pkt;
949 }
950 tq->stats.copy_skb_header++;
951 }
952 vmxnet3_prepare_tso(skb, &ctx);
953 } else {
954 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
955
956 /* non-tso pkts must not use more than
957 * VMXNET3_MAX_TXD_PER_PKT entries
958 */
959 if (skb_linearize(skb) != 0) {
960 tq->stats.drop_too_many_frags++;
961 goto drop_pkt;
962 }
963 tq->stats.linearized++;
964
965 /* recalculate the # of descriptors to use */
966 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
967 }
968 }
969
970 spin_lock_irqsave(&tq->tx_lock, flags);
971
972 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
973 tq->stats.tx_ring_full++;
974 netdev_dbg(adapter->netdev,
975 "tx queue stopped on %s, next2comp %u"
976 " next2fill %u\n", adapter->netdev->name,
977 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
978
979 vmxnet3_tq_stop(tq, adapter);
980 spin_unlock_irqrestore(&tq->tx_lock, flags);
981 return NETDEV_TX_BUSY;
982 }
983
984
985 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
986 if (ret >= 0) {
987 BUG_ON(ret <= 0 && ctx.copy_size != 0);
988 /* hdrs parsed, check against other limits */
989 if (ctx.mss) {
990 if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
991 VMXNET3_MAX_TX_BUF_SIZE)) {
992 goto hdr_too_big;
993 }
994 } else {
995 if (skb->ip_summed == CHECKSUM_PARTIAL) {
996 if (unlikely(ctx.eth_ip_hdr_size +
997 skb->csum_offset >
998 VMXNET3_MAX_CSUM_OFFSET)) {
999 goto hdr_too_big;
1000 }
1001 }
1002 }
1003 } else {
1004 tq->stats.drop_hdr_inspect_err++;
1005 goto unlock_drop_pkt;
1006 }
1007
1008 /* fill tx descs related to addr & len */
1009 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1010
1011 /* setup the EOP desc */
1012 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1013
1014 /* setup the SOP desc */
1015 #ifdef __BIG_ENDIAN_BITFIELD
1016 gdesc = &tempTxDesc;
1017 gdesc->dword[2] = ctx.sop_txd->dword[2];
1018 gdesc->dword[3] = ctx.sop_txd->dword[3];
1019 #else
1020 gdesc = ctx.sop_txd;
1021 #endif
1022 if (ctx.mss) {
1023 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1024 gdesc->txd.om = VMXNET3_OM_TSO;
1025 gdesc->txd.msscof = ctx.mss;
1026 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1027 gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
1028 } else {
1029 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1030 gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1031 gdesc->txd.om = VMXNET3_OM_CSUM;
1032 gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1033 skb->csum_offset;
1034 } else {
1035 gdesc->txd.om = 0;
1036 gdesc->txd.msscof = 0;
1037 }
1038 le32_add_cpu(&tq->shared->txNumDeferred, 1);
1039 }
1040
1041 if (vlan_tx_tag_present(skb)) {
1042 gdesc->txd.ti = 1;
1043 gdesc->txd.tci = vlan_tx_tag_get(skb);
1044 }
1045
1046 /* finally flips the GEN bit of the SOP desc. */
1047 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1048 VMXNET3_TXD_GEN);
1049 #ifdef __BIG_ENDIAN_BITFIELD
1050 /* Finished updating in bitfields of Tx Desc, so write them in original
1051 * place.
1052 */
1053 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1054 (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1055 gdesc = ctx.sop_txd;
1056 #endif
1057 netdev_dbg(adapter->netdev,
1058 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1059 (u32)(ctx.sop_txd -
1060 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1061 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1062
1063 spin_unlock_irqrestore(&tq->tx_lock, flags);
1064
1065 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1066 le32_to_cpu(tq->shared->txThreshold)) {
1067 tq->shared->txNumDeferred = 0;
1068 VMXNET3_WRITE_BAR0_REG(adapter,
1069 VMXNET3_REG_TXPROD + tq->qid * 8,
1070 tq->tx_ring.next2fill);
1071 }
1072
1073 return NETDEV_TX_OK;
1074
1075 hdr_too_big:
1076 tq->stats.drop_oversized_hdr++;
1077 unlock_drop_pkt:
1078 spin_unlock_irqrestore(&tq->tx_lock, flags);
1079 drop_pkt:
1080 tq->stats.drop_total++;
1081 dev_kfree_skb(skb);
1082 return NETDEV_TX_OK;
1083 }
1084
1085
1086 static netdev_tx_t
1087 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1088 {
1089 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1090
1091 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1092 return vmxnet3_tq_xmit(skb,
1093 &adapter->tx_queue[skb->queue_mapping],
1094 adapter, netdev);
1095 }
1096
1097
1098 static void
1099 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1100 struct sk_buff *skb,
1101 union Vmxnet3_GenericDesc *gdesc)
1102 {
1103 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1104 /* typical case: TCP/UDP over IP and both csums are correct */
1105 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
1106 VMXNET3_RCD_CSUM_OK) {
1107 skb->ip_summed = CHECKSUM_UNNECESSARY;
1108 BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1109 BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));
1110 BUG_ON(gdesc->rcd.frg);
1111 } else {
1112 if (gdesc->rcd.csum) {
1113 skb->csum = htons(gdesc->rcd.csum);
1114 skb->ip_summed = CHECKSUM_PARTIAL;
1115 } else {
1116 skb_checksum_none_assert(skb);
1117 }
1118 }
1119 } else {
1120 skb_checksum_none_assert(skb);
1121 }
1122 }
1123
1124
1125 static void
1126 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1127 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)
1128 {
1129 rq->stats.drop_err++;
1130 if (!rcd->fcs)
1131 rq->stats.drop_fcs++;
1132
1133 rq->stats.drop_total++;
1134
1135 /*
1136 * We do not unmap and chain the rx buffer to the skb.
1137 * We basically pretend this buffer is not used and will be recycled
1138 * by vmxnet3_rq_alloc_rx_buf()
1139 */
1140
1141 /*
1142 * ctx->skb may be NULL if this is the first and the only one
1143 * desc for the pkt
1144 */
1145 if (ctx->skb)
1146 dev_kfree_skb_irq(ctx->skb);
1147
1148 ctx->skb = NULL;
1149 }
1150
1151
1152 static int
1153 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1154 struct vmxnet3_adapter *adapter, int quota)
1155 {
1156 static const u32 rxprod_reg[2] = {
1157 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1158 };
1159 u32 num_rxd = 0;
1160 bool skip_page_frags = false;
1161 struct Vmxnet3_RxCompDesc *rcd;
1162 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1163 #ifdef __BIG_ENDIAN_BITFIELD
1164 struct Vmxnet3_RxDesc rxCmdDesc;
1165 struct Vmxnet3_RxCompDesc rxComp;
1166 #endif
1167 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1168 &rxComp);
1169 while (rcd->gen == rq->comp_ring.gen) {
1170 struct vmxnet3_rx_buf_info *rbi;
1171 struct sk_buff *skb, *new_skb = NULL;
1172 struct page *new_page = NULL;
1173 int num_to_alloc;
1174 struct Vmxnet3_RxDesc *rxd;
1175 u32 idx, ring_idx;
1176 struct vmxnet3_cmd_ring *ring = NULL;
1177 if (num_rxd >= quota) {
1178 /* we may stop even before we see the EOP desc of
1179 * the current pkt
1180 */
1181 break;
1182 }
1183 num_rxd++;
1184 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1185 idx = rcd->rxdIdx;
1186 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1187 ring = rq->rx_ring + ring_idx;
1188 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1189 &rxCmdDesc);
1190 rbi = rq->buf_info[ring_idx] + idx;
1191
1192 BUG_ON(rxd->addr != rbi->dma_addr ||
1193 rxd->len != rbi->len);
1194
1195 if (unlikely(rcd->eop && rcd->err)) {
1196 vmxnet3_rx_error(rq, rcd, ctx, adapter);
1197 goto rcd_done;
1198 }
1199
1200 if (rcd->sop) { /* first buf of the pkt */
1201 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1202 rcd->rqID != rq->qid);
1203
1204 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1205 BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1206
1207 if (unlikely(rcd->len == 0)) {
1208 /* Pretend the rx buffer is skipped. */
1209 BUG_ON(!(rcd->sop && rcd->eop));
1210 netdev_dbg(adapter->netdev,
1211 "rxRing[%u][%u] 0 length\n",
1212 ring_idx, idx);
1213 goto rcd_done;
1214 }
1215
1216 skip_page_frags = false;
1217 ctx->skb = rbi->skb;
1218 new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1219 rbi->len);
1220 if (new_skb == NULL) {
1221 /* Skb allocation failed, do not handover this
1222 * skb to stack. Reuse it. Drop the existing pkt
1223 */
1224 rq->stats.rx_buf_alloc_failure++;
1225 ctx->skb = NULL;
1226 rq->stats.drop_total++;
1227 skip_page_frags = true;
1228 goto rcd_done;
1229 }
1230
1231 dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
1232 rbi->len,
1233 PCI_DMA_FROMDEVICE);
1234
1235 #ifdef VMXNET3_RSS
1236 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1237 (adapter->netdev->features & NETIF_F_RXHASH))
1238 skb_set_hash(ctx->skb,
1239 le32_to_cpu(rcd->rssHash),
1240 PKT_HASH_TYPE_L3);
1241 #endif
1242 skb_put(ctx->skb, rcd->len);
1243
1244 /* Immediate refill */
1245 rbi->skb = new_skb;
1246 rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
1247 rbi->skb->data, rbi->len,
1248 PCI_DMA_FROMDEVICE);
1249 rxd->addr = cpu_to_le64(rbi->dma_addr);
1250 rxd->len = rbi->len;
1251
1252 } else {
1253 BUG_ON(ctx->skb == NULL && !skip_page_frags);
1254
1255 /* non SOP buffer must be type 1 in most cases */
1256 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1257 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1258
1259 /* If an sop buffer was dropped, skip all
1260 * following non-sop fragments. They will be reused.
1261 */
1262 if (skip_page_frags)
1263 goto rcd_done;
1264
1265 new_page = alloc_page(GFP_ATOMIC);
1266 if (unlikely(new_page == NULL)) {
1267 /* Replacement page frag could not be allocated.
1268 * Reuse this page. Drop the pkt and free the
1269 * skb which contained this page as a frag. Skip
1270 * processing all the following non-sop frags.
1271 */
1272 rq->stats.rx_buf_alloc_failure++;
1273 dev_kfree_skb(ctx->skb);
1274 ctx->skb = NULL;
1275 skip_page_frags = true;
1276 goto rcd_done;
1277 }
1278
1279 if (rcd->len) {
1280 dma_unmap_page(&adapter->pdev->dev,
1281 rbi->dma_addr, rbi->len,
1282 PCI_DMA_FROMDEVICE);
1283
1284 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1285 }
1286
1287 /* Immediate refill */
1288 rbi->page = new_page;
1289 rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
1290 rbi->page,
1291 0, PAGE_SIZE,
1292 PCI_DMA_FROMDEVICE);
1293 rxd->addr = cpu_to_le64(rbi->dma_addr);
1294 rxd->len = rbi->len;
1295 }
1296
1297
1298 skb = ctx->skb;
1299 if (rcd->eop) {
1300 skb->len += skb->data_len;
1301
1302 vmxnet3_rx_csum(adapter, skb,
1303 (union Vmxnet3_GenericDesc *)rcd);
1304 skb->protocol = eth_type_trans(skb, adapter->netdev);
1305
1306 if (unlikely(rcd->ts))
1307 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1308
1309 if (adapter->netdev->features & NETIF_F_LRO)
1310 netif_receive_skb(skb);
1311 else
1312 napi_gro_receive(&rq->napi, skb);
1313
1314 ctx->skb = NULL;
1315 }
1316
1317 rcd_done:
1318 /* device may have skipped some rx descs */
1319 ring->next2comp = idx;
1320 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1321 ring = rq->rx_ring + ring_idx;
1322 while (num_to_alloc) {
1323 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1324 &rxCmdDesc);
1325 BUG_ON(!rxd->addr);
1326
1327 /* Recv desc is ready to be used by the device */
1328 rxd->gen = ring->gen;
1329 vmxnet3_cmd_ring_adv_next2fill(ring);
1330 num_to_alloc--;
1331 }
1332
1333 /* if needed, update the register */
1334 if (unlikely(rq->shared->updateRxProd)) {
1335 VMXNET3_WRITE_BAR0_REG(adapter,
1336 rxprod_reg[ring_idx] + rq->qid * 8,
1337 ring->next2fill);
1338 }
1339
1340 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1341 vmxnet3_getRxComp(rcd,
1342 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1343 }
1344
1345 return num_rxd;
1346 }
1347
1348
1349 static void
1350 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1351 struct vmxnet3_adapter *adapter)
1352 {
1353 u32 i, ring_idx;
1354 struct Vmxnet3_RxDesc *rxd;
1355
1356 for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1357 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1358 #ifdef __BIG_ENDIAN_BITFIELD
1359 struct Vmxnet3_RxDesc rxDesc;
1360 #endif
1361 vmxnet3_getRxDesc(rxd,
1362 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1363
1364 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1365 rq->buf_info[ring_idx][i].skb) {
1366 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1367 rxd->len, PCI_DMA_FROMDEVICE);
1368 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1369 rq->buf_info[ring_idx][i].skb = NULL;
1370 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1371 rq->buf_info[ring_idx][i].page) {
1372 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1373 rxd->len, PCI_DMA_FROMDEVICE);
1374 put_page(rq->buf_info[ring_idx][i].page);
1375 rq->buf_info[ring_idx][i].page = NULL;
1376 }
1377 }
1378
1379 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1380 rq->rx_ring[ring_idx].next2fill =
1381 rq->rx_ring[ring_idx].next2comp = 0;
1382 }
1383
1384 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1385 rq->comp_ring.next2proc = 0;
1386 }
1387
1388
1389 static void
1390 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1391 {
1392 int i;
1393
1394 for (i = 0; i < adapter->num_rx_queues; i++)
1395 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1396 }
1397
1398
1399 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1400 struct vmxnet3_adapter *adapter)
1401 {
1402 int i;
1403 int j;
1404
1405 /* all rx buffers must have already been freed */
1406 for (i = 0; i < 2; i++) {
1407 if (rq->buf_info[i]) {
1408 for (j = 0; j < rq->rx_ring[i].size; j++)
1409 BUG_ON(rq->buf_info[i][j].page != NULL);
1410 }
1411 }
1412
1413
1414 for (i = 0; i < 2; i++) {
1415 if (rq->rx_ring[i].base) {
1416 dma_free_coherent(&adapter->pdev->dev,
1417 rq->rx_ring[i].size
1418 * sizeof(struct Vmxnet3_RxDesc),
1419 rq->rx_ring[i].base,
1420 rq->rx_ring[i].basePA);
1421 rq->rx_ring[i].base = NULL;
1422 }
1423 rq->buf_info[i] = NULL;
1424 }
1425
1426 if (rq->comp_ring.base) {
1427 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1428 * sizeof(struct Vmxnet3_RxCompDesc),
1429 rq->comp_ring.base, rq->comp_ring.basePA);
1430 rq->comp_ring.base = NULL;
1431 }
1432
1433 if (rq->buf_info[0]) {
1434 size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
1435 (rq->rx_ring[0].size + rq->rx_ring[1].size);
1436 dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
1437 rq->buf_info_pa);
1438 }
1439 }
1440
1441
1442 static int
1443 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1444 struct vmxnet3_adapter *adapter)
1445 {
1446 int i;
1447
1448 /* initialize buf_info */
1449 for (i = 0; i < rq->rx_ring[0].size; i++) {
1450
1451 /* 1st buf for a pkt is skbuff */
1452 if (i % adapter->rx_buf_per_pkt == 0) {
1453 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1454 rq->buf_info[0][i].len = adapter->skb_buf_size;
1455 } else { /* subsequent bufs for a pkt is frag */
1456 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1457 rq->buf_info[0][i].len = PAGE_SIZE;
1458 }
1459 }
1460 for (i = 0; i < rq->rx_ring[1].size; i++) {
1461 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1462 rq->buf_info[1][i].len = PAGE_SIZE;
1463 }
1464
1465 /* reset internal state and allocate buffers for both rings */
1466 for (i = 0; i < 2; i++) {
1467 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1468
1469 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1470 sizeof(struct Vmxnet3_RxDesc));
1471 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1472 }
1473 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1474 adapter) == 0) {
1475 /* at least has 1 rx buffer for the 1st ring */
1476 return -ENOMEM;
1477 }
1478 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1479
1480 /* reset the comp ring */
1481 rq->comp_ring.next2proc = 0;
1482 memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1483 sizeof(struct Vmxnet3_RxCompDesc));
1484 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1485
1486 /* reset rxctx */
1487 rq->rx_ctx.skb = NULL;
1488
1489 /* stats are not reset */
1490 return 0;
1491 }
1492
1493
1494 static int
1495 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1496 {
1497 int i, err = 0;
1498
1499 for (i = 0; i < adapter->num_rx_queues; i++) {
1500 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1501 if (unlikely(err)) {
1502 dev_err(&adapter->netdev->dev, "%s: failed to "
1503 "initialize rx queue%i\n",
1504 adapter->netdev->name, i);
1505 break;
1506 }
1507 }
1508 return err;
1509
1510 }
1511
1512
1513 static int
1514 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1515 {
1516 int i;
1517 size_t sz;
1518 struct vmxnet3_rx_buf_info *bi;
1519
1520 for (i = 0; i < 2; i++) {
1521
1522 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1523 rq->rx_ring[i].base = dma_alloc_coherent(
1524 &adapter->pdev->dev, sz,
1525 &rq->rx_ring[i].basePA,
1526 GFP_KERNEL);
1527 if (!rq->rx_ring[i].base) {
1528 netdev_err(adapter->netdev,
1529 "failed to allocate rx ring %d\n", i);
1530 goto err;
1531 }
1532 }
1533
1534 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1535 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1536 &rq->comp_ring.basePA,
1537 GFP_KERNEL);
1538 if (!rq->comp_ring.base) {
1539 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1540 goto err;
1541 }
1542
1543 sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1544 rq->rx_ring[1].size);
1545 bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
1546 GFP_KERNEL);
1547 if (!bi)
1548 goto err;
1549
1550 rq->buf_info[0] = bi;
1551 rq->buf_info[1] = bi + rq->rx_ring[0].size;
1552
1553 return 0;
1554
1555 err:
1556 vmxnet3_rq_destroy(rq, adapter);
1557 return -ENOMEM;
1558 }
1559
1560
1561 static int
1562 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1563 {
1564 int i, err = 0;
1565
1566 for (i = 0; i < adapter->num_rx_queues; i++) {
1567 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1568 if (unlikely(err)) {
1569 dev_err(&adapter->netdev->dev,
1570 "%s: failed to create rx queue%i\n",
1571 adapter->netdev->name, i);
1572 goto err_out;
1573 }
1574 }
1575 return err;
1576 err_out:
1577 vmxnet3_rq_destroy_all(adapter);
1578 return err;
1579
1580 }
1581
1582 /* Multiple queue aware polling function for tx and rx */
1583
1584 static int
1585 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1586 {
1587 int rcd_done = 0, i;
1588 if (unlikely(adapter->shared->ecr))
1589 vmxnet3_process_events(adapter);
1590 for (i = 0; i < adapter->num_tx_queues; i++)
1591 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1592
1593 for (i = 0; i < adapter->num_rx_queues; i++)
1594 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1595 adapter, budget);
1596 return rcd_done;
1597 }
1598
1599
1600 static int
1601 vmxnet3_poll(struct napi_struct *napi, int budget)
1602 {
1603 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1604 struct vmxnet3_rx_queue, napi);
1605 int rxd_done;
1606
1607 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1608
1609 if (rxd_done < budget) {
1610 napi_complete(napi);
1611 vmxnet3_enable_all_intrs(rx_queue->adapter);
1612 }
1613 return rxd_done;
1614 }
1615
1616 /*
1617 * NAPI polling function for MSI-X mode with multiple Rx queues
1618 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1619 */
1620
1621 static int
1622 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1623 {
1624 struct vmxnet3_rx_queue *rq = container_of(napi,
1625 struct vmxnet3_rx_queue, napi);
1626 struct vmxnet3_adapter *adapter = rq->adapter;
1627 int rxd_done;
1628
1629 /* When sharing interrupt with corresponding tx queue, process
1630 * tx completions in that queue as well
1631 */
1632 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1633 struct vmxnet3_tx_queue *tq =
1634 &adapter->tx_queue[rq - adapter->rx_queue];
1635 vmxnet3_tq_tx_complete(tq, adapter);
1636 }
1637
1638 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1639
1640 if (rxd_done < budget) {
1641 napi_complete(napi);
1642 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1643 }
1644 return rxd_done;
1645 }
1646
1647
1648 #ifdef CONFIG_PCI_MSI
1649
1650 /*
1651 * Handle completion interrupts on tx queues
1652 * Returns whether or not the intr is handled
1653 */
1654
1655 static irqreturn_t
1656 vmxnet3_msix_tx(int irq, void *data)
1657 {
1658 struct vmxnet3_tx_queue *tq = data;
1659 struct vmxnet3_adapter *adapter = tq->adapter;
1660
1661 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1662 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1663
1664 /* Handle the case where only one irq is allocate for all tx queues */
1665 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1666 int i;
1667 for (i = 0; i < adapter->num_tx_queues; i++) {
1668 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1669 vmxnet3_tq_tx_complete(txq, adapter);
1670 }
1671 } else {
1672 vmxnet3_tq_tx_complete(tq, adapter);
1673 }
1674 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1675
1676 return IRQ_HANDLED;
1677 }
1678
1679
1680 /*
1681 * Handle completion interrupts on rx queues. Returns whether or not the
1682 * intr is handled
1683 */
1684
1685 static irqreturn_t
1686 vmxnet3_msix_rx(int irq, void *data)
1687 {
1688 struct vmxnet3_rx_queue *rq = data;
1689 struct vmxnet3_adapter *adapter = rq->adapter;
1690
1691 /* disable intr if needed */
1692 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1693 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1694 napi_schedule(&rq->napi);
1695
1696 return IRQ_HANDLED;
1697 }
1698
1699 /*
1700 *----------------------------------------------------------------------------
1701 *
1702 * vmxnet3_msix_event --
1703 *
1704 * vmxnet3 msix event intr handler
1705 *
1706 * Result:
1707 * whether or not the intr is handled
1708 *
1709 *----------------------------------------------------------------------------
1710 */
1711
1712 static irqreturn_t
1713 vmxnet3_msix_event(int irq, void *data)
1714 {
1715 struct net_device *dev = data;
1716 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1717
1718 /* disable intr if needed */
1719 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1720 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1721
1722 if (adapter->shared->ecr)
1723 vmxnet3_process_events(adapter);
1724
1725 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1726
1727 return IRQ_HANDLED;
1728 }
1729
1730 #endif /* CONFIG_PCI_MSI */
1731
1732
1733 /* Interrupt handler for vmxnet3 */
1734 static irqreturn_t
1735 vmxnet3_intr(int irq, void *dev_id)
1736 {
1737 struct net_device *dev = dev_id;
1738 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1739
1740 if (adapter->intr.type == VMXNET3_IT_INTX) {
1741 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1742 if (unlikely(icr == 0))
1743 /* not ours */
1744 return IRQ_NONE;
1745 }
1746
1747
1748 /* disable intr if needed */
1749 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1750 vmxnet3_disable_all_intrs(adapter);
1751
1752 napi_schedule(&adapter->rx_queue[0].napi);
1753
1754 return IRQ_HANDLED;
1755 }
1756
1757 #ifdef CONFIG_NET_POLL_CONTROLLER
1758
1759 /* netpoll callback. */
1760 static void
1761 vmxnet3_netpoll(struct net_device *netdev)
1762 {
1763 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1764
1765 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1766 vmxnet3_disable_all_intrs(adapter);
1767
1768 vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
1769 vmxnet3_enable_all_intrs(adapter);
1770
1771 }
1772 #endif /* CONFIG_NET_POLL_CONTROLLER */
1773
1774 static int
1775 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1776 {
1777 struct vmxnet3_intr *intr = &adapter->intr;
1778 int err = 0, i;
1779 int vector = 0;
1780
1781 #ifdef CONFIG_PCI_MSI
1782 if (adapter->intr.type == VMXNET3_IT_MSIX) {
1783 for (i = 0; i < adapter->num_tx_queues; i++) {
1784 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1785 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1786 adapter->netdev->name, vector);
1787 err = request_irq(
1788 intr->msix_entries[vector].vector,
1789 vmxnet3_msix_tx, 0,
1790 adapter->tx_queue[i].name,
1791 &adapter->tx_queue[i]);
1792 } else {
1793 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1794 adapter->netdev->name, vector);
1795 }
1796 if (err) {
1797 dev_err(&adapter->netdev->dev,
1798 "Failed to request irq for MSIX, %s, "
1799 "error %d\n",
1800 adapter->tx_queue[i].name, err);
1801 return err;
1802 }
1803
1804 /* Handle the case where only 1 MSIx was allocated for
1805 * all tx queues */
1806 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1807 for (; i < adapter->num_tx_queues; i++)
1808 adapter->tx_queue[i].comp_ring.intr_idx
1809 = vector;
1810 vector++;
1811 break;
1812 } else {
1813 adapter->tx_queue[i].comp_ring.intr_idx
1814 = vector++;
1815 }
1816 }
1817 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1818 vector = 0;
1819
1820 for (i = 0; i < adapter->num_rx_queues; i++) {
1821 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1822 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1823 adapter->netdev->name, vector);
1824 else
1825 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1826 adapter->netdev->name, vector);
1827 err = request_irq(intr->msix_entries[vector].vector,
1828 vmxnet3_msix_rx, 0,
1829 adapter->rx_queue[i].name,
1830 &(adapter->rx_queue[i]));
1831 if (err) {
1832 netdev_err(adapter->netdev,
1833 "Failed to request irq for MSIX, "
1834 "%s, error %d\n",
1835 adapter->rx_queue[i].name, err);
1836 return err;
1837 }
1838
1839 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1840 }
1841
1842 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1843 adapter->netdev->name, vector);
1844 err = request_irq(intr->msix_entries[vector].vector,
1845 vmxnet3_msix_event, 0,
1846 intr->event_msi_vector_name, adapter->netdev);
1847 intr->event_intr_idx = vector;
1848
1849 } else if (intr->type == VMXNET3_IT_MSI) {
1850 adapter->num_rx_queues = 1;
1851 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1852 adapter->netdev->name, adapter->netdev);
1853 } else {
1854 #endif
1855 adapter->num_rx_queues = 1;
1856 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1857 IRQF_SHARED, adapter->netdev->name,
1858 adapter->netdev);
1859 #ifdef CONFIG_PCI_MSI
1860 }
1861 #endif
1862 intr->num_intrs = vector + 1;
1863 if (err) {
1864 netdev_err(adapter->netdev,
1865 "Failed to request irq (intr type:%d), error %d\n",
1866 intr->type, err);
1867 } else {
1868 /* Number of rx queues will not change after this */
1869 for (i = 0; i < adapter->num_rx_queues; i++) {
1870 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1871 rq->qid = i;
1872 rq->qid2 = i + adapter->num_rx_queues;
1873 }
1874
1875
1876
1877 /* init our intr settings */
1878 for (i = 0; i < intr->num_intrs; i++)
1879 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1880 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1881 adapter->intr.event_intr_idx = 0;
1882 for (i = 0; i < adapter->num_tx_queues; i++)
1883 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1884 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1885 }
1886
1887 netdev_info(adapter->netdev,
1888 "intr type %u, mode %u, %u vectors allocated\n",
1889 intr->type, intr->mask_mode, intr->num_intrs);
1890 }
1891
1892 return err;
1893 }
1894
1895
1896 static void
1897 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1898 {
1899 struct vmxnet3_intr *intr = &adapter->intr;
1900 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1901
1902 switch (intr->type) {
1903 #ifdef CONFIG_PCI_MSI
1904 case VMXNET3_IT_MSIX:
1905 {
1906 int i, vector = 0;
1907
1908 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1909 for (i = 0; i < adapter->num_tx_queues; i++) {
1910 free_irq(intr->msix_entries[vector++].vector,
1911 &(adapter->tx_queue[i]));
1912 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1913 break;
1914 }
1915 }
1916
1917 for (i = 0; i < adapter->num_rx_queues; i++) {
1918 free_irq(intr->msix_entries[vector++].vector,
1919 &(adapter->rx_queue[i]));
1920 }
1921
1922 free_irq(intr->msix_entries[vector].vector,
1923 adapter->netdev);
1924 BUG_ON(vector >= intr->num_intrs);
1925 break;
1926 }
1927 #endif
1928 case VMXNET3_IT_MSI:
1929 free_irq(adapter->pdev->irq, adapter->netdev);
1930 break;
1931 case VMXNET3_IT_INTX:
1932 free_irq(adapter->pdev->irq, adapter->netdev);
1933 break;
1934 default:
1935 BUG();
1936 }
1937 }
1938
1939
1940 static void
1941 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1942 {
1943 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1944 u16 vid;
1945
1946 /* allow untagged pkts */
1947 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1948
1949 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1950 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1951 }
1952
1953
1954 static int
1955 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1956 {
1957 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1958
1959 if (!(netdev->flags & IFF_PROMISC)) {
1960 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1961 unsigned long flags;
1962
1963 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1964 spin_lock_irqsave(&adapter->cmd_lock, flags);
1965 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1966 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1967 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1968 }
1969
1970 set_bit(vid, adapter->active_vlans);
1971
1972 return 0;
1973 }
1974
1975
1976 static int
1977 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1978 {
1979 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1980
1981 if (!(netdev->flags & IFF_PROMISC)) {
1982 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1983 unsigned long flags;
1984
1985 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1986 spin_lock_irqsave(&adapter->cmd_lock, flags);
1987 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1988 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1989 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1990 }
1991
1992 clear_bit(vid, adapter->active_vlans);
1993
1994 return 0;
1995 }
1996
1997
1998 static u8 *
1999 vmxnet3_copy_mc(struct net_device *netdev)
2000 {
2001 u8 *buf = NULL;
2002 u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2003
2004 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2005 if (sz <= 0xffff) {
2006 /* We may be called with BH disabled */
2007 buf = kmalloc(sz, GFP_ATOMIC);
2008 if (buf) {
2009 struct netdev_hw_addr *ha;
2010 int i = 0;
2011
2012 netdev_for_each_mc_addr(ha, netdev)
2013 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2014 ETH_ALEN);
2015 }
2016 }
2017 return buf;
2018 }
2019
2020
2021 static void
2022 vmxnet3_set_mc(struct net_device *netdev)
2023 {
2024 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2025 unsigned long flags;
2026 struct Vmxnet3_RxFilterConf *rxConf =
2027 &adapter->shared->devRead.rxFilterConf;
2028 u8 *new_table = NULL;
2029 dma_addr_t new_table_pa = 0;
2030 u32 new_mode = VMXNET3_RXM_UCAST;
2031
2032 if (netdev->flags & IFF_PROMISC) {
2033 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2034 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2035
2036 new_mode |= VMXNET3_RXM_PROMISC;
2037 } else {
2038 vmxnet3_restore_vlan(adapter);
2039 }
2040
2041 if (netdev->flags & IFF_BROADCAST)
2042 new_mode |= VMXNET3_RXM_BCAST;
2043
2044 if (netdev->flags & IFF_ALLMULTI)
2045 new_mode |= VMXNET3_RXM_ALL_MULTI;
2046 else
2047 if (!netdev_mc_empty(netdev)) {
2048 new_table = vmxnet3_copy_mc(netdev);
2049 if (new_table) {
2050 new_mode |= VMXNET3_RXM_MCAST;
2051 rxConf->mfTableLen = cpu_to_le16(
2052 netdev_mc_count(netdev) * ETH_ALEN);
2053 new_table_pa = dma_map_single(
2054 &adapter->pdev->dev,
2055 new_table,
2056 rxConf->mfTableLen,
2057 PCI_DMA_TODEVICE);
2058 rxConf->mfTablePA = cpu_to_le64(new_table_pa);
2059 } else {
2060 netdev_info(netdev, "failed to copy mcast list"
2061 ", setting ALL_MULTI\n");
2062 new_mode |= VMXNET3_RXM_ALL_MULTI;
2063 }
2064 }
2065
2066
2067 if (!(new_mode & VMXNET3_RXM_MCAST)) {
2068 rxConf->mfTableLen = 0;
2069 rxConf->mfTablePA = 0;
2070 }
2071
2072 spin_lock_irqsave(&adapter->cmd_lock, flags);
2073 if (new_mode != rxConf->rxMode) {
2074 rxConf->rxMode = cpu_to_le32(new_mode);
2075 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2076 VMXNET3_CMD_UPDATE_RX_MODE);
2077 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2078 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2079 }
2080
2081 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2082 VMXNET3_CMD_UPDATE_MAC_FILTERS);
2083 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2084
2085 if (new_table) {
2086 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2087 rxConf->mfTableLen, PCI_DMA_TODEVICE);
2088 kfree(new_table);
2089 }
2090 }
2091
2092 void
2093 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2094 {
2095 int i;
2096
2097 for (i = 0; i < adapter->num_rx_queues; i++)
2098 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2099 }
2100
2101
2102 /*
2103 * Set up driver_shared based on settings in adapter.
2104 */
2105
2106 static void
2107 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2108 {
2109 struct Vmxnet3_DriverShared *shared = adapter->shared;
2110 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2111 struct Vmxnet3_TxQueueConf *tqc;
2112 struct Vmxnet3_RxQueueConf *rqc;
2113 int i;
2114
2115 memset(shared, 0, sizeof(*shared));
2116
2117 /* driver settings */
2118 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2119 devRead->misc.driverInfo.version = cpu_to_le32(
2120 VMXNET3_DRIVER_VERSION_NUM);
2121 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2122 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2123 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2124 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2125 *((u32 *)&devRead->misc.driverInfo.gos));
2126 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2127 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2128
2129 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2130 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2131
2132 /* set up feature flags */
2133 if (adapter->netdev->features & NETIF_F_RXCSUM)
2134 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2135
2136 if (adapter->netdev->features & NETIF_F_LRO) {
2137 devRead->misc.uptFeatures |= UPT1_F_LRO;
2138 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2139 }
2140 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2141 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2142
2143 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2144 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2145 devRead->misc.queueDescLen = cpu_to_le32(
2146 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2147 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2148
2149 /* tx queue settings */
2150 devRead->misc.numTxQueues = adapter->num_tx_queues;
2151 for (i = 0; i < adapter->num_tx_queues; i++) {
2152 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2153 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2154 tqc = &adapter->tqd_start[i].conf;
2155 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
2156 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2157 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2158 tqc->ddPA = cpu_to_le64(tq->buf_info_pa);
2159 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
2160 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
2161 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
2162 tqc->ddLen = cpu_to_le32(
2163 sizeof(struct vmxnet3_tx_buf_info) *
2164 tqc->txRingSize);
2165 tqc->intrIdx = tq->comp_ring.intr_idx;
2166 }
2167
2168 /* rx queue settings */
2169 devRead->misc.numRxQueues = adapter->num_rx_queues;
2170 for (i = 0; i < adapter->num_rx_queues; i++) {
2171 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2172 rqc = &adapter->rqd_start[i].conf;
2173 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2174 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2175 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
2176 rqc->ddPA = cpu_to_le64(rq->buf_info_pa);
2177 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
2178 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
2179 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
2180 rqc->ddLen = cpu_to_le32(
2181 sizeof(struct vmxnet3_rx_buf_info) *
2182 (rqc->rxRingSize[0] +
2183 rqc->rxRingSize[1]));
2184 rqc->intrIdx = rq->comp_ring.intr_idx;
2185 }
2186
2187 #ifdef VMXNET3_RSS
2188 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2189
2190 if (adapter->rss) {
2191 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2192 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2193 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2194 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2195 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2196 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2197 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2198 };
2199
2200 devRead->misc.uptFeatures |= UPT1_F_RSS;
2201 devRead->misc.numRxQueues = adapter->num_rx_queues;
2202 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2203 UPT1_RSS_HASH_TYPE_IPV4 |
2204 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2205 UPT1_RSS_HASH_TYPE_IPV6;
2206 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2207 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2208 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2209 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2210
2211 for (i = 0; i < rssConf->indTableSize; i++)
2212 rssConf->indTable[i] = ethtool_rxfh_indir_default(
2213 i, adapter->num_rx_queues);
2214
2215 devRead->rssConfDesc.confVer = 1;
2216 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2217 devRead->rssConfDesc.confPA =
2218 cpu_to_le64(adapter->rss_conf_pa);
2219 }
2220
2221 #endif /* VMXNET3_RSS */
2222
2223 /* intr settings */
2224 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2225 VMXNET3_IMM_AUTO;
2226 devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2227 for (i = 0; i < adapter->intr.num_intrs; i++)
2228 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2229
2230 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2231 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2232
2233 /* rx filter settings */
2234 devRead->rxFilterConf.rxMode = 0;
2235 vmxnet3_restore_vlan(adapter);
2236 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2237
2238 /* the rest are already zeroed */
2239 }
2240
2241
2242 int
2243 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2244 {
2245 int err, i;
2246 u32 ret;
2247 unsigned long flags;
2248
2249 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2250 " ring sizes %u %u %u\n", adapter->netdev->name,
2251 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2252 adapter->tx_queue[0].tx_ring.size,
2253 adapter->rx_queue[0].rx_ring[0].size,
2254 adapter->rx_queue[0].rx_ring[1].size);
2255
2256 vmxnet3_tq_init_all(adapter);
2257 err = vmxnet3_rq_init_all(adapter);
2258 if (err) {
2259 netdev_err(adapter->netdev,
2260 "Failed to init rx queue error %d\n", err);
2261 goto rq_err;
2262 }
2263
2264 err = vmxnet3_request_irqs(adapter);
2265 if (err) {
2266 netdev_err(adapter->netdev,
2267 "Failed to setup irq for error %d\n", err);
2268 goto irq_err;
2269 }
2270
2271 vmxnet3_setup_driver_shared(adapter);
2272
2273 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2274 adapter->shared_pa));
2275 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2276 adapter->shared_pa));
2277 spin_lock_irqsave(&adapter->cmd_lock, flags);
2278 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2279 VMXNET3_CMD_ACTIVATE_DEV);
2280 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2281 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2282
2283 if (ret != 0) {
2284 netdev_err(adapter->netdev,
2285 "Failed to activate dev: error %u\n", ret);
2286 err = -EINVAL;
2287 goto activate_err;
2288 }
2289
2290 for (i = 0; i < adapter->num_rx_queues; i++) {
2291 VMXNET3_WRITE_BAR0_REG(adapter,
2292 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2293 adapter->rx_queue[i].rx_ring[0].next2fill);
2294 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2295 (i * VMXNET3_REG_ALIGN)),
2296 adapter->rx_queue[i].rx_ring[1].next2fill);
2297 }
2298
2299 /* Apply the rx filter settins last. */
2300 vmxnet3_set_mc(adapter->netdev);
2301
2302 /*
2303 * Check link state when first activating device. It will start the
2304 * tx queue if the link is up.
2305 */
2306 vmxnet3_check_link(adapter, true);
2307 for (i = 0; i < adapter->num_rx_queues; i++)
2308 napi_enable(&adapter->rx_queue[i].napi);
2309 vmxnet3_enable_all_intrs(adapter);
2310 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2311 return 0;
2312
2313 activate_err:
2314 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2315 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2316 vmxnet3_free_irqs(adapter);
2317 irq_err:
2318 rq_err:
2319 /* free up buffers we allocated */
2320 vmxnet3_rq_cleanup_all(adapter);
2321 return err;
2322 }
2323
2324
2325 void
2326 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2327 {
2328 unsigned long flags;
2329 spin_lock_irqsave(&adapter->cmd_lock, flags);
2330 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2331 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2332 }
2333
2334
2335 int
2336 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2337 {
2338 int i;
2339 unsigned long flags;
2340 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2341 return 0;
2342
2343
2344 spin_lock_irqsave(&adapter->cmd_lock, flags);
2345 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2346 VMXNET3_CMD_QUIESCE_DEV);
2347 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2348 vmxnet3_disable_all_intrs(adapter);
2349
2350 for (i = 0; i < adapter->num_rx_queues; i++)
2351 napi_disable(&adapter->rx_queue[i].napi);
2352 netif_tx_disable(adapter->netdev);
2353 adapter->link_speed = 0;
2354 netif_carrier_off(adapter->netdev);
2355
2356 vmxnet3_tq_cleanup_all(adapter);
2357 vmxnet3_rq_cleanup_all(adapter);
2358 vmxnet3_free_irqs(adapter);
2359 return 0;
2360 }
2361
2362
2363 static void
2364 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2365 {
2366 u32 tmp;
2367
2368 tmp = *(u32 *)mac;
2369 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2370
2371 tmp = (mac[5] << 8) | mac[4];
2372 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2373 }
2374
2375
2376 static int
2377 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2378 {
2379 struct sockaddr *addr = p;
2380 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2381
2382 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2383 vmxnet3_write_mac_addr(adapter, addr->sa_data);
2384
2385 return 0;
2386 }
2387
2388
2389 /* ==================== initialization and cleanup routines ============ */
2390
2391 static int
2392 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2393 {
2394 int err;
2395 unsigned long mmio_start, mmio_len;
2396 struct pci_dev *pdev = adapter->pdev;
2397
2398 err = pci_enable_device(pdev);
2399 if (err) {
2400 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2401 return err;
2402 }
2403
2404 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2405 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
2406 dev_err(&pdev->dev,
2407 "pci_set_consistent_dma_mask failed\n");
2408 err = -EIO;
2409 goto err_set_mask;
2410 }
2411 *dma64 = true;
2412 } else {
2413 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
2414 dev_err(&pdev->dev,
2415 "pci_set_dma_mask failed\n");
2416 err = -EIO;
2417 goto err_set_mask;
2418 }
2419 *dma64 = false;
2420 }
2421
2422 err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2423 vmxnet3_driver_name);
2424 if (err) {
2425 dev_err(&pdev->dev,
2426 "Failed to request region for adapter: error %d\n", err);
2427 goto err_set_mask;
2428 }
2429
2430 pci_set_master(pdev);
2431
2432 mmio_start = pci_resource_start(pdev, 0);
2433 mmio_len = pci_resource_len(pdev, 0);
2434 adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2435 if (!adapter->hw_addr0) {
2436 dev_err(&pdev->dev, "Failed to map bar0\n");
2437 err = -EIO;
2438 goto err_ioremap;
2439 }
2440
2441 mmio_start = pci_resource_start(pdev, 1);
2442 mmio_len = pci_resource_len(pdev, 1);
2443 adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2444 if (!adapter->hw_addr1) {
2445 dev_err(&pdev->dev, "Failed to map bar1\n");
2446 err = -EIO;
2447 goto err_bar1;
2448 }
2449 return 0;
2450
2451 err_bar1:
2452 iounmap(adapter->hw_addr0);
2453 err_ioremap:
2454 pci_release_selected_regions(pdev, (1 << 2) - 1);
2455 err_set_mask:
2456 pci_disable_device(pdev);
2457 return err;
2458 }
2459
2460
2461 static void
2462 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2463 {
2464 BUG_ON(!adapter->pdev);
2465
2466 iounmap(adapter->hw_addr0);
2467 iounmap(adapter->hw_addr1);
2468 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2469 pci_disable_device(adapter->pdev);
2470 }
2471
2472
2473 static void
2474 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2475 {
2476 size_t sz, i, ring0_size, ring1_size, comp_size;
2477 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2478
2479
2480 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2481 VMXNET3_MAX_ETH_HDR_SIZE) {
2482 adapter->skb_buf_size = adapter->netdev->mtu +
2483 VMXNET3_MAX_ETH_HDR_SIZE;
2484 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2485 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2486
2487 adapter->rx_buf_per_pkt = 1;
2488 } else {
2489 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2490 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2491 VMXNET3_MAX_ETH_HDR_SIZE;
2492 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2493 }
2494
2495 /*
2496 * for simplicity, force the ring0 size to be a multiple of
2497 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2498 */
2499 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2500 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2501 ring0_size = (ring0_size + sz - 1) / sz * sz;
2502 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
2503 sz * sz);
2504 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2505 comp_size = ring0_size + ring1_size;
2506
2507 for (i = 0; i < adapter->num_rx_queues; i++) {
2508 rq = &adapter->rx_queue[i];
2509 rq->rx_ring[0].size = ring0_size;
2510 rq->rx_ring[1].size = ring1_size;
2511 rq->comp_ring.size = comp_size;
2512 }
2513 }
2514
2515
2516 int
2517 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2518 u32 rx_ring_size, u32 rx_ring2_size)
2519 {
2520 int err = 0, i;
2521
2522 for (i = 0; i < adapter->num_tx_queues; i++) {
2523 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2524 tq->tx_ring.size = tx_ring_size;
2525 tq->data_ring.size = tx_ring_size;
2526 tq->comp_ring.size = tx_ring_size;
2527 tq->shared = &adapter->tqd_start[i].ctrl;
2528 tq->stopped = true;
2529 tq->adapter = adapter;
2530 tq->qid = i;
2531 err = vmxnet3_tq_create(tq, adapter);
2532 /*
2533 * Too late to change num_tx_queues. We cannot do away with
2534 * lesser number of queues than what we asked for
2535 */
2536 if (err)
2537 goto queue_err;
2538 }
2539
2540 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2541 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2542 vmxnet3_adjust_rx_ring_size(adapter);
2543 for (i = 0; i < adapter->num_rx_queues; i++) {
2544 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2545 /* qid and qid2 for rx queues will be assigned later when num
2546 * of rx queues is finalized after allocating intrs */
2547 rq->shared = &adapter->rqd_start[i].ctrl;
2548 rq->adapter = adapter;
2549 err = vmxnet3_rq_create(rq, adapter);
2550 if (err) {
2551 if (i == 0) {
2552 netdev_err(adapter->netdev,
2553 "Could not allocate any rx queues. "
2554 "Aborting.\n");
2555 goto queue_err;
2556 } else {
2557 netdev_info(adapter->netdev,
2558 "Number of rx queues changed "
2559 "to : %d.\n", i);
2560 adapter->num_rx_queues = i;
2561 err = 0;
2562 break;
2563 }
2564 }
2565 }
2566 return err;
2567 queue_err:
2568 vmxnet3_tq_destroy_all(adapter);
2569 return err;
2570 }
2571
2572 static int
2573 vmxnet3_open(struct net_device *netdev)
2574 {
2575 struct vmxnet3_adapter *adapter;
2576 int err, i;
2577
2578 adapter = netdev_priv(netdev);
2579
2580 for (i = 0; i < adapter->num_tx_queues; i++)
2581 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2582
2583 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2584 VMXNET3_DEF_RX_RING_SIZE,
2585 VMXNET3_DEF_RX_RING_SIZE);
2586 if (err)
2587 goto queue_err;
2588
2589 err = vmxnet3_activate_dev(adapter);
2590 if (err)
2591 goto activate_err;
2592
2593 return 0;
2594
2595 activate_err:
2596 vmxnet3_rq_destroy_all(adapter);
2597 vmxnet3_tq_destroy_all(adapter);
2598 queue_err:
2599 return err;
2600 }
2601
2602
2603 static int
2604 vmxnet3_close(struct net_device *netdev)
2605 {
2606 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2607
2608 /*
2609 * Reset_work may be in the middle of resetting the device, wait for its
2610 * completion.
2611 */
2612 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2613 msleep(1);
2614
2615 vmxnet3_quiesce_dev(adapter);
2616
2617 vmxnet3_rq_destroy_all(adapter);
2618 vmxnet3_tq_destroy_all(adapter);
2619
2620 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2621
2622
2623 return 0;
2624 }
2625
2626
2627 void
2628 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2629 {
2630 int i;
2631
2632 /*
2633 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2634 * vmxnet3_close() will deadlock.
2635 */
2636 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2637
2638 /* we need to enable NAPI, otherwise dev_close will deadlock */
2639 for (i = 0; i < adapter->num_rx_queues; i++)
2640 napi_enable(&adapter->rx_queue[i].napi);
2641 dev_close(adapter->netdev);
2642 }
2643
2644
2645 static int
2646 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2647 {
2648 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2649 int err = 0;
2650
2651 if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2652 return -EINVAL;
2653
2654 netdev->mtu = new_mtu;
2655
2656 /*
2657 * Reset_work may be in the middle of resetting the device, wait for its
2658 * completion.
2659 */
2660 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2661 msleep(1);
2662
2663 if (netif_running(netdev)) {
2664 vmxnet3_quiesce_dev(adapter);
2665 vmxnet3_reset_dev(adapter);
2666
2667 /* we need to re-create the rx queue based on the new mtu */
2668 vmxnet3_rq_destroy_all(adapter);
2669 vmxnet3_adjust_rx_ring_size(adapter);
2670 err = vmxnet3_rq_create_all(adapter);
2671 if (err) {
2672 netdev_err(netdev,
2673 "failed to re-create rx queues, "
2674 " error %d. Closing it.\n", err);
2675 goto out;
2676 }
2677
2678 err = vmxnet3_activate_dev(adapter);
2679 if (err) {
2680 netdev_err(netdev,
2681 "failed to re-activate, error %d. "
2682 "Closing it\n", err);
2683 goto out;
2684 }
2685 }
2686
2687 out:
2688 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2689 if (err)
2690 vmxnet3_force_close(adapter);
2691
2692 return err;
2693 }
2694
2695
2696 static void
2697 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2698 {
2699 struct net_device *netdev = adapter->netdev;
2700
2701 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2702 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2703 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2704 NETIF_F_LRO;
2705 if (dma64)
2706 netdev->hw_features |= NETIF_F_HIGHDMA;
2707 netdev->vlan_features = netdev->hw_features &
2708 ~(NETIF_F_HW_VLAN_CTAG_TX |
2709 NETIF_F_HW_VLAN_CTAG_RX);
2710 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
2711 }
2712
2713
2714 static void
2715 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2716 {
2717 u32 tmp;
2718
2719 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2720 *(u32 *)mac = tmp;
2721
2722 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2723 mac[4] = tmp & 0xff;
2724 mac[5] = (tmp >> 8) & 0xff;
2725 }
2726
2727 #ifdef CONFIG_PCI_MSI
2728
2729 /*
2730 * Enable MSIx vectors.
2731 * Returns :
2732 * 0 on successful enabling of required vectors,
2733 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
2734 * could be enabled.
2735 * number of vectors which can be enabled otherwise (this number is smaller
2736 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2737 */
2738
2739 static int
2740 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
2741 int vectors)
2742 {
2743 int err = 0, vector_threshold;
2744 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
2745
2746 while (vectors >= vector_threshold) {
2747 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2748 vectors);
2749 if (!err) {
2750 adapter->intr.num_intrs = vectors;
2751 return 0;
2752 } else if (err < 0) {
2753 dev_err(&adapter->netdev->dev,
2754 "Failed to enable MSI-X, error: %d\n", err);
2755 vectors = 0;
2756 } else if (err < vector_threshold) {
2757 break;
2758 } else {
2759 /* If fails to enable required number of MSI-x vectors
2760 * try enabling minimum number of vectors required.
2761 */
2762 dev_err(&adapter->netdev->dev,
2763 "Failed to enable %d MSI-X, trying %d instead\n",
2764 vectors, vector_threshold);
2765 vectors = vector_threshold;
2766 }
2767 }
2768
2769 dev_info(&adapter->pdev->dev,
2770 "Number of MSI-X interrupts which can be allocated "
2771 "is lower than min threshold required.\n");
2772 return err;
2773 }
2774
2775
2776 #endif /* CONFIG_PCI_MSI */
2777
2778 static void
2779 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2780 {
2781 u32 cfg;
2782 unsigned long flags;
2783
2784 /* intr settings */
2785 spin_lock_irqsave(&adapter->cmd_lock, flags);
2786 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2787 VMXNET3_CMD_GET_CONF_INTR);
2788 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2789 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2790 adapter->intr.type = cfg & 0x3;
2791 adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2792
2793 if (adapter->intr.type == VMXNET3_IT_AUTO) {
2794 adapter->intr.type = VMXNET3_IT_MSIX;
2795 }
2796
2797 #ifdef CONFIG_PCI_MSI
2798 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2799 int vector, err = 0;
2800
2801 adapter->intr.num_intrs = (adapter->share_intr ==
2802 VMXNET3_INTR_TXSHARE) ? 1 :
2803 adapter->num_tx_queues;
2804 adapter->intr.num_intrs += (adapter->share_intr ==
2805 VMXNET3_INTR_BUDDYSHARE) ? 0 :
2806 adapter->num_rx_queues;
2807 adapter->intr.num_intrs += 1; /* for link event */
2808
2809 adapter->intr.num_intrs = (adapter->intr.num_intrs >
2810 VMXNET3_LINUX_MIN_MSIX_VECT
2811 ? adapter->intr.num_intrs :
2812 VMXNET3_LINUX_MIN_MSIX_VECT);
2813
2814 for (vector = 0; vector < adapter->intr.num_intrs; vector++)
2815 adapter->intr.msix_entries[vector].entry = vector;
2816
2817 err = vmxnet3_acquire_msix_vectors(adapter,
2818 adapter->intr.num_intrs);
2819 /* If we cannot allocate one MSIx vector per queue
2820 * then limit the number of rx queues to 1
2821 */
2822 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
2823 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2824 || adapter->num_rx_queues != 1) {
2825 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2826 netdev_err(adapter->netdev,
2827 "Number of rx queues : 1\n");
2828 adapter->num_rx_queues = 1;
2829 adapter->intr.num_intrs =
2830 VMXNET3_LINUX_MIN_MSIX_VECT;
2831 }
2832 return;
2833 }
2834 if (!err)
2835 return;
2836
2837 /* If we cannot allocate MSIx vectors use only one rx queue */
2838 dev_info(&adapter->pdev->dev,
2839 "Failed to enable MSI-X, error %d. "
2840 "Limiting #rx queues to 1, try MSI.\n", err);
2841
2842 adapter->intr.type = VMXNET3_IT_MSI;
2843 }
2844
2845 if (adapter->intr.type == VMXNET3_IT_MSI) {
2846 int err;
2847 err = pci_enable_msi(adapter->pdev);
2848 if (!err) {
2849 adapter->num_rx_queues = 1;
2850 adapter->intr.num_intrs = 1;
2851 return;
2852 }
2853 }
2854 #endif /* CONFIG_PCI_MSI */
2855
2856 adapter->num_rx_queues = 1;
2857 dev_info(&adapter->netdev->dev,
2858 "Using INTx interrupt, #Rx queues: 1.\n");
2859 adapter->intr.type = VMXNET3_IT_INTX;
2860
2861 /* INT-X related setting */
2862 adapter->intr.num_intrs = 1;
2863 }
2864
2865
2866 static void
2867 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2868 {
2869 if (adapter->intr.type == VMXNET3_IT_MSIX)
2870 pci_disable_msix(adapter->pdev);
2871 else if (adapter->intr.type == VMXNET3_IT_MSI)
2872 pci_disable_msi(adapter->pdev);
2873 else
2874 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2875 }
2876
2877
2878 static void
2879 vmxnet3_tx_timeout(struct net_device *netdev)
2880 {
2881 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2882 adapter->tx_timeout_count++;
2883
2884 netdev_err(adapter->netdev, "tx hang\n");
2885 schedule_work(&adapter->work);
2886 netif_wake_queue(adapter->netdev);
2887 }
2888
2889
2890 static void
2891 vmxnet3_reset_work(struct work_struct *data)
2892 {
2893 struct vmxnet3_adapter *adapter;
2894
2895 adapter = container_of(data, struct vmxnet3_adapter, work);
2896
2897 /* if another thread is resetting the device, no need to proceed */
2898 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2899 return;
2900
2901 /* if the device is closed, we must leave it alone */
2902 rtnl_lock();
2903 if (netif_running(adapter->netdev)) {
2904 netdev_notice(adapter->netdev, "resetting\n");
2905 vmxnet3_quiesce_dev(adapter);
2906 vmxnet3_reset_dev(adapter);
2907 vmxnet3_activate_dev(adapter);
2908 } else {
2909 netdev_info(adapter->netdev, "already closed\n");
2910 }
2911 rtnl_unlock();
2912
2913 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2914 }
2915
2916
2917 static int
2918 vmxnet3_probe_device(struct pci_dev *pdev,
2919 const struct pci_device_id *id)
2920 {
2921 static const struct net_device_ops vmxnet3_netdev_ops = {
2922 .ndo_open = vmxnet3_open,
2923 .ndo_stop = vmxnet3_close,
2924 .ndo_start_xmit = vmxnet3_xmit_frame,
2925 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2926 .ndo_change_mtu = vmxnet3_change_mtu,
2927 .ndo_set_features = vmxnet3_set_features,
2928 .ndo_get_stats64 = vmxnet3_get_stats64,
2929 .ndo_tx_timeout = vmxnet3_tx_timeout,
2930 .ndo_set_rx_mode = vmxnet3_set_mc,
2931 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2932 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2933 #ifdef CONFIG_NET_POLL_CONTROLLER
2934 .ndo_poll_controller = vmxnet3_netpoll,
2935 #endif
2936 };
2937 int err;
2938 bool dma64 = false; /* stupid gcc */
2939 u32 ver;
2940 struct net_device *netdev;
2941 struct vmxnet3_adapter *adapter;
2942 u8 mac[ETH_ALEN];
2943 int size;
2944 int num_tx_queues;
2945 int num_rx_queues;
2946
2947 if (!pci_msi_enabled())
2948 enable_mq = 0;
2949
2950 #ifdef VMXNET3_RSS
2951 if (enable_mq)
2952 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2953 (int)num_online_cpus());
2954 else
2955 #endif
2956 num_rx_queues = 1;
2957 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
2958
2959 if (enable_mq)
2960 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2961 (int)num_online_cpus());
2962 else
2963 num_tx_queues = 1;
2964
2965 num_tx_queues = rounddown_pow_of_two(num_tx_queues);
2966 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2967 max(num_tx_queues, num_rx_queues));
2968 dev_info(&pdev->dev,
2969 "# of Tx queues : %d, # of Rx queues : %d\n",
2970 num_tx_queues, num_rx_queues);
2971
2972 if (!netdev)
2973 return -ENOMEM;
2974
2975 pci_set_drvdata(pdev, netdev);
2976 adapter = netdev_priv(netdev);
2977 adapter->netdev = netdev;
2978 adapter->pdev = pdev;
2979
2980 spin_lock_init(&adapter->cmd_lock);
2981 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
2982 sizeof(struct vmxnet3_adapter),
2983 PCI_DMA_TODEVICE);
2984 adapter->shared = dma_alloc_coherent(
2985 &adapter->pdev->dev,
2986 sizeof(struct Vmxnet3_DriverShared),
2987 &adapter->shared_pa, GFP_KERNEL);
2988 if (!adapter->shared) {
2989 dev_err(&pdev->dev, "Failed to allocate memory\n");
2990 err = -ENOMEM;
2991 goto err_alloc_shared;
2992 }
2993
2994 adapter->num_rx_queues = num_rx_queues;
2995 adapter->num_tx_queues = num_tx_queues;
2996 adapter->rx_buf_per_pkt = 1;
2997
2998 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2999 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
3000 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
3001 &adapter->queue_desc_pa,
3002 GFP_KERNEL);
3003
3004 if (!adapter->tqd_start) {
3005 dev_err(&pdev->dev, "Failed to allocate memory\n");
3006 err = -ENOMEM;
3007 goto err_alloc_queue_desc;
3008 }
3009 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3010 adapter->num_tx_queues);
3011
3012 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3013 sizeof(struct Vmxnet3_PMConf),
3014 &adapter->pm_conf_pa,
3015 GFP_KERNEL);
3016 if (adapter->pm_conf == NULL) {
3017 err = -ENOMEM;
3018 goto err_alloc_pm;
3019 }
3020
3021 #ifdef VMXNET3_RSS
3022
3023 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3024 sizeof(struct UPT1_RSSConf),
3025 &adapter->rss_conf_pa,
3026 GFP_KERNEL);
3027 if (adapter->rss_conf == NULL) {
3028 err = -ENOMEM;
3029 goto err_alloc_rss;
3030 }
3031 #endif /* VMXNET3_RSS */
3032
3033 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
3034 if (err < 0)
3035 goto err_alloc_pci;
3036
3037 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3038 if (ver & 1) {
3039 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
3040 } else {
3041 dev_err(&pdev->dev,
3042 "Incompatible h/w version (0x%x) for adapter\n", ver);
3043 err = -EBUSY;
3044 goto err_ver;
3045 }
3046
3047 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3048 if (ver & 1) {
3049 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3050 } else {
3051 dev_err(&pdev->dev,
3052 "Incompatible upt version (0x%x) for adapter\n", ver);
3053 err = -EBUSY;
3054 goto err_ver;
3055 }
3056
3057 SET_NETDEV_DEV(netdev, &pdev->dev);
3058 vmxnet3_declare_features(adapter, dma64);
3059
3060 if (adapter->num_tx_queues == adapter->num_rx_queues)
3061 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3062 else
3063 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3064
3065 vmxnet3_alloc_intr_resources(adapter);
3066
3067 #ifdef VMXNET3_RSS
3068 if (adapter->num_rx_queues > 1 &&
3069 adapter->intr.type == VMXNET3_IT_MSIX) {
3070 adapter->rss = true;
3071 netdev->hw_features |= NETIF_F_RXHASH;
3072 netdev->features |= NETIF_F_RXHASH;
3073 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3074 } else {
3075 adapter->rss = false;
3076 }
3077 #endif
3078
3079 vmxnet3_read_mac_addr(adapter, mac);
3080 memcpy(netdev->dev_addr, mac, netdev->addr_len);
3081
3082 netdev->netdev_ops = &vmxnet3_netdev_ops;
3083 vmxnet3_set_ethtool_ops(netdev);
3084 netdev->watchdog_timeo = 5 * HZ;
3085
3086 INIT_WORK(&adapter->work, vmxnet3_reset_work);
3087 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3088
3089 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3090 int i;
3091 for (i = 0; i < adapter->num_rx_queues; i++) {
3092 netif_napi_add(adapter->netdev,
3093 &adapter->rx_queue[i].napi,
3094 vmxnet3_poll_rx_only, 64);
3095 }
3096 } else {
3097 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3098 vmxnet3_poll, 64);
3099 }
3100
3101 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3102 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3103
3104 netif_carrier_off(netdev);
3105 err = register_netdev(netdev);
3106
3107 if (err) {
3108 dev_err(&pdev->dev, "Failed to register adapter\n");
3109 goto err_register;
3110 }
3111
3112 vmxnet3_check_link(adapter, false);
3113 return 0;
3114
3115 err_register:
3116 vmxnet3_free_intr_resources(adapter);
3117 err_ver:
3118 vmxnet3_free_pci_resources(adapter);
3119 err_alloc_pci:
3120 #ifdef VMXNET3_RSS
3121 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3122 adapter->rss_conf, adapter->rss_conf_pa);
3123 err_alloc_rss:
3124 #endif
3125 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3126 adapter->pm_conf, adapter->pm_conf_pa);
3127 err_alloc_pm:
3128 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3129 adapter->queue_desc_pa);
3130 err_alloc_queue_desc:
3131 dma_free_coherent(&adapter->pdev->dev,
3132 sizeof(struct Vmxnet3_DriverShared),
3133 adapter->shared, adapter->shared_pa);
3134 err_alloc_shared:
3135 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3136 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3137 free_netdev(netdev);
3138 return err;
3139 }
3140
3141
3142 static void
3143 vmxnet3_remove_device(struct pci_dev *pdev)
3144 {
3145 struct net_device *netdev = pci_get_drvdata(pdev);
3146 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3147 int size = 0;
3148 int num_rx_queues;
3149
3150 #ifdef VMXNET3_RSS
3151 if (enable_mq)
3152 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3153 (int)num_online_cpus());
3154 else
3155 #endif
3156 num_rx_queues = 1;
3157 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3158
3159 cancel_work_sync(&adapter->work);
3160
3161 unregister_netdev(netdev);
3162
3163 vmxnet3_free_intr_resources(adapter);
3164 vmxnet3_free_pci_resources(adapter);
3165 #ifdef VMXNET3_RSS
3166 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3167 adapter->rss_conf, adapter->rss_conf_pa);
3168 #endif
3169 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3170 adapter->pm_conf, adapter->pm_conf_pa);
3171
3172 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3173 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3174 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3175 adapter->queue_desc_pa);
3176 dma_free_coherent(&adapter->pdev->dev,
3177 sizeof(struct Vmxnet3_DriverShared),
3178 adapter->shared, adapter->shared_pa);
3179 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3180 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3181 free_netdev(netdev);
3182 }
3183
3184
3185 #ifdef CONFIG_PM
3186
3187 static int
3188 vmxnet3_suspend(struct device *device)
3189 {
3190 struct pci_dev *pdev = to_pci_dev(device);
3191 struct net_device *netdev = pci_get_drvdata(pdev);
3192 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3193 struct Vmxnet3_PMConf *pmConf;
3194 struct ethhdr *ehdr;
3195 struct arphdr *ahdr;
3196 u8 *arpreq;
3197 struct in_device *in_dev;
3198 struct in_ifaddr *ifa;
3199 unsigned long flags;
3200 int i = 0;
3201
3202 if (!netif_running(netdev))
3203 return 0;
3204
3205 for (i = 0; i < adapter->num_rx_queues; i++)
3206 napi_disable(&adapter->rx_queue[i].napi);
3207
3208 vmxnet3_disable_all_intrs(adapter);
3209 vmxnet3_free_irqs(adapter);
3210 vmxnet3_free_intr_resources(adapter);
3211
3212 netif_device_detach(netdev);
3213 netif_tx_stop_all_queues(netdev);
3214
3215 /* Create wake-up filters. */
3216 pmConf = adapter->pm_conf;
3217 memset(pmConf, 0, sizeof(*pmConf));
3218
3219 if (adapter->wol & WAKE_UCAST) {
3220 pmConf->filters[i].patternSize = ETH_ALEN;
3221 pmConf->filters[i].maskSize = 1;
3222 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3223 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3224
3225 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3226 i++;
3227 }
3228
3229 if (adapter->wol & WAKE_ARP) {
3230 in_dev = in_dev_get(netdev);
3231 if (!in_dev)
3232 goto skip_arp;
3233
3234 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3235 if (!ifa)
3236 goto skip_arp;
3237
3238 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3239 sizeof(struct arphdr) + /* ARP header */
3240 2 * ETH_ALEN + /* 2 Ethernet addresses*/
3241 2 * sizeof(u32); /*2 IPv4 addresses */
3242 pmConf->filters[i].maskSize =
3243 (pmConf->filters[i].patternSize - 1) / 8 + 1;
3244
3245 /* ETH_P_ARP in Ethernet header. */
3246 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3247 ehdr->h_proto = htons(ETH_P_ARP);
3248
3249 /* ARPOP_REQUEST in ARP header. */
3250 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3251 ahdr->ar_op = htons(ARPOP_REQUEST);
3252 arpreq = (u8 *)(ahdr + 1);
3253
3254 /* The Unicast IPv4 address in 'tip' field. */
3255 arpreq += 2 * ETH_ALEN + sizeof(u32);
3256 *(u32 *)arpreq = ifa->ifa_address;
3257
3258 /* The mask for the relevant bits. */
3259 pmConf->filters[i].mask[0] = 0x00;
3260 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3261 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3262 pmConf->filters[i].mask[3] = 0x00;
3263 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3264 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3265 in_dev_put(in_dev);
3266
3267 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3268 i++;
3269 }
3270
3271 skip_arp:
3272 if (adapter->wol & WAKE_MAGIC)
3273 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
3274
3275 pmConf->numFilters = i;
3276
3277 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3278 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3279 *pmConf));
3280 adapter->shared->devRead.pmConfDesc.confPA =
3281 cpu_to_le64(adapter->pm_conf_pa);
3282
3283 spin_lock_irqsave(&adapter->cmd_lock, flags);
3284 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3285 VMXNET3_CMD_UPDATE_PMCFG);
3286 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3287
3288 pci_save_state(pdev);
3289 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3290 adapter->wol);
3291 pci_disable_device(pdev);
3292 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3293
3294 return 0;
3295 }
3296
3297
3298 static int
3299 vmxnet3_resume(struct device *device)
3300 {
3301 int err, i = 0;
3302 unsigned long flags;
3303 struct pci_dev *pdev = to_pci_dev(device);
3304 struct net_device *netdev = pci_get_drvdata(pdev);
3305 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3306 struct Vmxnet3_PMConf *pmConf;
3307
3308 if (!netif_running(netdev))
3309 return 0;
3310
3311 /* Destroy wake-up filters. */
3312 pmConf = adapter->pm_conf;
3313 memset(pmConf, 0, sizeof(*pmConf));
3314
3315 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3316 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3317 *pmConf));
3318 adapter->shared->devRead.pmConfDesc.confPA =
3319 cpu_to_le64(adapter->pm_conf_pa);
3320
3321 netif_device_attach(netdev);
3322 pci_set_power_state(pdev, PCI_D0);
3323 pci_restore_state(pdev);
3324 err = pci_enable_device_mem(pdev);
3325 if (err != 0)
3326 return err;
3327
3328 pci_enable_wake(pdev, PCI_D0, 0);
3329
3330 spin_lock_irqsave(&adapter->cmd_lock, flags);
3331 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3332 VMXNET3_CMD_UPDATE_PMCFG);
3333 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3334 vmxnet3_alloc_intr_resources(adapter);
3335 vmxnet3_request_irqs(adapter);
3336 for (i = 0; i < adapter->num_rx_queues; i++)
3337 napi_enable(&adapter->rx_queue[i].napi);
3338 vmxnet3_enable_all_intrs(adapter);
3339
3340 return 0;
3341 }
3342
3343 static const struct dev_pm_ops vmxnet3_pm_ops = {
3344 .suspend = vmxnet3_suspend,
3345 .resume = vmxnet3_resume,
3346 };
3347 #endif
3348
3349 static struct pci_driver vmxnet3_driver = {
3350 .name = vmxnet3_driver_name,
3351 .id_table = vmxnet3_pciid_table,
3352 .probe = vmxnet3_probe_device,
3353 .remove = vmxnet3_remove_device,
3354 #ifdef CONFIG_PM
3355 .driver.pm = &vmxnet3_pm_ops,
3356 #endif
3357 };
3358
3359
3360 static int __init
3361 vmxnet3_init_module(void)
3362 {
3363 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
3364 VMXNET3_DRIVER_VERSION_REPORT);
3365 return pci_register_driver(&vmxnet3_driver);
3366 }
3367
3368 module_init(vmxnet3_init_module);
3369
3370
3371 static void
3372 vmxnet3_exit_module(void)
3373 {
3374 pci_unregister_driver(&vmxnet3_driver);
3375 }
3376
3377 module_exit(vmxnet3_exit_module);
3378
3379 MODULE_AUTHOR("VMware, Inc.");
3380 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3381 MODULE_LICENSE("GPL v2");
3382 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
Linux with added FPC-III support