?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mfd: wm8350-i2c: Make sure the i2c regmap functions are compiled
[linux/fpc-iii.git] / drivers / net / vmxnet3 / vmxnet3_drv.c
blob55d89390b4bc2b5f65ddd2460998582d03d0d320
1 /*
2 * Linux driver for VMware's vmxnet3 ethernet NIC.
3 *
4 * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License and no later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more
14 * details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * The full GNU General Public License is included in this distribution in
21 * the file called "COPYING".
22 *
23 * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
24 *
25 */
26
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
29
30 #include "vmxnet3_int.h"
31
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
34
35 /*
36 * PCI Device ID Table
37 * Last entry must be all 0s
38 */
39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
41 {0}
42 };
43
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
45
46 static int enable_mq = 1;
47
48 static void
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
50
51 /*
52 * Enable/Disable the given intr
53 */
54 static void
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
56 {
57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
58 }
59
60
61 static void
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
63 {
64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
65 }
66
67
68 /*
69 * Enable/Disable all intrs used by the device
70 */
71 static void
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
73 {
74 int i;
75
76 for (i = 0; i < adapter->intr.num_intrs; i++)
77 vmxnet3_enable_intr(adapter, i);
78 adapter->shared->devRead.intrConf.intrCtrl &=
79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
80 }
81
82
83 static void
84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
85 {
86 int i;
87
88 adapter->shared->devRead.intrConf.intrCtrl |=
89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
90 for (i = 0; i < adapter->intr.num_intrs; i++)
91 vmxnet3_disable_intr(adapter, i);
92 }
93
94
95 static void
96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
97 {
98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
99 }
100
101
102 static bool
103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
104 {
105 return tq->stopped;
106 }
107
108
109 static void
110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111 {
112 tq->stopped = false;
113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
114 }
115
116
117 static void
118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119 {
120 tq->stopped = false;
121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
122 }
123
124
125 static void
126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
127 {
128 tq->stopped = true;
129 tq->num_stop++;
130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
131 }
132
133
134 /*
135 * Check the link state. This may start or stop the tx queue.
136 */
137 static void
138 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
139 {
140 u32 ret;
141 int i;
142 unsigned long flags;
143
144 spin_lock_irqsave(&adapter->cmd_lock, flags);
145 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
146 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
147 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
148
149 adapter->link_speed = ret >> 16;
150 if (ret & 1) { /* Link is up. */
151 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
152 adapter->link_speed);
153 netif_carrier_on(adapter->netdev);
154
155 if (affectTxQueue) {
156 for (i = 0; i < adapter->num_tx_queues; i++)
157 vmxnet3_tq_start(&adapter->tx_queue[i],
158 adapter);
159 }
160 } else {
161 netdev_info(adapter->netdev, "NIC Link is Down\n");
162 netif_carrier_off(adapter->netdev);
163
164 if (affectTxQueue) {
165 for (i = 0; i < adapter->num_tx_queues; i++)
166 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
167 }
168 }
169 }
170
171 static void
172 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
173 {
174 int i;
175 unsigned long flags;
176 u32 events = le32_to_cpu(adapter->shared->ecr);
177 if (!events)
178 return;
179
180 vmxnet3_ack_events(adapter, events);
181
182 /* Check if link state has changed */
183 if (events & VMXNET3_ECR_LINK)
184 vmxnet3_check_link(adapter, true);
185
186 /* Check if there is an error on xmit/recv queues */
187 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
188 spin_lock_irqsave(&adapter->cmd_lock, flags);
189 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
190 VMXNET3_CMD_GET_QUEUE_STATUS);
191 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
192
193 for (i = 0; i < adapter->num_tx_queues; i++)
194 if (adapter->tqd_start[i].status.stopped)
195 dev_err(&adapter->netdev->dev,
196 "%s: tq[%d] error 0x%x\n",
197 adapter->netdev->name, i, le32_to_cpu(
198 adapter->tqd_start[i].status.error));
199 for (i = 0; i < adapter->num_rx_queues; i++)
200 if (adapter->rqd_start[i].status.stopped)
201 dev_err(&adapter->netdev->dev,
202 "%s: rq[%d] error 0x%x\n",
203 adapter->netdev->name, i,
204 adapter->rqd_start[i].status.error);
205
206 schedule_work(&adapter->work);
207 }
208 }
209
210 #ifdef __BIG_ENDIAN_BITFIELD
211 /*
212 * The device expects the bitfields in shared structures to be written in
213 * little endian. When CPU is big endian, the following routines are used to
214 * correctly read and write into ABI.
215 * The general technique used here is : double word bitfields are defined in
216 * opposite order for big endian architecture. Then before reading them in
217 * driver the complete double word is translated using le32_to_cpu. Similarly
218 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
219 * double words into required format.
220 * In order to avoid touching bits in shared structure more than once, temporary
221 * descriptors are used. These are passed as srcDesc to following functions.
222 */
223 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
224 struct Vmxnet3_RxDesc *dstDesc)
225 {
226 u32 *src = (u32 *)srcDesc + 2;
227 u32 *dst = (u32 *)dstDesc + 2;
228 dstDesc->addr = le64_to_cpu(srcDesc->addr);
229 *dst = le32_to_cpu(*src);
230 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
231 }
232
233 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
234 struct Vmxnet3_TxDesc *dstDesc)
235 {
236 int i;
237 u32 *src = (u32 *)(srcDesc + 1);
238 u32 *dst = (u32 *)(dstDesc + 1);
239
240 /* Working backwards so that the gen bit is set at the end. */
241 for (i = 2; i > 0; i--) {
242 src--;
243 dst--;
244 *dst = cpu_to_le32(*src);
245 }
246 }
247
248
249 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
250 struct Vmxnet3_RxCompDesc *dstDesc)
251 {
252 int i = 0;
253 u32 *src = (u32 *)srcDesc;
254 u32 *dst = (u32 *)dstDesc;
255 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
256 *dst = le32_to_cpu(*src);
257 src++;
258 dst++;
259 }
260 }
261
262
263 /* Used to read bitfield values from double words. */
264 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
265 {
266 u32 temp = le32_to_cpu(*bitfield);
267 u32 mask = ((1 << size) - 1) << pos;
268 temp &= mask;
269 temp >>= pos;
270 return temp;
271 }
272
273
274
275 #endif /* __BIG_ENDIAN_BITFIELD */
276
277 #ifdef __BIG_ENDIAN_BITFIELD
278
279 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
280 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
281 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
282 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
283 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
284 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
285 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
286 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
287 VMXNET3_TCD_GEN_SIZE)
288 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
289 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
290 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
291 (dstrcd) = (tmp); \
292 vmxnet3_RxCompToCPU((rcd), (tmp)); \
293 } while (0)
294 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
295 (dstrxd) = (tmp); \
296 vmxnet3_RxDescToCPU((rxd), (tmp)); \
297 } while (0)
298
299 #else
300
301 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
302 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
303 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
304 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
305 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
306 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
307
308 #endif /* __BIG_ENDIAN_BITFIELD */
309
310
311 static void
312 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
313 struct pci_dev *pdev)
314 {
315 if (tbi->map_type == VMXNET3_MAP_SINGLE)
316 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
317 PCI_DMA_TODEVICE);
318 else if (tbi->map_type == VMXNET3_MAP_PAGE)
319 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
320 PCI_DMA_TODEVICE);
321 else
322 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
323
324 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
325 }
326
327
328 static int
329 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
330 struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
331 {
332 struct sk_buff *skb;
333 int entries = 0;
334
335 /* no out of order completion */
336 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
337 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
338
339 skb = tq->buf_info[eop_idx].skb;
340 BUG_ON(skb == NULL);
341 tq->buf_info[eop_idx].skb = NULL;
342
343 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
344
345 while (tq->tx_ring.next2comp != eop_idx) {
346 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
347 pdev);
348
349 /* update next2comp w/o tx_lock. Since we are marking more,
350 * instead of less, tx ring entries avail, the worst case is
351 * that the tx routine incorrectly re-queues a pkt due to
352 * insufficient tx ring entries.
353 */
354 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
355 entries++;
356 }
357
358 dev_kfree_skb_any(skb);
359 return entries;
360 }
361
362
363 static int
364 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
365 struct vmxnet3_adapter *adapter)
366 {
367 int completed = 0;
368 union Vmxnet3_GenericDesc *gdesc;
369
370 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
371 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
372 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
373 &gdesc->tcd), tq, adapter->pdev,
374 adapter);
375
376 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
377 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
378 }
379
380 if (completed) {
381 spin_lock(&tq->tx_lock);
382 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
383 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
384 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
385 netif_carrier_ok(adapter->netdev))) {
386 vmxnet3_tq_wake(tq, adapter);
387 }
388 spin_unlock(&tq->tx_lock);
389 }
390 return completed;
391 }
392
393
394 static void
395 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
396 struct vmxnet3_adapter *adapter)
397 {
398 int i;
399
400 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
401 struct vmxnet3_tx_buf_info *tbi;
402
403 tbi = tq->buf_info + tq->tx_ring.next2comp;
404
405 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
406 if (tbi->skb) {
407 dev_kfree_skb_any(tbi->skb);
408 tbi->skb = NULL;
409 }
410 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
411 }
412
413 /* sanity check, verify all buffers are indeed unmapped and freed */
414 for (i = 0; i < tq->tx_ring.size; i++) {
415 BUG_ON(tq->buf_info[i].skb != NULL ||
416 tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
417 }
418
419 tq->tx_ring.gen = VMXNET3_INIT_GEN;
420 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
421
422 tq->comp_ring.gen = VMXNET3_INIT_GEN;
423 tq->comp_ring.next2proc = 0;
424 }
425
426
427 static void
428 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
429 struct vmxnet3_adapter *adapter)
430 {
431 if (tq->tx_ring.base) {
432 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
433 sizeof(struct Vmxnet3_TxDesc),
434 tq->tx_ring.base, tq->tx_ring.basePA);
435 tq->tx_ring.base = NULL;
436 }
437 if (tq->data_ring.base) {
438 dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
439 sizeof(struct Vmxnet3_TxDataDesc),
440 tq->data_ring.base, tq->data_ring.basePA);
441 tq->data_ring.base = NULL;
442 }
443 if (tq->comp_ring.base) {
444 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
445 sizeof(struct Vmxnet3_TxCompDesc),
446 tq->comp_ring.base, tq->comp_ring.basePA);
447 tq->comp_ring.base = NULL;
448 }
449 if (tq->buf_info) {
450 dma_free_coherent(&adapter->pdev->dev,
451 tq->tx_ring.size * sizeof(tq->buf_info[0]),
452 tq->buf_info, tq->buf_info_pa);
453 tq->buf_info = NULL;
454 }
455 }
456
457
458 /* Destroy all tx queues */
459 void
460 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
461 {
462 int i;
463
464 for (i = 0; i < adapter->num_tx_queues; i++)
465 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
466 }
467
468
469 static void
470 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
471 struct vmxnet3_adapter *adapter)
472 {
473 int i;
474
475 /* reset the tx ring contents to 0 and reset the tx ring states */
476 memset(tq->tx_ring.base, 0, tq->tx_ring.size *
477 sizeof(struct Vmxnet3_TxDesc));
478 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
479 tq->tx_ring.gen = VMXNET3_INIT_GEN;
480
481 memset(tq->data_ring.base, 0, tq->data_ring.size *
482 sizeof(struct Vmxnet3_TxDataDesc));
483
484 /* reset the tx comp ring contents to 0 and reset comp ring states */
485 memset(tq->comp_ring.base, 0, tq->comp_ring.size *
486 sizeof(struct Vmxnet3_TxCompDesc));
487 tq->comp_ring.next2proc = 0;
488 tq->comp_ring.gen = VMXNET3_INIT_GEN;
489
490 /* reset the bookkeeping data */
491 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
492 for (i = 0; i < tq->tx_ring.size; i++)
493 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
494
495 /* stats are not reset */
496 }
497
498
499 static int
500 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
501 struct vmxnet3_adapter *adapter)
502 {
503 size_t sz;
504
505 BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
506 tq->comp_ring.base || tq->buf_info);
507
508 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
509 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
510 &tq->tx_ring.basePA, GFP_KERNEL);
511 if (!tq->tx_ring.base) {
512 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
513 goto err;
514 }
515
516 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
517 tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
518 &tq->data_ring.basePA, GFP_KERNEL);
519 if (!tq->data_ring.base) {
520 netdev_err(adapter->netdev, "failed to allocate data ring\n");
521 goto err;
522 }
523
524 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
525 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
526 &tq->comp_ring.basePA, GFP_KERNEL);
527 if (!tq->comp_ring.base) {
528 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
529 goto err;
530 }
531
532 sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
533 tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
534 &tq->buf_info_pa, GFP_KERNEL);
535 if (!tq->buf_info)
536 goto err;
537
538 return 0;
539
540 err:
541 vmxnet3_tq_destroy(tq, adapter);
542 return -ENOMEM;
543 }
544
545 static void
546 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
547 {
548 int i;
549
550 for (i = 0; i < adapter->num_tx_queues; i++)
551 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
552 }
553
554 /*
555 * starting from ring->next2fill, allocate rx buffers for the given ring
556 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers
557 * are allocated or allocation fails
558 */
559
560 static int
561 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
562 int num_to_alloc, struct vmxnet3_adapter *adapter)
563 {
564 int num_allocated = 0;
565 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
566 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
567 u32 val;
568
569 while (num_allocated <= num_to_alloc) {
570 struct vmxnet3_rx_buf_info *rbi;
571 union Vmxnet3_GenericDesc *gd;
572
573 rbi = rbi_base + ring->next2fill;
574 gd = ring->base + ring->next2fill;
575
576 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
577 if (rbi->skb == NULL) {
578 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
579 rbi->len,
580 GFP_KERNEL);
581 if (unlikely(rbi->skb == NULL)) {
582 rq->stats.rx_buf_alloc_failure++;
583 break;
584 }
585
586 rbi->dma_addr = dma_map_single(
587 &adapter->pdev->dev,
588 rbi->skb->data, rbi->len,
589 PCI_DMA_FROMDEVICE);
590 } else {
591 /* rx buffer skipped by the device */
592 }
593 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
594 } else {
595 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
596 rbi->len != PAGE_SIZE);
597
598 if (rbi->page == NULL) {
599 rbi->page = alloc_page(GFP_ATOMIC);
600 if (unlikely(rbi->page == NULL)) {
601 rq->stats.rx_buf_alloc_failure++;
602 break;
603 }
604 rbi->dma_addr = dma_map_page(
605 &adapter->pdev->dev,
606 rbi->page, 0, PAGE_SIZE,
607 PCI_DMA_FROMDEVICE);
608 } else {
609 /* rx buffers skipped by the device */
610 }
611 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
612 }
613
614 BUG_ON(rbi->dma_addr == 0);
615 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
616 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
617 | val | rbi->len);
618
619 /* Fill the last buffer but dont mark it ready, or else the
620 * device will think that the queue is full */
621 if (num_allocated == num_to_alloc)
622 break;
623
624 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
625 num_allocated++;
626 vmxnet3_cmd_ring_adv_next2fill(ring);
627 }
628
629 netdev_dbg(adapter->netdev,
630 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
631 num_allocated, ring->next2fill, ring->next2comp);
632
633 /* so that the device can distinguish a full ring and an empty ring */
634 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
635
636 return num_allocated;
637 }
638
639
640 static void
641 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
642 struct vmxnet3_rx_buf_info *rbi)
643 {
644 struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
645 skb_shinfo(skb)->nr_frags;
646
647 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
648
649 __skb_frag_set_page(frag, rbi->page);
650 frag->page_offset = 0;
651 skb_frag_size_set(frag, rcd->len);
652 skb->data_len += rcd->len;
653 skb->truesize += PAGE_SIZE;
654 skb_shinfo(skb)->nr_frags++;
655 }
656
657
658 static void
659 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
660 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
661 struct vmxnet3_adapter *adapter)
662 {
663 u32 dw2, len;
664 unsigned long buf_offset;
665 int i;
666 union Vmxnet3_GenericDesc *gdesc;
667 struct vmxnet3_tx_buf_info *tbi = NULL;
668
669 BUG_ON(ctx->copy_size > skb_headlen(skb));
670
671 /* use the previous gen bit for the SOP desc */
672 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
673
674 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
675 gdesc = ctx->sop_txd; /* both loops below can be skipped */
676
677 /* no need to map the buffer if headers are copied */
678 if (ctx->copy_size) {
679 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
680 tq->tx_ring.next2fill *
681 sizeof(struct Vmxnet3_TxDataDesc));
682 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
683 ctx->sop_txd->dword[3] = 0;
684
685 tbi = tq->buf_info + tq->tx_ring.next2fill;
686 tbi->map_type = VMXNET3_MAP_NONE;
687
688 netdev_dbg(adapter->netdev,
689 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
690 tq->tx_ring.next2fill,
691 le64_to_cpu(ctx->sop_txd->txd.addr),
692 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
693 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
694
695 /* use the right gen for non-SOP desc */
696 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
697 }
698
699 /* linear part can use multiple tx desc if it's big */
700 len = skb_headlen(skb) - ctx->copy_size;
701 buf_offset = ctx->copy_size;
702 while (len) {
703 u32 buf_size;
704
705 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
706 buf_size = len;
707 dw2 |= len;
708 } else {
709 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
710 /* spec says that for TxDesc.len, 0 == 2^14 */
711 }
712
713 tbi = tq->buf_info + tq->tx_ring.next2fill;
714 tbi->map_type = VMXNET3_MAP_SINGLE;
715 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
716 skb->data + buf_offset, buf_size,
717 PCI_DMA_TODEVICE);
718
719 tbi->len = buf_size;
720
721 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
722 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
723
724 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
725 gdesc->dword[2] = cpu_to_le32(dw2);
726 gdesc->dword[3] = 0;
727
728 netdev_dbg(adapter->netdev,
729 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
730 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
731 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
732 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
733 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
734
735 len -= buf_size;
736 buf_offset += buf_size;
737 }
738
739 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
740 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
741 u32 buf_size;
742
743 buf_offset = 0;
744 len = skb_frag_size(frag);
745 while (len) {
746 tbi = tq->buf_info + tq->tx_ring.next2fill;
747 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
748 buf_size = len;
749 dw2 |= len;
750 } else {
751 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
752 /* spec says that for TxDesc.len, 0 == 2^14 */
753 }
754 tbi->map_type = VMXNET3_MAP_PAGE;
755 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
756 buf_offset, buf_size,
757 DMA_TO_DEVICE);
758
759 tbi->len = buf_size;
760
761 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
762 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
763
764 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
765 gdesc->dword[2] = cpu_to_le32(dw2);
766 gdesc->dword[3] = 0;
767
768 netdev_dbg(adapter->netdev,
769 "txd[%u]: 0x%llu %u %u\n",
770 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
771 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
772 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
773 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
774
775 len -= buf_size;
776 buf_offset += buf_size;
777 }
778 }
779
780 ctx->eop_txd = gdesc;
781
782 /* set the last buf_info for the pkt */
783 tbi->skb = skb;
784 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
785 }
786
787
788 /* Init all tx queues */
789 static void
790 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
791 {
792 int i;
793
794 for (i = 0; i < adapter->num_tx_queues; i++)
795 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
796 }
797
798
799 /*
800 * parse and copy relevant protocol headers:
801 * For a tso pkt, relevant headers are L2/3/4 including options
802 * For a pkt requesting csum offloading, they are L2/3 and may include L4
803 * if it's a TCP/UDP pkt
804 *
805 * Returns:
806 * -1: error happens during parsing
807 * 0: protocol headers parsed, but too big to be copied
808 * 1: protocol headers parsed and copied
809 *
810 * Other effects:
811 * 1. related *ctx fields are updated.
812 * 2. ctx->copy_size is # of bytes copied
813 * 3. the portion copied is guaranteed to be in the linear part
814 *
815 */
816 static int
817 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
818 struct vmxnet3_tx_ctx *ctx,
819 struct vmxnet3_adapter *adapter)
820 {
821 struct Vmxnet3_TxDataDesc *tdd;
822
823 if (ctx->mss) { /* TSO */
824 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
825 ctx->l4_hdr_size = tcp_hdrlen(skb);
826 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
827 } else {
828 if (skb->ip_summed == CHECKSUM_PARTIAL) {
829 ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
830
831 if (ctx->ipv4) {
832 const struct iphdr *iph = ip_hdr(skb);
833
834 if (iph->protocol == IPPROTO_TCP)
835 ctx->l4_hdr_size = tcp_hdrlen(skb);
836 else if (iph->protocol == IPPROTO_UDP)
837 ctx->l4_hdr_size = sizeof(struct udphdr);
838 else
839 ctx->l4_hdr_size = 0;
840 } else {
841 /* for simplicity, don't copy L4 headers */
842 ctx->l4_hdr_size = 0;
843 }
844 ctx->copy_size = min(ctx->eth_ip_hdr_size +
845 ctx->l4_hdr_size, skb->len);
846 } else {
847 ctx->eth_ip_hdr_size = 0;
848 ctx->l4_hdr_size = 0;
849 /* copy as much as allowed */
850 ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
851 , skb_headlen(skb));
852 }
853
854 /* make sure headers are accessible directly */
855 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
856 goto err;
857 }
858
859 if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
860 tq->stats.oversized_hdr++;
861 ctx->copy_size = 0;
862 return 0;
863 }
864
865 tdd = tq->data_ring.base + tq->tx_ring.next2fill;
866
867 memcpy(tdd->data, skb->data, ctx->copy_size);
868 netdev_dbg(adapter->netdev,
869 "copy %u bytes to dataRing[%u]\n",
870 ctx->copy_size, tq->tx_ring.next2fill);
871 return 1;
872
873 err:
874 return -1;
875 }
876
877
878 static void
879 vmxnet3_prepare_tso(struct sk_buff *skb,
880 struct vmxnet3_tx_ctx *ctx)
881 {
882 struct tcphdr *tcph = tcp_hdr(skb);
883
884 if (ctx->ipv4) {
885 struct iphdr *iph = ip_hdr(skb);
886
887 iph->check = 0;
888 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
889 IPPROTO_TCP, 0);
890 } else {
891 struct ipv6hdr *iph = ipv6_hdr(skb);
892
893 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
894 IPPROTO_TCP, 0);
895 }
896 }
897
898 static int txd_estimate(const struct sk_buff *skb)
899 {
900 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
901 int i;
902
903 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
904 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
905
906 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
907 }
908 return count;
909 }
910
911 /*
912 * Transmits a pkt thru a given tq
913 * Returns:
914 * NETDEV_TX_OK: descriptors are setup successfully
915 * NETDEV_TX_OK: error occurred, the pkt is dropped
916 * NETDEV_TX_BUSY: tx ring is full, queue is stopped
917 *
918 * Side-effects:
919 * 1. tx ring may be changed
920 * 2. tq stats may be updated accordingly
921 * 3. shared->txNumDeferred may be updated
922 */
923
924 static int
925 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926 struct vmxnet3_adapter *adapter, struct net_device *netdev)
927 {
928 int ret;
929 u32 count;
930 unsigned long flags;
931 struct vmxnet3_tx_ctx ctx;
932 union Vmxnet3_GenericDesc *gdesc;
933 #ifdef __BIG_ENDIAN_BITFIELD
934 /* Use temporary descriptor to avoid touching bits multiple times */
935 union Vmxnet3_GenericDesc tempTxDesc;
936 #endif
937
938 count = txd_estimate(skb);
939
940 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
941
942 ctx.mss = skb_shinfo(skb)->gso_size;
943 if (ctx.mss) {
944 if (skb_header_cloned(skb)) {
945 if (unlikely(pskb_expand_head(skb, 0, 0,
946 GFP_ATOMIC) != 0)) {
947 tq->stats.drop_tso++;
948 goto drop_pkt;
949 }
950 tq->stats.copy_skb_header++;
951 }
952 vmxnet3_prepare_tso(skb, &ctx);
953 } else {
954 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
955
956 /* non-tso pkts must not use more than
957 * VMXNET3_MAX_TXD_PER_PKT entries
958 */
959 if (skb_linearize(skb) != 0) {
960 tq->stats.drop_too_many_frags++;
961 goto drop_pkt;
962 }
963 tq->stats.linearized++;
964
965 /* recalculate the # of descriptors to use */
966 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
967 }
968 }
969
970 spin_lock_irqsave(&tq->tx_lock, flags);
971
972 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
973 tq->stats.tx_ring_full++;
974 netdev_dbg(adapter->netdev,
975 "tx queue stopped on %s, next2comp %u"
976 " next2fill %u\n", adapter->netdev->name,
977 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
978
979 vmxnet3_tq_stop(tq, adapter);
980 spin_unlock_irqrestore(&tq->tx_lock, flags);
981 return NETDEV_TX_BUSY;
982 }
983
984
985 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
986 if (ret >= 0) {
987 BUG_ON(ret <= 0 && ctx.copy_size != 0);
988 /* hdrs parsed, check against other limits */
989 if (ctx.mss) {
990 if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
991 VMXNET3_MAX_TX_BUF_SIZE)) {
992 goto hdr_too_big;
993 }
994 } else {
995 if (skb->ip_summed == CHECKSUM_PARTIAL) {
996 if (unlikely(ctx.eth_ip_hdr_size +
997 skb->csum_offset >
998 VMXNET3_MAX_CSUM_OFFSET)) {
999 goto hdr_too_big;
1000 }
1001 }
1002 }
1003 } else {
1004 tq->stats.drop_hdr_inspect_err++;
1005 goto unlock_drop_pkt;
1006 }
1007
1008 /* fill tx descs related to addr & len */
1009 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1010
1011 /* setup the EOP desc */
1012 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1013
1014 /* setup the SOP desc */
1015 #ifdef __BIG_ENDIAN_BITFIELD
1016 gdesc = &tempTxDesc;
1017 gdesc->dword[2] = ctx.sop_txd->dword[2];
1018 gdesc->dword[3] = ctx.sop_txd->dword[3];
1019 #else
1020 gdesc = ctx.sop_txd;
1021 #endif
1022 if (ctx.mss) {
1023 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1024 gdesc->txd.om = VMXNET3_OM_TSO;
1025 gdesc->txd.msscof = ctx.mss;
1026 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1027 gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
1028 } else {
1029 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1030 gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1031 gdesc->txd.om = VMXNET3_OM_CSUM;
1032 gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1033 skb->csum_offset;
1034 } else {
1035 gdesc->txd.om = 0;
1036 gdesc->txd.msscof = 0;
1037 }
1038 le32_add_cpu(&tq->shared->txNumDeferred, 1);
1039 }
1040
1041 if (vlan_tx_tag_present(skb)) {
1042 gdesc->txd.ti = 1;
1043 gdesc->txd.tci = vlan_tx_tag_get(skb);
1044 }
1045
1046 /* finally flips the GEN bit of the SOP desc. */
1047 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1048 VMXNET3_TXD_GEN);
1049 #ifdef __BIG_ENDIAN_BITFIELD
1050 /* Finished updating in bitfields of Tx Desc, so write them in original
1051 * place.
1052 */
1053 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1054 (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1055 gdesc = ctx.sop_txd;
1056 #endif
1057 netdev_dbg(adapter->netdev,
1058 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1059 (u32)(ctx.sop_txd -
1060 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1061 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1062
1063 spin_unlock_irqrestore(&tq->tx_lock, flags);
1064
1065 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1066 le32_to_cpu(tq->shared->txThreshold)) {
1067 tq->shared->txNumDeferred = 0;
1068 VMXNET3_WRITE_BAR0_REG(adapter,
1069 VMXNET3_REG_TXPROD + tq->qid * 8,
1070 tq->tx_ring.next2fill);
1071 }
1072
1073 return NETDEV_TX_OK;
1074
1075 hdr_too_big:
1076 tq->stats.drop_oversized_hdr++;
1077 unlock_drop_pkt:
1078 spin_unlock_irqrestore(&tq->tx_lock, flags);
1079 drop_pkt:
1080 tq->stats.drop_total++;
1081 dev_kfree_skb(skb);
1082 return NETDEV_TX_OK;
1083 }
1084
1085
1086 static netdev_tx_t
1087 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1088 {
1089 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1090
1091 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1092 return vmxnet3_tq_xmit(skb,
1093 &adapter->tx_queue[skb->queue_mapping],
1094 adapter, netdev);
1095 }
1096
1097
1098 static void
1099 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1100 struct sk_buff *skb,
1101 union Vmxnet3_GenericDesc *gdesc)
1102 {
1103 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1104 /* typical case: TCP/UDP over IP and both csums are correct */
1105 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
1106 VMXNET3_RCD_CSUM_OK) {
1107 skb->ip_summed = CHECKSUM_UNNECESSARY;
1108 BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1109 BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));
1110 BUG_ON(gdesc->rcd.frg);
1111 } else {
1112 if (gdesc->rcd.csum) {
1113 skb->csum = htons(gdesc->rcd.csum);
1114 skb->ip_summed = CHECKSUM_PARTIAL;
1115 } else {
1116 skb_checksum_none_assert(skb);
1117 }
1118 }
1119 } else {
1120 skb_checksum_none_assert(skb);
1121 }
1122 }
1123
1124
1125 static void
1126 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1127 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)
1128 {
1129 rq->stats.drop_err++;
1130 if (!rcd->fcs)
1131 rq->stats.drop_fcs++;
1132
1133 rq->stats.drop_total++;
1134
1135 /*
1136 * We do not unmap and chain the rx buffer to the skb.
1137 * We basically pretend this buffer is not used and will be recycled
1138 * by vmxnet3_rq_alloc_rx_buf()
1139 */
1140
1141 /*
1142 * ctx->skb may be NULL if this is the first and the only one
1143 * desc for the pkt
1144 */
1145 if (ctx->skb)
1146 dev_kfree_skb_irq(ctx->skb);
1147
1148 ctx->skb = NULL;
1149 }
1150
1151
1152 static int
1153 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1154 struct vmxnet3_adapter *adapter, int quota)
1155 {
1156 static const u32 rxprod_reg[2] = {
1157 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1158 };
1159 u32 num_rxd = 0;
1160 bool skip_page_frags = false;
1161 struct Vmxnet3_RxCompDesc *rcd;
1162 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1163 #ifdef __BIG_ENDIAN_BITFIELD
1164 struct Vmxnet3_RxDesc rxCmdDesc;
1165 struct Vmxnet3_RxCompDesc rxComp;
1166 #endif
1167 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1168 &rxComp);
1169 while (rcd->gen == rq->comp_ring.gen) {
1170 struct vmxnet3_rx_buf_info *rbi;
1171 struct sk_buff *skb, *new_skb = NULL;
1172 struct page *new_page = NULL;
1173 int num_to_alloc;
1174 struct Vmxnet3_RxDesc *rxd;
1175 u32 idx, ring_idx;
1176 struct vmxnet3_cmd_ring *ring = NULL;
1177 if (num_rxd >= quota) {
1178 /* we may stop even before we see the EOP desc of
1179 * the current pkt
1180 */
1181 break;
1182 }
1183 num_rxd++;
1184 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1185 idx = rcd->rxdIdx;
1186 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1187 ring = rq->rx_ring + ring_idx;
1188 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1189 &rxCmdDesc);
1190 rbi = rq->buf_info[ring_idx] + idx;
1191
1192 BUG_ON(rxd->addr != rbi->dma_addr ||
1193 rxd->len != rbi->len);
1194
1195 if (unlikely(rcd->eop && rcd->err)) {
1196 vmxnet3_rx_error(rq, rcd, ctx, adapter);
1197 goto rcd_done;
1198 }
1199
1200 if (rcd->sop) { /* first buf of the pkt */
1201 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1202 rcd->rqID != rq->qid);
1203
1204 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1205 BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1206
1207 if (unlikely(rcd->len == 0)) {
1208 /* Pretend the rx buffer is skipped. */
1209 BUG_ON(!(rcd->sop && rcd->eop));
1210 netdev_dbg(adapter->netdev,
1211 "rxRing[%u][%u] 0 length\n",
1212 ring_idx, idx);
1213 goto rcd_done;
1214 }
1215
1216 skip_page_frags = false;
1217 ctx->skb = rbi->skb;
1218 new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1219 rbi->len);
1220 if (new_skb == NULL) {
1221 /* Skb allocation failed, do not handover this
1222 * skb to stack. Reuse it. Drop the existing pkt
1223 */
1224 rq->stats.rx_buf_alloc_failure++;
1225 ctx->skb = NULL;
1226 rq->stats.drop_total++;
1227 skip_page_frags = true;
1228 goto rcd_done;
1229 }
1230
1231 dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
1232 rbi->len,
1233 PCI_DMA_FROMDEVICE);
1234
1235 #ifdef VMXNET3_RSS
1236 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1237 (adapter->netdev->features & NETIF_F_RXHASH))
1238 ctx->skb->rxhash = le32_to_cpu(rcd->rssHash);
1239 #endif
1240 skb_put(ctx->skb, rcd->len);
1241
1242 /* Immediate refill */
1243 rbi->skb = new_skb;
1244 rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
1245 rbi->skb->data, rbi->len,
1246 PCI_DMA_FROMDEVICE);
1247 rxd->addr = cpu_to_le64(rbi->dma_addr);
1248 rxd->len = rbi->len;
1249
1250 } else {
1251 BUG_ON(ctx->skb == NULL && !skip_page_frags);
1252
1253 /* non SOP buffer must be type 1 in most cases */
1254 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1255 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1256
1257 /* If an sop buffer was dropped, skip all
1258 * following non-sop fragments. They will be reused.
1259 */
1260 if (skip_page_frags)
1261 goto rcd_done;
1262
1263 new_page = alloc_page(GFP_ATOMIC);
1264 if (unlikely(new_page == NULL)) {
1265 /* Replacement page frag could not be allocated.
1266 * Reuse this page. Drop the pkt and free the
1267 * skb which contained this page as a frag. Skip
1268 * processing all the following non-sop frags.
1269 */
1270 rq->stats.rx_buf_alloc_failure++;
1271 dev_kfree_skb(ctx->skb);
1272 ctx->skb = NULL;
1273 skip_page_frags = true;
1274 goto rcd_done;
1275 }
1276
1277 if (rcd->len) {
1278 dma_unmap_page(&adapter->pdev->dev,
1279 rbi->dma_addr, rbi->len,
1280 PCI_DMA_FROMDEVICE);
1281
1282 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1283 }
1284
1285 /* Immediate refill */
1286 rbi->page = new_page;
1287 rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
1288 rbi->page,
1289 0, PAGE_SIZE,
1290 PCI_DMA_FROMDEVICE);
1291 rxd->addr = cpu_to_le64(rbi->dma_addr);
1292 rxd->len = rbi->len;
1293 }
1294
1295
1296 skb = ctx->skb;
1297 if (rcd->eop) {
1298 skb->len += skb->data_len;
1299
1300 vmxnet3_rx_csum(adapter, skb,
1301 (union Vmxnet3_GenericDesc *)rcd);
1302 skb->protocol = eth_type_trans(skb, adapter->netdev);
1303
1304 if (unlikely(rcd->ts))
1305 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1306
1307 if (adapter->netdev->features & NETIF_F_LRO)
1308 netif_receive_skb(skb);
1309 else
1310 napi_gro_receive(&rq->napi, skb);
1311
1312 ctx->skb = NULL;
1313 }
1314
1315 rcd_done:
1316 /* device may have skipped some rx descs */
1317 ring->next2comp = idx;
1318 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1319 ring = rq->rx_ring + ring_idx;
1320 while (num_to_alloc) {
1321 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1322 &rxCmdDesc);
1323 BUG_ON(!rxd->addr);
1324
1325 /* Recv desc is ready to be used by the device */
1326 rxd->gen = ring->gen;
1327 vmxnet3_cmd_ring_adv_next2fill(ring);
1328 num_to_alloc--;
1329 }
1330
1331 /* if needed, update the register */
1332 if (unlikely(rq->shared->updateRxProd)) {
1333 VMXNET3_WRITE_BAR0_REG(adapter,
1334 rxprod_reg[ring_idx] + rq->qid * 8,
1335 ring->next2fill);
1336 }
1337
1338 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1339 vmxnet3_getRxComp(rcd,
1340 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1341 }
1342
1343 return num_rxd;
1344 }
1345
1346
1347 static void
1348 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1349 struct vmxnet3_adapter *adapter)
1350 {
1351 u32 i, ring_idx;
1352 struct Vmxnet3_RxDesc *rxd;
1353
1354 for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1355 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1356 #ifdef __BIG_ENDIAN_BITFIELD
1357 struct Vmxnet3_RxDesc rxDesc;
1358 #endif
1359 vmxnet3_getRxDesc(rxd,
1360 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1361
1362 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1363 rq->buf_info[ring_idx][i].skb) {
1364 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1365 rxd->len, PCI_DMA_FROMDEVICE);
1366 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1367 rq->buf_info[ring_idx][i].skb = NULL;
1368 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1369 rq->buf_info[ring_idx][i].page) {
1370 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1371 rxd->len, PCI_DMA_FROMDEVICE);
1372 put_page(rq->buf_info[ring_idx][i].page);
1373 rq->buf_info[ring_idx][i].page = NULL;
1374 }
1375 }
1376
1377 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1378 rq->rx_ring[ring_idx].next2fill =
1379 rq->rx_ring[ring_idx].next2comp = 0;
1380 }
1381
1382 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1383 rq->comp_ring.next2proc = 0;
1384 }
1385
1386
1387 static void
1388 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1389 {
1390 int i;
1391
1392 for (i = 0; i < adapter->num_rx_queues; i++)
1393 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1394 }
1395
1396
1397 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1398 struct vmxnet3_adapter *adapter)
1399 {
1400 int i;
1401 int j;
1402
1403 /* all rx buffers must have already been freed */
1404 for (i = 0; i < 2; i++) {
1405 if (rq->buf_info[i]) {
1406 for (j = 0; j < rq->rx_ring[i].size; j++)
1407 BUG_ON(rq->buf_info[i][j].page != NULL);
1408 }
1409 }
1410
1411
1412 for (i = 0; i < 2; i++) {
1413 if (rq->rx_ring[i].base) {
1414 dma_free_coherent(&adapter->pdev->dev,
1415 rq->rx_ring[i].size
1416 * sizeof(struct Vmxnet3_RxDesc),
1417 rq->rx_ring[i].base,
1418 rq->rx_ring[i].basePA);
1419 rq->rx_ring[i].base = NULL;
1420 }
1421 rq->buf_info[i] = NULL;
1422 }
1423
1424 if (rq->comp_ring.base) {
1425 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1426 * sizeof(struct Vmxnet3_RxCompDesc),
1427 rq->comp_ring.base, rq->comp_ring.basePA);
1428 rq->comp_ring.base = NULL;
1429 }
1430
1431 if (rq->buf_info[0]) {
1432 size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
1433 (rq->rx_ring[0].size + rq->rx_ring[1].size);
1434 dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
1435 rq->buf_info_pa);
1436 }
1437 }
1438
1439
1440 static int
1441 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1442 struct vmxnet3_adapter *adapter)
1443 {
1444 int i;
1445
1446 /* initialize buf_info */
1447 for (i = 0; i < rq->rx_ring[0].size; i++) {
1448
1449 /* 1st buf for a pkt is skbuff */
1450 if (i % adapter->rx_buf_per_pkt == 0) {
1451 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1452 rq->buf_info[0][i].len = adapter->skb_buf_size;
1453 } else { /* subsequent bufs for a pkt is frag */
1454 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1455 rq->buf_info[0][i].len = PAGE_SIZE;
1456 }
1457 }
1458 for (i = 0; i < rq->rx_ring[1].size; i++) {
1459 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1460 rq->buf_info[1][i].len = PAGE_SIZE;
1461 }
1462
1463 /* reset internal state and allocate buffers for both rings */
1464 for (i = 0; i < 2; i++) {
1465 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1466
1467 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1468 sizeof(struct Vmxnet3_RxDesc));
1469 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1470 }
1471 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1472 adapter) == 0) {
1473 /* at least has 1 rx buffer for the 1st ring */
1474 return -ENOMEM;
1475 }
1476 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1477
1478 /* reset the comp ring */
1479 rq->comp_ring.next2proc = 0;
1480 memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1481 sizeof(struct Vmxnet3_RxCompDesc));
1482 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1483
1484 /* reset rxctx */
1485 rq->rx_ctx.skb = NULL;
1486
1487 /* stats are not reset */
1488 return 0;
1489 }
1490
1491
1492 static int
1493 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1494 {
1495 int i, err = 0;
1496
1497 for (i = 0; i < adapter->num_rx_queues; i++) {
1498 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1499 if (unlikely(err)) {
1500 dev_err(&adapter->netdev->dev, "%s: failed to "
1501 "initialize rx queue%i\n",
1502 adapter->netdev->name, i);
1503 break;
1504 }
1505 }
1506 return err;
1507
1508 }
1509
1510
1511 static int
1512 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1513 {
1514 int i;
1515 size_t sz;
1516 struct vmxnet3_rx_buf_info *bi;
1517
1518 for (i = 0; i < 2; i++) {
1519
1520 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1521 rq->rx_ring[i].base = dma_alloc_coherent(
1522 &adapter->pdev->dev, sz,
1523 &rq->rx_ring[i].basePA,
1524 GFP_KERNEL);
1525 if (!rq->rx_ring[i].base) {
1526 netdev_err(adapter->netdev,
1527 "failed to allocate rx ring %d\n", i);
1528 goto err;
1529 }
1530 }
1531
1532 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1533 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1534 &rq->comp_ring.basePA,
1535 GFP_KERNEL);
1536 if (!rq->comp_ring.base) {
1537 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1538 goto err;
1539 }
1540
1541 sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1542 rq->rx_ring[1].size);
1543 bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
1544 GFP_KERNEL);
1545 if (!bi)
1546 goto err;
1547
1548 rq->buf_info[0] = bi;
1549 rq->buf_info[1] = bi + rq->rx_ring[0].size;
1550
1551 return 0;
1552
1553 err:
1554 vmxnet3_rq_destroy(rq, adapter);
1555 return -ENOMEM;
1556 }
1557
1558
1559 static int
1560 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1561 {
1562 int i, err = 0;
1563
1564 for (i = 0; i < adapter->num_rx_queues; i++) {
1565 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1566 if (unlikely(err)) {
1567 dev_err(&adapter->netdev->dev,
1568 "%s: failed to create rx queue%i\n",
1569 adapter->netdev->name, i);
1570 goto err_out;
1571 }
1572 }
1573 return err;
1574 err_out:
1575 vmxnet3_rq_destroy_all(adapter);
1576 return err;
1577
1578 }
1579
1580 /* Multiple queue aware polling function for tx and rx */
1581
1582 static int
1583 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1584 {
1585 int rcd_done = 0, i;
1586 if (unlikely(adapter->shared->ecr))
1587 vmxnet3_process_events(adapter);
1588 for (i = 0; i < adapter->num_tx_queues; i++)
1589 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1590
1591 for (i = 0; i < adapter->num_rx_queues; i++)
1592 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1593 adapter, budget);
1594 return rcd_done;
1595 }
1596
1597
1598 static int
1599 vmxnet3_poll(struct napi_struct *napi, int budget)
1600 {
1601 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1602 struct vmxnet3_rx_queue, napi);
1603 int rxd_done;
1604
1605 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1606
1607 if (rxd_done < budget) {
1608 napi_complete(napi);
1609 vmxnet3_enable_all_intrs(rx_queue->adapter);
1610 }
1611 return rxd_done;
1612 }
1613
1614 /*
1615 * NAPI polling function for MSI-X mode with multiple Rx queues
1616 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1617 */
1618
1619 static int
1620 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1621 {
1622 struct vmxnet3_rx_queue *rq = container_of(napi,
1623 struct vmxnet3_rx_queue, napi);
1624 struct vmxnet3_adapter *adapter = rq->adapter;
1625 int rxd_done;
1626
1627 /* When sharing interrupt with corresponding tx queue, process
1628 * tx completions in that queue as well
1629 */
1630 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1631 struct vmxnet3_tx_queue *tq =
1632 &adapter->tx_queue[rq - adapter->rx_queue];
1633 vmxnet3_tq_tx_complete(tq, adapter);
1634 }
1635
1636 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1637
1638 if (rxd_done < budget) {
1639 napi_complete(napi);
1640 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1641 }
1642 return rxd_done;
1643 }
1644
1645
1646 #ifdef CONFIG_PCI_MSI
1647
1648 /*
1649 * Handle completion interrupts on tx queues
1650 * Returns whether or not the intr is handled
1651 */
1652
1653 static irqreturn_t
1654 vmxnet3_msix_tx(int irq, void *data)
1655 {
1656 struct vmxnet3_tx_queue *tq = data;
1657 struct vmxnet3_adapter *adapter = tq->adapter;
1658
1659 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1660 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1661
1662 /* Handle the case where only one irq is allocate for all tx queues */
1663 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1664 int i;
1665 for (i = 0; i < adapter->num_tx_queues; i++) {
1666 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1667 vmxnet3_tq_tx_complete(txq, adapter);
1668 }
1669 } else {
1670 vmxnet3_tq_tx_complete(tq, adapter);
1671 }
1672 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1673
1674 return IRQ_HANDLED;
1675 }
1676
1677
1678 /*
1679 * Handle completion interrupts on rx queues. Returns whether or not the
1680 * intr is handled
1681 */
1682
1683 static irqreturn_t
1684 vmxnet3_msix_rx(int irq, void *data)
1685 {
1686 struct vmxnet3_rx_queue *rq = data;
1687 struct vmxnet3_adapter *adapter = rq->adapter;
1688
1689 /* disable intr if needed */
1690 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1691 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1692 napi_schedule(&rq->napi);
1693
1694 return IRQ_HANDLED;
1695 }
1696
1697 /*
1698 *----------------------------------------------------------------------------
1699 *
1700 * vmxnet3_msix_event --
1701 *
1702 * vmxnet3 msix event intr handler
1703 *
1704 * Result:
1705 * whether or not the intr is handled
1706 *
1707 *----------------------------------------------------------------------------
1708 */
1709
1710 static irqreturn_t
1711 vmxnet3_msix_event(int irq, void *data)
1712 {
1713 struct net_device *dev = data;
1714 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1715
1716 /* disable intr if needed */
1717 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1718 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1719
1720 if (adapter->shared->ecr)
1721 vmxnet3_process_events(adapter);
1722
1723 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1724
1725 return IRQ_HANDLED;
1726 }
1727
1728 #endif /* CONFIG_PCI_MSI */
1729
1730
1731 /* Interrupt handler for vmxnet3 */
1732 static irqreturn_t
1733 vmxnet3_intr(int irq, void *dev_id)
1734 {
1735 struct net_device *dev = dev_id;
1736 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1737
1738 if (adapter->intr.type == VMXNET3_IT_INTX) {
1739 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1740 if (unlikely(icr == 0))
1741 /* not ours */
1742 return IRQ_NONE;
1743 }
1744
1745
1746 /* disable intr if needed */
1747 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1748 vmxnet3_disable_all_intrs(adapter);
1749
1750 napi_schedule(&adapter->rx_queue[0].napi);
1751
1752 return IRQ_HANDLED;
1753 }
1754
1755 #ifdef CONFIG_NET_POLL_CONTROLLER
1756
1757 /* netpoll callback. */
1758 static void
1759 vmxnet3_netpoll(struct net_device *netdev)
1760 {
1761 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1762
1763 switch (adapter->intr.type) {
1764 #ifdef CONFIG_PCI_MSI
1765 case VMXNET3_IT_MSIX: {
1766 int i;
1767 for (i = 0; i < adapter->num_rx_queues; i++)
1768 vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
1769 break;
1770 }
1771 #endif
1772 case VMXNET3_IT_MSI:
1773 default:
1774 vmxnet3_intr(0, adapter->netdev);
1775 break;
1776 }
1777
1778 }
1779 #endif /* CONFIG_NET_POLL_CONTROLLER */
1780
1781 static int
1782 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1783 {
1784 struct vmxnet3_intr *intr = &adapter->intr;
1785 int err = 0, i;
1786 int vector = 0;
1787
1788 #ifdef CONFIG_PCI_MSI
1789 if (adapter->intr.type == VMXNET3_IT_MSIX) {
1790 for (i = 0; i < adapter->num_tx_queues; i++) {
1791 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1792 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1793 adapter->netdev->name, vector);
1794 err = request_irq(
1795 intr->msix_entries[vector].vector,
1796 vmxnet3_msix_tx, 0,
1797 adapter->tx_queue[i].name,
1798 &adapter->tx_queue[i]);
1799 } else {
1800 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1801 adapter->netdev->name, vector);
1802 }
1803 if (err) {
1804 dev_err(&adapter->netdev->dev,
1805 "Failed to request irq for MSIX, %s, "
1806 "error %d\n",
1807 adapter->tx_queue[i].name, err);
1808 return err;
1809 }
1810
1811 /* Handle the case where only 1 MSIx was allocated for
1812 * all tx queues */
1813 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1814 for (; i < adapter->num_tx_queues; i++)
1815 adapter->tx_queue[i].comp_ring.intr_idx
1816 = vector;
1817 vector++;
1818 break;
1819 } else {
1820 adapter->tx_queue[i].comp_ring.intr_idx
1821 = vector++;
1822 }
1823 }
1824 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1825 vector = 0;
1826
1827 for (i = 0; i < adapter->num_rx_queues; i++) {
1828 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1829 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1830 adapter->netdev->name, vector);
1831 else
1832 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1833 adapter->netdev->name, vector);
1834 err = request_irq(intr->msix_entries[vector].vector,
1835 vmxnet3_msix_rx, 0,
1836 adapter->rx_queue[i].name,
1837 &(adapter->rx_queue[i]));
1838 if (err) {
1839 netdev_err(adapter->netdev,
1840 "Failed to request irq for MSIX, "
1841 "%s, error %d\n",
1842 adapter->rx_queue[i].name, err);
1843 return err;
1844 }
1845
1846 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1847 }
1848
1849 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1850 adapter->netdev->name, vector);
1851 err = request_irq(intr->msix_entries[vector].vector,
1852 vmxnet3_msix_event, 0,
1853 intr->event_msi_vector_name, adapter->netdev);
1854 intr->event_intr_idx = vector;
1855
1856 } else if (intr->type == VMXNET3_IT_MSI) {
1857 adapter->num_rx_queues = 1;
1858 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1859 adapter->netdev->name, adapter->netdev);
1860 } else {
1861 #endif
1862 adapter->num_rx_queues = 1;
1863 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1864 IRQF_SHARED, adapter->netdev->name,
1865 adapter->netdev);
1866 #ifdef CONFIG_PCI_MSI
1867 }
1868 #endif
1869 intr->num_intrs = vector + 1;
1870 if (err) {
1871 netdev_err(adapter->netdev,
1872 "Failed to request irq (intr type:%d), error %d\n",
1873 intr->type, err);
1874 } else {
1875 /* Number of rx queues will not change after this */
1876 for (i = 0; i < adapter->num_rx_queues; i++) {
1877 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1878 rq->qid = i;
1879 rq->qid2 = i + adapter->num_rx_queues;
1880 }
1881
1882
1883
1884 /* init our intr settings */
1885 for (i = 0; i < intr->num_intrs; i++)
1886 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1887 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1888 adapter->intr.event_intr_idx = 0;
1889 for (i = 0; i < adapter->num_tx_queues; i++)
1890 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1891 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1892 }
1893
1894 netdev_info(adapter->netdev,
1895 "intr type %u, mode %u, %u vectors allocated\n",
1896 intr->type, intr->mask_mode, intr->num_intrs);
1897 }
1898
1899 return err;
1900 }
1901
1902
1903 static void
1904 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1905 {
1906 struct vmxnet3_intr *intr = &adapter->intr;
1907 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1908
1909 switch (intr->type) {
1910 #ifdef CONFIG_PCI_MSI
1911 case VMXNET3_IT_MSIX:
1912 {
1913 int i, vector = 0;
1914
1915 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1916 for (i = 0; i < adapter->num_tx_queues; i++) {
1917 free_irq(intr->msix_entries[vector++].vector,
1918 &(adapter->tx_queue[i]));
1919 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1920 break;
1921 }
1922 }
1923
1924 for (i = 0; i < adapter->num_rx_queues; i++) {
1925 free_irq(intr->msix_entries[vector++].vector,
1926 &(adapter->rx_queue[i]));
1927 }
1928
1929 free_irq(intr->msix_entries[vector].vector,
1930 adapter->netdev);
1931 BUG_ON(vector >= intr->num_intrs);
1932 break;
1933 }
1934 #endif
1935 case VMXNET3_IT_MSI:
1936 free_irq(adapter->pdev->irq, adapter->netdev);
1937 break;
1938 case VMXNET3_IT_INTX:
1939 free_irq(adapter->pdev->irq, adapter->netdev);
1940 break;
1941 default:
1942 BUG();
1943 }
1944 }
1945
1946
1947 static void
1948 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1949 {
1950 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1951 u16 vid;
1952
1953 /* allow untagged pkts */
1954 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1955
1956 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1957 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1958 }
1959
1960
1961 static int
1962 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1963 {
1964 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1965
1966 if (!(netdev->flags & IFF_PROMISC)) {
1967 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1968 unsigned long flags;
1969
1970 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1971 spin_lock_irqsave(&adapter->cmd_lock, flags);
1972 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1973 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1974 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1975 }
1976
1977 set_bit(vid, adapter->active_vlans);
1978
1979 return 0;
1980 }
1981
1982
1983 static int
1984 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1985 {
1986 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1987
1988 if (!(netdev->flags & IFF_PROMISC)) {
1989 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1990 unsigned long flags;
1991
1992 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1993 spin_lock_irqsave(&adapter->cmd_lock, flags);
1994 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1995 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1996 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1997 }
1998
1999 clear_bit(vid, adapter->active_vlans);
2000
2001 return 0;
2002 }
2003
2004
2005 static u8 *
2006 vmxnet3_copy_mc(struct net_device *netdev)
2007 {
2008 u8 *buf = NULL;
2009 u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2010
2011 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2012 if (sz <= 0xffff) {
2013 /* We may be called with BH disabled */
2014 buf = kmalloc(sz, GFP_ATOMIC);
2015 if (buf) {
2016 struct netdev_hw_addr *ha;
2017 int i = 0;
2018
2019 netdev_for_each_mc_addr(ha, netdev)
2020 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2021 ETH_ALEN);
2022 }
2023 }
2024 return buf;
2025 }
2026
2027
2028 static void
2029 vmxnet3_set_mc(struct net_device *netdev)
2030 {
2031 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2032 unsigned long flags;
2033 struct Vmxnet3_RxFilterConf *rxConf =
2034 &adapter->shared->devRead.rxFilterConf;
2035 u8 *new_table = NULL;
2036 dma_addr_t new_table_pa = 0;
2037 u32 new_mode = VMXNET3_RXM_UCAST;
2038
2039 if (netdev->flags & IFF_PROMISC) {
2040 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2041 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2042
2043 new_mode |= VMXNET3_RXM_PROMISC;
2044 } else {
2045 vmxnet3_restore_vlan(adapter);
2046 }
2047
2048 if (netdev->flags & IFF_BROADCAST)
2049 new_mode |= VMXNET3_RXM_BCAST;
2050
2051 if (netdev->flags & IFF_ALLMULTI)
2052 new_mode |= VMXNET3_RXM_ALL_MULTI;
2053 else
2054 if (!netdev_mc_empty(netdev)) {
2055 new_table = vmxnet3_copy_mc(netdev);
2056 if (new_table) {
2057 new_mode |= VMXNET3_RXM_MCAST;
2058 rxConf->mfTableLen = cpu_to_le16(
2059 netdev_mc_count(netdev) * ETH_ALEN);
2060 new_table_pa = dma_map_single(
2061 &adapter->pdev->dev,
2062 new_table,
2063 rxConf->mfTableLen,
2064 PCI_DMA_TODEVICE);
2065 rxConf->mfTablePA = cpu_to_le64(new_table_pa);
2066 } else {
2067 netdev_info(netdev, "failed to copy mcast list"
2068 ", setting ALL_MULTI\n");
2069 new_mode |= VMXNET3_RXM_ALL_MULTI;
2070 }
2071 }
2072
2073
2074 if (!(new_mode & VMXNET3_RXM_MCAST)) {
2075 rxConf->mfTableLen = 0;
2076 rxConf->mfTablePA = 0;
2077 }
2078
2079 spin_lock_irqsave(&adapter->cmd_lock, flags);
2080 if (new_mode != rxConf->rxMode) {
2081 rxConf->rxMode = cpu_to_le32(new_mode);
2082 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2083 VMXNET3_CMD_UPDATE_RX_MODE);
2084 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2085 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2086 }
2087
2088 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2089 VMXNET3_CMD_UPDATE_MAC_FILTERS);
2090 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2091
2092 if (new_table) {
2093 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2094 rxConf->mfTableLen, PCI_DMA_TODEVICE);
2095 kfree(new_table);
2096 }
2097 }
2098
2099 void
2100 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2101 {
2102 int i;
2103
2104 for (i = 0; i < adapter->num_rx_queues; i++)
2105 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2106 }
2107
2108
2109 /*
2110 * Set up driver_shared based on settings in adapter.
2111 */
2112
2113 static void
2114 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2115 {
2116 struct Vmxnet3_DriverShared *shared = adapter->shared;
2117 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2118 struct Vmxnet3_TxQueueConf *tqc;
2119 struct Vmxnet3_RxQueueConf *rqc;
2120 int i;
2121
2122 memset(shared, 0, sizeof(*shared));
2123
2124 /* driver settings */
2125 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2126 devRead->misc.driverInfo.version = cpu_to_le32(
2127 VMXNET3_DRIVER_VERSION_NUM);
2128 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2129 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2130 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2131 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2132 *((u32 *)&devRead->misc.driverInfo.gos));
2133 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2134 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2135
2136 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2137 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2138
2139 /* set up feature flags */
2140 if (adapter->netdev->features & NETIF_F_RXCSUM)
2141 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2142
2143 if (adapter->netdev->features & NETIF_F_LRO) {
2144 devRead->misc.uptFeatures |= UPT1_F_LRO;
2145 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2146 }
2147 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2148 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2149
2150 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2151 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2152 devRead->misc.queueDescLen = cpu_to_le32(
2153 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2154 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2155
2156 /* tx queue settings */
2157 devRead->misc.numTxQueues = adapter->num_tx_queues;
2158 for (i = 0; i < adapter->num_tx_queues; i++) {
2159 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2160 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2161 tqc = &adapter->tqd_start[i].conf;
2162 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
2163 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2164 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2165 tqc->ddPA = cpu_to_le64(tq->buf_info_pa);
2166 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
2167 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
2168 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
2169 tqc->ddLen = cpu_to_le32(
2170 sizeof(struct vmxnet3_tx_buf_info) *
2171 tqc->txRingSize);
2172 tqc->intrIdx = tq->comp_ring.intr_idx;
2173 }
2174
2175 /* rx queue settings */
2176 devRead->misc.numRxQueues = adapter->num_rx_queues;
2177 for (i = 0; i < adapter->num_rx_queues; i++) {
2178 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2179 rqc = &adapter->rqd_start[i].conf;
2180 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2181 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2182 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
2183 rqc->ddPA = cpu_to_le64(rq->buf_info_pa);
2184 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
2185 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
2186 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
2187 rqc->ddLen = cpu_to_le32(
2188 sizeof(struct vmxnet3_rx_buf_info) *
2189 (rqc->rxRingSize[0] +
2190 rqc->rxRingSize[1]));
2191 rqc->intrIdx = rq->comp_ring.intr_idx;
2192 }
2193
2194 #ifdef VMXNET3_RSS
2195 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2196
2197 if (adapter->rss) {
2198 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2199 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2200 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2201 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2202 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2203 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2204 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2205 };
2206
2207 devRead->misc.uptFeatures |= UPT1_F_RSS;
2208 devRead->misc.numRxQueues = adapter->num_rx_queues;
2209 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2210 UPT1_RSS_HASH_TYPE_IPV4 |
2211 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2212 UPT1_RSS_HASH_TYPE_IPV6;
2213 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2214 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2215 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2216 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2217
2218 for (i = 0; i < rssConf->indTableSize; i++)
2219 rssConf->indTable[i] = ethtool_rxfh_indir_default(
2220 i, adapter->num_rx_queues);
2221
2222 devRead->rssConfDesc.confVer = 1;
2223 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2224 devRead->rssConfDesc.confPA =
2225 cpu_to_le64(adapter->rss_conf_pa);
2226 }
2227
2228 #endif /* VMXNET3_RSS */
2229
2230 /* intr settings */
2231 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2232 VMXNET3_IMM_AUTO;
2233 devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2234 for (i = 0; i < adapter->intr.num_intrs; i++)
2235 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2236
2237 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2238 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2239
2240 /* rx filter settings */
2241 devRead->rxFilterConf.rxMode = 0;
2242 vmxnet3_restore_vlan(adapter);
2243 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2244
2245 /* the rest are already zeroed */
2246 }
2247
2248
2249 int
2250 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2251 {
2252 int err, i;
2253 u32 ret;
2254 unsigned long flags;
2255
2256 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2257 " ring sizes %u %u %u\n", adapter->netdev->name,
2258 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2259 adapter->tx_queue[0].tx_ring.size,
2260 adapter->rx_queue[0].rx_ring[0].size,
2261 adapter->rx_queue[0].rx_ring[1].size);
2262
2263 vmxnet3_tq_init_all(adapter);
2264 err = vmxnet3_rq_init_all(adapter);
2265 if (err) {
2266 netdev_err(adapter->netdev,
2267 "Failed to init rx queue error %d\n", err);
2268 goto rq_err;
2269 }
2270
2271 err = vmxnet3_request_irqs(adapter);
2272 if (err) {
2273 netdev_err(adapter->netdev,
2274 "Failed to setup irq for error %d\n", err);
2275 goto irq_err;
2276 }
2277
2278 vmxnet3_setup_driver_shared(adapter);
2279
2280 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2281 adapter->shared_pa));
2282 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2283 adapter->shared_pa));
2284 spin_lock_irqsave(&adapter->cmd_lock, flags);
2285 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2286 VMXNET3_CMD_ACTIVATE_DEV);
2287 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2288 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2289
2290 if (ret != 0) {
2291 netdev_err(adapter->netdev,
2292 "Failed to activate dev: error %u\n", ret);
2293 err = -EINVAL;
2294 goto activate_err;
2295 }
2296
2297 for (i = 0; i < adapter->num_rx_queues; i++) {
2298 VMXNET3_WRITE_BAR0_REG(adapter,
2299 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2300 adapter->rx_queue[i].rx_ring[0].next2fill);
2301 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2302 (i * VMXNET3_REG_ALIGN)),
2303 adapter->rx_queue[i].rx_ring[1].next2fill);
2304 }
2305
2306 /* Apply the rx filter settins last. */
2307 vmxnet3_set_mc(adapter->netdev);
2308
2309 /*
2310 * Check link state when first activating device. It will start the
2311 * tx queue if the link is up.
2312 */
2313 vmxnet3_check_link(adapter, true);
2314 for (i = 0; i < adapter->num_rx_queues; i++)
2315 napi_enable(&adapter->rx_queue[i].napi);
2316 vmxnet3_enable_all_intrs(adapter);
2317 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2318 return 0;
2319
2320 activate_err:
2321 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2322 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2323 vmxnet3_free_irqs(adapter);
2324 irq_err:
2325 rq_err:
2326 /* free up buffers we allocated */
2327 vmxnet3_rq_cleanup_all(adapter);
2328 return err;
2329 }
2330
2331
2332 void
2333 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2334 {
2335 unsigned long flags;
2336 spin_lock_irqsave(&adapter->cmd_lock, flags);
2337 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2338 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2339 }
2340
2341
2342 int
2343 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2344 {
2345 int i;
2346 unsigned long flags;
2347 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2348 return 0;
2349
2350
2351 spin_lock_irqsave(&adapter->cmd_lock, flags);
2352 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2353 VMXNET3_CMD_QUIESCE_DEV);
2354 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2355 vmxnet3_disable_all_intrs(adapter);
2356
2357 for (i = 0; i < adapter->num_rx_queues; i++)
2358 napi_disable(&adapter->rx_queue[i].napi);
2359 netif_tx_disable(adapter->netdev);
2360 adapter->link_speed = 0;
2361 netif_carrier_off(adapter->netdev);
2362
2363 vmxnet3_tq_cleanup_all(adapter);
2364 vmxnet3_rq_cleanup_all(adapter);
2365 vmxnet3_free_irqs(adapter);
2366 return 0;
2367 }
2368
2369
2370 static void
2371 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2372 {
2373 u32 tmp;
2374
2375 tmp = *(u32 *)mac;
2376 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2377
2378 tmp = (mac[5] << 8) | mac[4];
2379 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2380 }
2381
2382
2383 static int
2384 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2385 {
2386 struct sockaddr *addr = p;
2387 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2388
2389 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2390 vmxnet3_write_mac_addr(adapter, addr->sa_data);
2391
2392 return 0;
2393 }
2394
2395
2396 /* ==================== initialization and cleanup routines ============ */
2397
2398 static int
2399 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2400 {
2401 int err;
2402 unsigned long mmio_start, mmio_len;
2403 struct pci_dev *pdev = adapter->pdev;
2404
2405 err = pci_enable_device(pdev);
2406 if (err) {
2407 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2408 return err;
2409 }
2410
2411 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2412 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
2413 dev_err(&pdev->dev,
2414 "pci_set_consistent_dma_mask failed\n");
2415 err = -EIO;
2416 goto err_set_mask;
2417 }
2418 *dma64 = true;
2419 } else {
2420 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
2421 dev_err(&pdev->dev,
2422 "pci_set_dma_mask failed\n");
2423 err = -EIO;
2424 goto err_set_mask;
2425 }
2426 *dma64 = false;
2427 }
2428
2429 err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2430 vmxnet3_driver_name);
2431 if (err) {
2432 dev_err(&pdev->dev,
2433 "Failed to request region for adapter: error %d\n", err);
2434 goto err_set_mask;
2435 }
2436
2437 pci_set_master(pdev);
2438
2439 mmio_start = pci_resource_start(pdev, 0);
2440 mmio_len = pci_resource_len(pdev, 0);
2441 adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2442 if (!adapter->hw_addr0) {
2443 dev_err(&pdev->dev, "Failed to map bar0\n");
2444 err = -EIO;
2445 goto err_ioremap;
2446 }
2447
2448 mmio_start = pci_resource_start(pdev, 1);
2449 mmio_len = pci_resource_len(pdev, 1);
2450 adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2451 if (!adapter->hw_addr1) {
2452 dev_err(&pdev->dev, "Failed to map bar1\n");
2453 err = -EIO;
2454 goto err_bar1;
2455 }
2456 return 0;
2457
2458 err_bar1:
2459 iounmap(adapter->hw_addr0);
2460 err_ioremap:
2461 pci_release_selected_regions(pdev, (1 << 2) - 1);
2462 err_set_mask:
2463 pci_disable_device(pdev);
2464 return err;
2465 }
2466
2467
2468 static void
2469 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2470 {
2471 BUG_ON(!adapter->pdev);
2472
2473 iounmap(adapter->hw_addr0);
2474 iounmap(adapter->hw_addr1);
2475 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2476 pci_disable_device(adapter->pdev);
2477 }
2478
2479
2480 static void
2481 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2482 {
2483 size_t sz, i, ring0_size, ring1_size, comp_size;
2484 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2485
2486
2487 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2488 VMXNET3_MAX_ETH_HDR_SIZE) {
2489 adapter->skb_buf_size = adapter->netdev->mtu +
2490 VMXNET3_MAX_ETH_HDR_SIZE;
2491 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2492 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2493
2494 adapter->rx_buf_per_pkt = 1;
2495 } else {
2496 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2497 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2498 VMXNET3_MAX_ETH_HDR_SIZE;
2499 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2500 }
2501
2502 /*
2503 * for simplicity, force the ring0 size to be a multiple of
2504 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2505 */
2506 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2507 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2508 ring0_size = (ring0_size + sz - 1) / sz * sz;
2509 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
2510 sz * sz);
2511 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2512 comp_size = ring0_size + ring1_size;
2513
2514 for (i = 0; i < adapter->num_rx_queues; i++) {
2515 rq = &adapter->rx_queue[i];
2516 rq->rx_ring[0].size = ring0_size;
2517 rq->rx_ring[1].size = ring1_size;
2518 rq->comp_ring.size = comp_size;
2519 }
2520 }
2521
2522
2523 int
2524 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2525 u32 rx_ring_size, u32 rx_ring2_size)
2526 {
2527 int err = 0, i;
2528
2529 for (i = 0; i < adapter->num_tx_queues; i++) {
2530 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2531 tq->tx_ring.size = tx_ring_size;
2532 tq->data_ring.size = tx_ring_size;
2533 tq->comp_ring.size = tx_ring_size;
2534 tq->shared = &adapter->tqd_start[i].ctrl;
2535 tq->stopped = true;
2536 tq->adapter = adapter;
2537 tq->qid = i;
2538 err = vmxnet3_tq_create(tq, adapter);
2539 /*
2540 * Too late to change num_tx_queues. We cannot do away with
2541 * lesser number of queues than what we asked for
2542 */
2543 if (err)
2544 goto queue_err;
2545 }
2546
2547 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2548 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2549 vmxnet3_adjust_rx_ring_size(adapter);
2550 for (i = 0; i < adapter->num_rx_queues; i++) {
2551 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2552 /* qid and qid2 for rx queues will be assigned later when num
2553 * of rx queues is finalized after allocating intrs */
2554 rq->shared = &adapter->rqd_start[i].ctrl;
2555 rq->adapter = adapter;
2556 err = vmxnet3_rq_create(rq, adapter);
2557 if (err) {
2558 if (i == 0) {
2559 netdev_err(adapter->netdev,
2560 "Could not allocate any rx queues. "
2561 "Aborting.\n");
2562 goto queue_err;
2563 } else {
2564 netdev_info(adapter->netdev,
2565 "Number of rx queues changed "
2566 "to : %d.\n", i);
2567 adapter->num_rx_queues = i;
2568 err = 0;
2569 break;
2570 }
2571 }
2572 }
2573 return err;
2574 queue_err:
2575 vmxnet3_tq_destroy_all(adapter);
2576 return err;
2577 }
2578
2579 static int
2580 vmxnet3_open(struct net_device *netdev)
2581 {
2582 struct vmxnet3_adapter *adapter;
2583 int err, i;
2584
2585 adapter = netdev_priv(netdev);
2586
2587 for (i = 0; i < adapter->num_tx_queues; i++)
2588 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2589
2590 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2591 VMXNET3_DEF_RX_RING_SIZE,
2592 VMXNET3_DEF_RX_RING_SIZE);
2593 if (err)
2594 goto queue_err;
2595
2596 err = vmxnet3_activate_dev(adapter);
2597 if (err)
2598 goto activate_err;
2599
2600 return 0;
2601
2602 activate_err:
2603 vmxnet3_rq_destroy_all(adapter);
2604 vmxnet3_tq_destroy_all(adapter);
2605 queue_err:
2606 return err;
2607 }
2608
2609
2610 static int
2611 vmxnet3_close(struct net_device *netdev)
2612 {
2613 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2614
2615 /*
2616 * Reset_work may be in the middle of resetting the device, wait for its
2617 * completion.
2618 */
2619 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2620 msleep(1);
2621
2622 vmxnet3_quiesce_dev(adapter);
2623
2624 vmxnet3_rq_destroy_all(adapter);
2625 vmxnet3_tq_destroy_all(adapter);
2626
2627 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2628
2629
2630 return 0;
2631 }
2632
2633
2634 void
2635 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2636 {
2637 int i;
2638
2639 /*
2640 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2641 * vmxnet3_close() will deadlock.
2642 */
2643 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2644
2645 /* we need to enable NAPI, otherwise dev_close will deadlock */
2646 for (i = 0; i < adapter->num_rx_queues; i++)
2647 napi_enable(&adapter->rx_queue[i].napi);
2648 dev_close(adapter->netdev);
2649 }
2650
2651
2652 static int
2653 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2654 {
2655 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2656 int err = 0;
2657
2658 if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2659 return -EINVAL;
2660
2661 netdev->mtu = new_mtu;
2662
2663 /*
2664 * Reset_work may be in the middle of resetting the device, wait for its
2665 * completion.
2666 */
2667 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2668 msleep(1);
2669
2670 if (netif_running(netdev)) {
2671 vmxnet3_quiesce_dev(adapter);
2672 vmxnet3_reset_dev(adapter);
2673
2674 /* we need to re-create the rx queue based on the new mtu */
2675 vmxnet3_rq_destroy_all(adapter);
2676 vmxnet3_adjust_rx_ring_size(adapter);
2677 err = vmxnet3_rq_create_all(adapter);
2678 if (err) {
2679 netdev_err(netdev,
2680 "failed to re-create rx queues, "
2681 " error %d. Closing it.\n", err);
2682 goto out;
2683 }
2684
2685 err = vmxnet3_activate_dev(adapter);
2686 if (err) {
2687 netdev_err(netdev,
2688 "failed to re-activate, error %d. "
2689 "Closing it\n", err);
2690 goto out;
2691 }
2692 }
2693
2694 out:
2695 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2696 if (err)
2697 vmxnet3_force_close(adapter);
2698
2699 return err;
2700 }
2701
2702
2703 static void
2704 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2705 {
2706 struct net_device *netdev = adapter->netdev;
2707
2708 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2709 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2710 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2711 NETIF_F_LRO;
2712 if (dma64)
2713 netdev->hw_features |= NETIF_F_HIGHDMA;
2714 netdev->vlan_features = netdev->hw_features &
2715 ~(NETIF_F_HW_VLAN_CTAG_TX |
2716 NETIF_F_HW_VLAN_CTAG_RX);
2717 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
2718 }
2719
2720
2721 static void
2722 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2723 {
2724 u32 tmp;
2725
2726 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2727 *(u32 *)mac = tmp;
2728
2729 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2730 mac[4] = tmp & 0xff;
2731 mac[5] = (tmp >> 8) & 0xff;
2732 }
2733
2734 #ifdef CONFIG_PCI_MSI
2735
2736 /*
2737 * Enable MSIx vectors.
2738 * Returns :
2739 * 0 on successful enabling of required vectors,
2740 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
2741 * could be enabled.
2742 * number of vectors which can be enabled otherwise (this number is smaller
2743 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2744 */
2745
2746 static int
2747 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
2748 int vectors)
2749 {
2750 int err = 0, vector_threshold;
2751 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
2752
2753 while (vectors >= vector_threshold) {
2754 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2755 vectors);
2756 if (!err) {
2757 adapter->intr.num_intrs = vectors;
2758 return 0;
2759 } else if (err < 0) {
2760 dev_err(&adapter->netdev->dev,
2761 "Failed to enable MSI-X, error: %d\n", err);
2762 vectors = 0;
2763 } else if (err < vector_threshold) {
2764 break;
2765 } else {
2766 /* If fails to enable required number of MSI-x vectors
2767 * try enabling minimum number of vectors required.
2768 */
2769 dev_err(&adapter->netdev->dev,
2770 "Failed to enable %d MSI-X, trying %d instead\n",
2771 vectors, vector_threshold);
2772 vectors = vector_threshold;
2773 }
2774 }
2775
2776 dev_info(&adapter->pdev->dev,
2777 "Number of MSI-X interrupts which can be allocated "
2778 "is lower than min threshold required.\n");
2779 return err;
2780 }
2781
2782
2783 #endif /* CONFIG_PCI_MSI */
2784
2785 static void
2786 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2787 {
2788 u32 cfg;
2789 unsigned long flags;
2790
2791 /* intr settings */
2792 spin_lock_irqsave(&adapter->cmd_lock, flags);
2793 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2794 VMXNET3_CMD_GET_CONF_INTR);
2795 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2796 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2797 adapter->intr.type = cfg & 0x3;
2798 adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2799
2800 if (adapter->intr.type == VMXNET3_IT_AUTO) {
2801 adapter->intr.type = VMXNET3_IT_MSIX;
2802 }
2803
2804 #ifdef CONFIG_PCI_MSI
2805 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2806 int vector, err = 0;
2807
2808 adapter->intr.num_intrs = (adapter->share_intr ==
2809 VMXNET3_INTR_TXSHARE) ? 1 :
2810 adapter->num_tx_queues;
2811 adapter->intr.num_intrs += (adapter->share_intr ==
2812 VMXNET3_INTR_BUDDYSHARE) ? 0 :
2813 adapter->num_rx_queues;
2814 adapter->intr.num_intrs += 1; /* for link event */
2815
2816 adapter->intr.num_intrs = (adapter->intr.num_intrs >
2817 VMXNET3_LINUX_MIN_MSIX_VECT
2818 ? adapter->intr.num_intrs :
2819 VMXNET3_LINUX_MIN_MSIX_VECT);
2820
2821 for (vector = 0; vector < adapter->intr.num_intrs; vector++)
2822 adapter->intr.msix_entries[vector].entry = vector;
2823
2824 err = vmxnet3_acquire_msix_vectors(adapter,
2825 adapter->intr.num_intrs);
2826 /* If we cannot allocate one MSIx vector per queue
2827 * then limit the number of rx queues to 1
2828 */
2829 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
2830 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2831 || adapter->num_rx_queues != 1) {
2832 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2833 netdev_err(adapter->netdev,
2834 "Number of rx queues : 1\n");
2835 adapter->num_rx_queues = 1;
2836 adapter->intr.num_intrs =
2837 VMXNET3_LINUX_MIN_MSIX_VECT;
2838 }
2839 return;
2840 }
2841 if (!err)
2842 return;
2843
2844 /* If we cannot allocate MSIx vectors use only one rx queue */
2845 dev_info(&adapter->pdev->dev,
2846 "Failed to enable MSI-X, error %d. "
2847 "Limiting #rx queues to 1, try MSI.\n", err);
2848
2849 adapter->intr.type = VMXNET3_IT_MSI;
2850 }
2851
2852 if (adapter->intr.type == VMXNET3_IT_MSI) {
2853 int err;
2854 err = pci_enable_msi(adapter->pdev);
2855 if (!err) {
2856 adapter->num_rx_queues = 1;
2857 adapter->intr.num_intrs = 1;
2858 return;
2859 }
2860 }
2861 #endif /* CONFIG_PCI_MSI */
2862
2863 adapter->num_rx_queues = 1;
2864 dev_info(&adapter->netdev->dev,
2865 "Using INTx interrupt, #Rx queues: 1.\n");
2866 adapter->intr.type = VMXNET3_IT_INTX;
2867
2868 /* INT-X related setting */
2869 adapter->intr.num_intrs = 1;
2870 }
2871
2872
2873 static void
2874 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2875 {
2876 if (adapter->intr.type == VMXNET3_IT_MSIX)
2877 pci_disable_msix(adapter->pdev);
2878 else if (adapter->intr.type == VMXNET3_IT_MSI)
2879 pci_disable_msi(adapter->pdev);
2880 else
2881 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2882 }
2883
2884
2885 static void
2886 vmxnet3_tx_timeout(struct net_device *netdev)
2887 {
2888 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2889 adapter->tx_timeout_count++;
2890
2891 netdev_err(adapter->netdev, "tx hang\n");
2892 schedule_work(&adapter->work);
2893 netif_wake_queue(adapter->netdev);
2894 }
2895
2896
2897 static void
2898 vmxnet3_reset_work(struct work_struct *data)
2899 {
2900 struct vmxnet3_adapter *adapter;
2901
2902 adapter = container_of(data, struct vmxnet3_adapter, work);
2903
2904 /* if another thread is resetting the device, no need to proceed */
2905 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2906 return;
2907
2908 /* if the device is closed, we must leave it alone */
2909 rtnl_lock();
2910 if (netif_running(adapter->netdev)) {
2911 netdev_notice(adapter->netdev, "resetting\n");
2912 vmxnet3_quiesce_dev(adapter);
2913 vmxnet3_reset_dev(adapter);
2914 vmxnet3_activate_dev(adapter);
2915 } else {
2916 netdev_info(adapter->netdev, "already closed\n");
2917 }
2918 rtnl_unlock();
2919
2920 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2921 }
2922
2923
2924 static int
2925 vmxnet3_probe_device(struct pci_dev *pdev,
2926 const struct pci_device_id *id)
2927 {
2928 static const struct net_device_ops vmxnet3_netdev_ops = {
2929 .ndo_open = vmxnet3_open,
2930 .ndo_stop = vmxnet3_close,
2931 .ndo_start_xmit = vmxnet3_xmit_frame,
2932 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2933 .ndo_change_mtu = vmxnet3_change_mtu,
2934 .ndo_set_features = vmxnet3_set_features,
2935 .ndo_get_stats64 = vmxnet3_get_stats64,
2936 .ndo_tx_timeout = vmxnet3_tx_timeout,
2937 .ndo_set_rx_mode = vmxnet3_set_mc,
2938 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2939 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2940 #ifdef CONFIG_NET_POLL_CONTROLLER
2941 .ndo_poll_controller = vmxnet3_netpoll,
2942 #endif
2943 };
2944 int err;
2945 bool dma64 = false; /* stupid gcc */
2946 u32 ver;
2947 struct net_device *netdev;
2948 struct vmxnet3_adapter *adapter;
2949 u8 mac[ETH_ALEN];
2950 int size;
2951 int num_tx_queues;
2952 int num_rx_queues;
2953
2954 if (!pci_msi_enabled())
2955 enable_mq = 0;
2956
2957 #ifdef VMXNET3_RSS
2958 if (enable_mq)
2959 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2960 (int)num_online_cpus());
2961 else
2962 #endif
2963 num_rx_queues = 1;
2964 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
2965
2966 if (enable_mq)
2967 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2968 (int)num_online_cpus());
2969 else
2970 num_tx_queues = 1;
2971
2972 num_tx_queues = rounddown_pow_of_two(num_tx_queues);
2973 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2974 max(num_tx_queues, num_rx_queues));
2975 dev_info(&pdev->dev,
2976 "# of Tx queues : %d, # of Rx queues : %d\n",
2977 num_tx_queues, num_rx_queues);
2978
2979 if (!netdev)
2980 return -ENOMEM;
2981
2982 pci_set_drvdata(pdev, netdev);
2983 adapter = netdev_priv(netdev);
2984 adapter->netdev = netdev;
2985 adapter->pdev = pdev;
2986
2987 spin_lock_init(&adapter->cmd_lock);
2988 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
2989 sizeof(struct vmxnet3_adapter),
2990 PCI_DMA_TODEVICE);
2991 adapter->shared = dma_alloc_coherent(
2992 &adapter->pdev->dev,
2993 sizeof(struct Vmxnet3_DriverShared),
2994 &adapter->shared_pa, GFP_KERNEL);
2995 if (!adapter->shared) {
2996 dev_err(&pdev->dev, "Failed to allocate memory\n");
2997 err = -ENOMEM;
2998 goto err_alloc_shared;
2999 }
3000
3001 adapter->num_rx_queues = num_rx_queues;
3002 adapter->num_tx_queues = num_tx_queues;
3003 adapter->rx_buf_per_pkt = 1;
3004
3005 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3006 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
3007 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
3008 &adapter->queue_desc_pa,
3009 GFP_KERNEL);
3010
3011 if (!adapter->tqd_start) {
3012 dev_err(&pdev->dev, "Failed to allocate memory\n");
3013 err = -ENOMEM;
3014 goto err_alloc_queue_desc;
3015 }
3016 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3017 adapter->num_tx_queues);
3018
3019 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3020 sizeof(struct Vmxnet3_PMConf),
3021 &adapter->pm_conf_pa,
3022 GFP_KERNEL);
3023 if (adapter->pm_conf == NULL) {
3024 err = -ENOMEM;
3025 goto err_alloc_pm;
3026 }
3027
3028 #ifdef VMXNET3_RSS
3029
3030 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3031 sizeof(struct UPT1_RSSConf),
3032 &adapter->rss_conf_pa,
3033 GFP_KERNEL);
3034 if (adapter->rss_conf == NULL) {
3035 err = -ENOMEM;
3036 goto err_alloc_rss;
3037 }
3038 #endif /* VMXNET3_RSS */
3039
3040 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
3041 if (err < 0)
3042 goto err_alloc_pci;
3043
3044 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3045 if (ver & 1) {
3046 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
3047 } else {
3048 dev_err(&pdev->dev,
3049 "Incompatible h/w version (0x%x) for adapter\n", ver);
3050 err = -EBUSY;
3051 goto err_ver;
3052 }
3053
3054 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3055 if (ver & 1) {
3056 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3057 } else {
3058 dev_err(&pdev->dev,
3059 "Incompatible upt version (0x%x) for adapter\n", ver);
3060 err = -EBUSY;
3061 goto err_ver;
3062 }
3063
3064 SET_NETDEV_DEV(netdev, &pdev->dev);
3065 vmxnet3_declare_features(adapter, dma64);
3066
3067 if (adapter->num_tx_queues == adapter->num_rx_queues)
3068 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3069 else
3070 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3071
3072 vmxnet3_alloc_intr_resources(adapter);
3073
3074 #ifdef VMXNET3_RSS
3075 if (adapter->num_rx_queues > 1 &&
3076 adapter->intr.type == VMXNET3_IT_MSIX) {
3077 adapter->rss = true;
3078 netdev->hw_features |= NETIF_F_RXHASH;
3079 netdev->features |= NETIF_F_RXHASH;
3080 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3081 } else {
3082 adapter->rss = false;
3083 }
3084 #endif
3085
3086 vmxnet3_read_mac_addr(adapter, mac);
3087 memcpy(netdev->dev_addr, mac, netdev->addr_len);
3088
3089 netdev->netdev_ops = &vmxnet3_netdev_ops;
3090 vmxnet3_set_ethtool_ops(netdev);
3091 netdev->watchdog_timeo = 5 * HZ;
3092
3093 INIT_WORK(&adapter->work, vmxnet3_reset_work);
3094 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3095
3096 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3097 int i;
3098 for (i = 0; i < adapter->num_rx_queues; i++) {
3099 netif_napi_add(adapter->netdev,
3100 &adapter->rx_queue[i].napi,
3101 vmxnet3_poll_rx_only, 64);
3102 }
3103 } else {
3104 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3105 vmxnet3_poll, 64);
3106 }
3107
3108 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3109 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3110
3111 netif_carrier_off(netdev);
3112 err = register_netdev(netdev);
3113
3114 if (err) {
3115 dev_err(&pdev->dev, "Failed to register adapter\n");
3116 goto err_register;
3117 }
3118
3119 vmxnet3_check_link(adapter, false);
3120 return 0;
3121
3122 err_register:
3123 vmxnet3_free_intr_resources(adapter);
3124 err_ver:
3125 vmxnet3_free_pci_resources(adapter);
3126 err_alloc_pci:
3127 #ifdef VMXNET3_RSS
3128 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3129 adapter->rss_conf, adapter->rss_conf_pa);
3130 err_alloc_rss:
3131 #endif
3132 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3133 adapter->pm_conf, adapter->pm_conf_pa);
3134 err_alloc_pm:
3135 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3136 adapter->queue_desc_pa);
3137 err_alloc_queue_desc:
3138 dma_free_coherent(&adapter->pdev->dev,
3139 sizeof(struct Vmxnet3_DriverShared),
3140 adapter->shared, adapter->shared_pa);
3141 err_alloc_shared:
3142 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3143 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3144 pci_set_drvdata(pdev, NULL);
3145 free_netdev(netdev);
3146 return err;
3147 }
3148
3149
3150 static void
3151 vmxnet3_remove_device(struct pci_dev *pdev)
3152 {
3153 struct net_device *netdev = pci_get_drvdata(pdev);
3154 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3155 int size = 0;
3156 int num_rx_queues;
3157
3158 #ifdef VMXNET3_RSS
3159 if (enable_mq)
3160 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3161 (int)num_online_cpus());
3162 else
3163 #endif
3164 num_rx_queues = 1;
3165 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3166
3167 cancel_work_sync(&adapter->work);
3168
3169 unregister_netdev(netdev);
3170
3171 vmxnet3_free_intr_resources(adapter);
3172 vmxnet3_free_pci_resources(adapter);
3173 #ifdef VMXNET3_RSS
3174 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3175 adapter->rss_conf, adapter->rss_conf_pa);
3176 #endif
3177 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3178 adapter->pm_conf, adapter->pm_conf_pa);
3179
3180 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3181 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3182 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3183 adapter->queue_desc_pa);
3184 dma_free_coherent(&adapter->pdev->dev,
3185 sizeof(struct Vmxnet3_DriverShared),
3186 adapter->shared, adapter->shared_pa);
3187 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3188 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3189 free_netdev(netdev);
3190 }
3191
3192
3193 #ifdef CONFIG_PM
3194
3195 static int
3196 vmxnet3_suspend(struct device *device)
3197 {
3198 struct pci_dev *pdev = to_pci_dev(device);
3199 struct net_device *netdev = pci_get_drvdata(pdev);
3200 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3201 struct Vmxnet3_PMConf *pmConf;
3202 struct ethhdr *ehdr;
3203 struct arphdr *ahdr;
3204 u8 *arpreq;
3205 struct in_device *in_dev;
3206 struct in_ifaddr *ifa;
3207 unsigned long flags;
3208 int i = 0;
3209
3210 if (!netif_running(netdev))
3211 return 0;
3212
3213 for (i = 0; i < adapter->num_rx_queues; i++)
3214 napi_disable(&adapter->rx_queue[i].napi);
3215
3216 vmxnet3_disable_all_intrs(adapter);
3217 vmxnet3_free_irqs(adapter);
3218 vmxnet3_free_intr_resources(adapter);
3219
3220 netif_device_detach(netdev);
3221 netif_tx_stop_all_queues(netdev);
3222
3223 /* Create wake-up filters. */
3224 pmConf = adapter->pm_conf;
3225 memset(pmConf, 0, sizeof(*pmConf));
3226
3227 if (adapter->wol & WAKE_UCAST) {
3228 pmConf->filters[i].patternSize = ETH_ALEN;
3229 pmConf->filters[i].maskSize = 1;
3230 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3231 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3232
3233 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3234 i++;
3235 }
3236
3237 if (adapter->wol & WAKE_ARP) {
3238 in_dev = in_dev_get(netdev);
3239 if (!in_dev)
3240 goto skip_arp;
3241
3242 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3243 if (!ifa)
3244 goto skip_arp;
3245
3246 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3247 sizeof(struct arphdr) + /* ARP header */
3248 2 * ETH_ALEN + /* 2 Ethernet addresses*/
3249 2 * sizeof(u32); /*2 IPv4 addresses */
3250 pmConf->filters[i].maskSize =
3251 (pmConf->filters[i].patternSize - 1) / 8 + 1;
3252
3253 /* ETH_P_ARP in Ethernet header. */
3254 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3255 ehdr->h_proto = htons(ETH_P_ARP);
3256
3257 /* ARPOP_REQUEST in ARP header. */
3258 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3259 ahdr->ar_op = htons(ARPOP_REQUEST);
3260 arpreq = (u8 *)(ahdr + 1);
3261
3262 /* The Unicast IPv4 address in 'tip' field. */
3263 arpreq += 2 * ETH_ALEN + sizeof(u32);
3264 *(u32 *)arpreq = ifa->ifa_address;
3265
3266 /* The mask for the relevant bits. */
3267 pmConf->filters[i].mask[0] = 0x00;
3268 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3269 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3270 pmConf->filters[i].mask[3] = 0x00;
3271 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3272 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3273 in_dev_put(in_dev);
3274
3275 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3276 i++;
3277 }
3278
3279 skip_arp:
3280 if (adapter->wol & WAKE_MAGIC)
3281 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
3282
3283 pmConf->numFilters = i;
3284
3285 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3286 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3287 *pmConf));
3288 adapter->shared->devRead.pmConfDesc.confPA =
3289 cpu_to_le64(adapter->pm_conf_pa);
3290
3291 spin_lock_irqsave(&adapter->cmd_lock, flags);
3292 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3293 VMXNET3_CMD_UPDATE_PMCFG);
3294 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3295
3296 pci_save_state(pdev);
3297 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3298 adapter->wol);
3299 pci_disable_device(pdev);
3300 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3301
3302 return 0;
3303 }
3304
3305
3306 static int
3307 vmxnet3_resume(struct device *device)
3308 {
3309 int err, i = 0;
3310 unsigned long flags;
3311 struct pci_dev *pdev = to_pci_dev(device);
3312 struct net_device *netdev = pci_get_drvdata(pdev);
3313 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3314 struct Vmxnet3_PMConf *pmConf;
3315
3316 if (!netif_running(netdev))
3317 return 0;
3318
3319 /* Destroy wake-up filters. */
3320 pmConf = adapter->pm_conf;
3321 memset(pmConf, 0, sizeof(*pmConf));
3322
3323 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3324 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3325 *pmConf));
3326 adapter->shared->devRead.pmConfDesc.confPA =
3327 cpu_to_le64(adapter->pm_conf_pa);
3328
3329 netif_device_attach(netdev);
3330 pci_set_power_state(pdev, PCI_D0);
3331 pci_restore_state(pdev);
3332 err = pci_enable_device_mem(pdev);
3333 if (err != 0)
3334 return err;
3335
3336 pci_enable_wake(pdev, PCI_D0, 0);
3337
3338 spin_lock_irqsave(&adapter->cmd_lock, flags);
3339 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3340 VMXNET3_CMD_UPDATE_PMCFG);
3341 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3342 vmxnet3_alloc_intr_resources(adapter);
3343 vmxnet3_request_irqs(adapter);
3344 for (i = 0; i < adapter->num_rx_queues; i++)
3345 napi_enable(&adapter->rx_queue[i].napi);
3346 vmxnet3_enable_all_intrs(adapter);
3347
3348 return 0;
3349 }
3350
3351 static const struct dev_pm_ops vmxnet3_pm_ops = {
3352 .suspend = vmxnet3_suspend,
3353 .resume = vmxnet3_resume,
3354 };
3355 #endif
3356
3357 static struct pci_driver vmxnet3_driver = {
3358 .name = vmxnet3_driver_name,
3359 .id_table = vmxnet3_pciid_table,
3360 .probe = vmxnet3_probe_device,
3361 .remove = vmxnet3_remove_device,
3362 #ifdef CONFIG_PM
3363 .driver.pm = &vmxnet3_pm_ops,
3364 #endif
3365 };
3366
3367
3368 static int __init
3369 vmxnet3_init_module(void)
3370 {
3371 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
3372 VMXNET3_DRIVER_VERSION_REPORT);
3373 return pci_register_driver(&vmxnet3_driver);
3374 }
3375
3376 module_init(vmxnet3_init_module);
3377
3378
3379 static void
3380 vmxnet3_exit_module(void)
3381 {
3382 pci_unregister_driver(&vmxnet3_driver);
3383 }
3384
3385 module_exit(vmxnet3_exit_module);
3386
3387 MODULE_AUTHOR("VMware, Inc.");
3388 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3389 MODULE_LICENSE("GPL v2");
3390 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
Linux with added FPC-III support