?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ARM: mm: Recreate kernel mappings in early_paging_init()
[linux/fpc-iii.git] / drivers / net / vmxnet3 / vmxnet3_drv.c
blob7e2788c488edce4536fa0039337b5b267d2dc270
1 /*
2 * Linux driver for VMware's vmxnet3 ethernet NIC.
3 *
4 * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License as published by the
8 * Free Software Foundation; version 2 of the License and no later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more
14 * details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * The full GNU General Public License is included in this distribution in
21 * the file called "COPYING".
22 *
23 * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
24 *
25 */
26
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
29
30 #include "vmxnet3_int.h"
31
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
34
35 /*
36 * PCI Device ID Table
37 * Last entry must be all 0s
38 */
39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
41 {0}
42 };
43
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
45
46 static int enable_mq = 1;
47
48 static void
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
50
51 /*
52 * Enable/Disable the given intr
53 */
54 static void
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
56 {
57 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
58 }
59
60
61 static void
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
63 {
64 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
65 }
66
67
68 /*
69 * Enable/Disable all intrs used by the device
70 */
71 static void
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
73 {
74 int i;
75
76 for (i = 0; i < adapter->intr.num_intrs; i++)
77 vmxnet3_enable_intr(adapter, i);
78 adapter->shared->devRead.intrConf.intrCtrl &=
79 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
80 }
81
82
83 static void
84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
85 {
86 int i;
87
88 adapter->shared->devRead.intrConf.intrCtrl |=
89 cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
90 for (i = 0; i < adapter->intr.num_intrs; i++)
91 vmxnet3_disable_intr(adapter, i);
92 }
93
94
95 static void
96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
97 {
98 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
99 }
100
101
102 static bool
103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
104 {
105 return tq->stopped;
106 }
107
108
109 static void
110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111 {
112 tq->stopped = false;
113 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
114 }
115
116
117 static void
118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119 {
120 tq->stopped = false;
121 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
122 }
123
124
125 static void
126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
127 {
128 tq->stopped = true;
129 tq->num_stop++;
130 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
131 }
132
133
134 /*
135 * Check the link state. This may start or stop the tx queue.
136 */
137 static void
138 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
139 {
140 u32 ret;
141 int i;
142 unsigned long flags;
143
144 spin_lock_irqsave(&adapter->cmd_lock, flags);
145 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
146 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
147 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
148
149 adapter->link_speed = ret >> 16;
150 if (ret & 1) { /* Link is up. */
151 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
152 adapter->link_speed);
153 netif_carrier_on(adapter->netdev);
154
155 if (affectTxQueue) {
156 for (i = 0; i < adapter->num_tx_queues; i++)
157 vmxnet3_tq_start(&adapter->tx_queue[i],
158 adapter);
159 }
160 } else {
161 netdev_info(adapter->netdev, "NIC Link is Down\n");
162 netif_carrier_off(adapter->netdev);
163
164 if (affectTxQueue) {
165 for (i = 0; i < adapter->num_tx_queues; i++)
166 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
167 }
168 }
169 }
170
171 static void
172 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
173 {
174 int i;
175 unsigned long flags;
176 u32 events = le32_to_cpu(adapter->shared->ecr);
177 if (!events)
178 return;
179
180 vmxnet3_ack_events(adapter, events);
181
182 /* Check if link state has changed */
183 if (events & VMXNET3_ECR_LINK)
184 vmxnet3_check_link(adapter, true);
185
186 /* Check if there is an error on xmit/recv queues */
187 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
188 spin_lock_irqsave(&adapter->cmd_lock, flags);
189 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
190 VMXNET3_CMD_GET_QUEUE_STATUS);
191 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
192
193 for (i = 0; i < adapter->num_tx_queues; i++)
194 if (adapter->tqd_start[i].status.stopped)
195 dev_err(&adapter->netdev->dev,
196 "%s: tq[%d] error 0x%x\n",
197 adapter->netdev->name, i, le32_to_cpu(
198 adapter->tqd_start[i].status.error));
199 for (i = 0; i < adapter->num_rx_queues; i++)
200 if (adapter->rqd_start[i].status.stopped)
201 dev_err(&adapter->netdev->dev,
202 "%s: rq[%d] error 0x%x\n",
203 adapter->netdev->name, i,
204 adapter->rqd_start[i].status.error);
205
206 schedule_work(&adapter->work);
207 }
208 }
209
210 #ifdef __BIG_ENDIAN_BITFIELD
211 /*
212 * The device expects the bitfields in shared structures to be written in
213 * little endian. When CPU is big endian, the following routines are used to
214 * correctly read and write into ABI.
215 * The general technique used here is : double word bitfields are defined in
216 * opposite order for big endian architecture. Then before reading them in
217 * driver the complete double word is translated using le32_to_cpu. Similarly
218 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
219 * double words into required format.
220 * In order to avoid touching bits in shared structure more than once, temporary
221 * descriptors are used. These are passed as srcDesc to following functions.
222 */
223 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
224 struct Vmxnet3_RxDesc *dstDesc)
225 {
226 u32 *src = (u32 *)srcDesc + 2;
227 u32 *dst = (u32 *)dstDesc + 2;
228 dstDesc->addr = le64_to_cpu(srcDesc->addr);
229 *dst = le32_to_cpu(*src);
230 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
231 }
232
233 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
234 struct Vmxnet3_TxDesc *dstDesc)
235 {
236 int i;
237 u32 *src = (u32 *)(srcDesc + 1);
238 u32 *dst = (u32 *)(dstDesc + 1);
239
240 /* Working backwards so that the gen bit is set at the end. */
241 for (i = 2; i > 0; i--) {
242 src--;
243 dst--;
244 *dst = cpu_to_le32(*src);
245 }
246 }
247
248
249 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
250 struct Vmxnet3_RxCompDesc *dstDesc)
251 {
252 int i = 0;
253 u32 *src = (u32 *)srcDesc;
254 u32 *dst = (u32 *)dstDesc;
255 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
256 *dst = le32_to_cpu(*src);
257 src++;
258 dst++;
259 }
260 }
261
262
263 /* Used to read bitfield values from double words. */
264 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
265 {
266 u32 temp = le32_to_cpu(*bitfield);
267 u32 mask = ((1 << size) - 1) << pos;
268 temp &= mask;
269 temp >>= pos;
270 return temp;
271 }
272
273
274
275 #endif /* __BIG_ENDIAN_BITFIELD */
276
277 #ifdef __BIG_ENDIAN_BITFIELD
278
279 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
280 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
281 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
282 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
283 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
284 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
285 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
286 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
287 VMXNET3_TCD_GEN_SIZE)
288 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
289 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
290 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
291 (dstrcd) = (tmp); \
292 vmxnet3_RxCompToCPU((rcd), (tmp)); \
293 } while (0)
294 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
295 (dstrxd) = (tmp); \
296 vmxnet3_RxDescToCPU((rxd), (tmp)); \
297 } while (0)
298
299 #else
300
301 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
302 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
303 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
304 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
305 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
306 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
307
308 #endif /* __BIG_ENDIAN_BITFIELD */
309
310
311 static void
312 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
313 struct pci_dev *pdev)
314 {
315 if (tbi->map_type == VMXNET3_MAP_SINGLE)
316 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
317 PCI_DMA_TODEVICE);
318 else if (tbi->map_type == VMXNET3_MAP_PAGE)
319 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
320 PCI_DMA_TODEVICE);
321 else
322 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
323
324 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
325 }
326
327
328 static int
329 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
330 struct pci_dev *pdev, struct vmxnet3_adapter *adapter)
331 {
332 struct sk_buff *skb;
333 int entries = 0;
334
335 /* no out of order completion */
336 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
337 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
338
339 skb = tq->buf_info[eop_idx].skb;
340 BUG_ON(skb == NULL);
341 tq->buf_info[eop_idx].skb = NULL;
342
343 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
344
345 while (tq->tx_ring.next2comp != eop_idx) {
346 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
347 pdev);
348
349 /* update next2comp w/o tx_lock. Since we are marking more,
350 * instead of less, tx ring entries avail, the worst case is
351 * that the tx routine incorrectly re-queues a pkt due to
352 * insufficient tx ring entries.
353 */
354 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
355 entries++;
356 }
357
358 dev_kfree_skb_any(skb);
359 return entries;
360 }
361
362
363 static int
364 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
365 struct vmxnet3_adapter *adapter)
366 {
367 int completed = 0;
368 union Vmxnet3_GenericDesc *gdesc;
369
370 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
371 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
372 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
373 &gdesc->tcd), tq, adapter->pdev,
374 adapter);
375
376 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
377 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
378 }
379
380 if (completed) {
381 spin_lock(&tq->tx_lock);
382 if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
383 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
384 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
385 netif_carrier_ok(adapter->netdev))) {
386 vmxnet3_tq_wake(tq, adapter);
387 }
388 spin_unlock(&tq->tx_lock);
389 }
390 return completed;
391 }
392
393
394 static void
395 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
396 struct vmxnet3_adapter *adapter)
397 {
398 int i;
399
400 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
401 struct vmxnet3_tx_buf_info *tbi;
402
403 tbi = tq->buf_info + tq->tx_ring.next2comp;
404
405 vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
406 if (tbi->skb) {
407 dev_kfree_skb_any(tbi->skb);
408 tbi->skb = NULL;
409 }
410 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
411 }
412
413 /* sanity check, verify all buffers are indeed unmapped and freed */
414 for (i = 0; i < tq->tx_ring.size; i++) {
415 BUG_ON(tq->buf_info[i].skb != NULL ||
416 tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
417 }
418
419 tq->tx_ring.gen = VMXNET3_INIT_GEN;
420 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
421
422 tq->comp_ring.gen = VMXNET3_INIT_GEN;
423 tq->comp_ring.next2proc = 0;
424 }
425
426
427 static void
428 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
429 struct vmxnet3_adapter *adapter)
430 {
431 if (tq->tx_ring.base) {
432 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
433 sizeof(struct Vmxnet3_TxDesc),
434 tq->tx_ring.base, tq->tx_ring.basePA);
435 tq->tx_ring.base = NULL;
436 }
437 if (tq->data_ring.base) {
438 dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
439 sizeof(struct Vmxnet3_TxDataDesc),
440 tq->data_ring.base, tq->data_ring.basePA);
441 tq->data_ring.base = NULL;
442 }
443 if (tq->comp_ring.base) {
444 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
445 sizeof(struct Vmxnet3_TxCompDesc),
446 tq->comp_ring.base, tq->comp_ring.basePA);
447 tq->comp_ring.base = NULL;
448 }
449 if (tq->buf_info) {
450 dma_free_coherent(&adapter->pdev->dev,
451 tq->tx_ring.size * sizeof(tq->buf_info[0]),
452 tq->buf_info, tq->buf_info_pa);
453 tq->buf_info = NULL;
454 }
455 }
456
457
458 /* Destroy all tx queues */
459 void
460 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
461 {
462 int i;
463
464 for (i = 0; i < adapter->num_tx_queues; i++)
465 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
466 }
467
468
469 static void
470 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
471 struct vmxnet3_adapter *adapter)
472 {
473 int i;
474
475 /* reset the tx ring contents to 0 and reset the tx ring states */
476 memset(tq->tx_ring.base, 0, tq->tx_ring.size *
477 sizeof(struct Vmxnet3_TxDesc));
478 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
479 tq->tx_ring.gen = VMXNET3_INIT_GEN;
480
481 memset(tq->data_ring.base, 0, tq->data_ring.size *
482 sizeof(struct Vmxnet3_TxDataDesc));
483
484 /* reset the tx comp ring contents to 0 and reset comp ring states */
485 memset(tq->comp_ring.base, 0, tq->comp_ring.size *
486 sizeof(struct Vmxnet3_TxCompDesc));
487 tq->comp_ring.next2proc = 0;
488 tq->comp_ring.gen = VMXNET3_INIT_GEN;
489
490 /* reset the bookkeeping data */
491 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
492 for (i = 0; i < tq->tx_ring.size; i++)
493 tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
494
495 /* stats are not reset */
496 }
497
498
499 static int
500 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
501 struct vmxnet3_adapter *adapter)
502 {
503 size_t sz;
504
505 BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
506 tq->comp_ring.base || tq->buf_info);
507
508 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
509 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
510 &tq->tx_ring.basePA, GFP_KERNEL);
511 if (!tq->tx_ring.base) {
512 netdev_err(adapter->netdev, "failed to allocate tx ring\n");
513 goto err;
514 }
515
516 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
517 tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
518 &tq->data_ring.basePA, GFP_KERNEL);
519 if (!tq->data_ring.base) {
520 netdev_err(adapter->netdev, "failed to allocate data ring\n");
521 goto err;
522 }
523
524 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
525 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
526 &tq->comp_ring.basePA, GFP_KERNEL);
527 if (!tq->comp_ring.base) {
528 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
529 goto err;
530 }
531
532 sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
533 tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
534 &tq->buf_info_pa, GFP_KERNEL);
535 if (!tq->buf_info)
536 goto err;
537
538 return 0;
539
540 err:
541 vmxnet3_tq_destroy(tq, adapter);
542 return -ENOMEM;
543 }
544
545 static void
546 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
547 {
548 int i;
549
550 for (i = 0; i < adapter->num_tx_queues; i++)
551 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
552 }
553
554 /*
555 * starting from ring->next2fill, allocate rx buffers for the given ring
556 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers
557 * are allocated or allocation fails
558 */
559
560 static int
561 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
562 int num_to_alloc, struct vmxnet3_adapter *adapter)
563 {
564 int num_allocated = 0;
565 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
566 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
567 u32 val;
568
569 while (num_allocated <= num_to_alloc) {
570 struct vmxnet3_rx_buf_info *rbi;
571 union Vmxnet3_GenericDesc *gd;
572
573 rbi = rbi_base + ring->next2fill;
574 gd = ring->base + ring->next2fill;
575
576 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
577 if (rbi->skb == NULL) {
578 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
579 rbi->len,
580 GFP_KERNEL);
581 if (unlikely(rbi->skb == NULL)) {
582 rq->stats.rx_buf_alloc_failure++;
583 break;
584 }
585
586 rbi->dma_addr = dma_map_single(
587 &adapter->pdev->dev,
588 rbi->skb->data, rbi->len,
589 PCI_DMA_FROMDEVICE);
590 } else {
591 /* rx buffer skipped by the device */
592 }
593 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
594 } else {
595 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
596 rbi->len != PAGE_SIZE);
597
598 if (rbi->page == NULL) {
599 rbi->page = alloc_page(GFP_ATOMIC);
600 if (unlikely(rbi->page == NULL)) {
601 rq->stats.rx_buf_alloc_failure++;
602 break;
603 }
604 rbi->dma_addr = dma_map_page(
605 &adapter->pdev->dev,
606 rbi->page, 0, PAGE_SIZE,
607 PCI_DMA_FROMDEVICE);
608 } else {
609 /* rx buffers skipped by the device */
610 }
611 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
612 }
613
614 BUG_ON(rbi->dma_addr == 0);
615 gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
616 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
617 | val | rbi->len);
618
619 /* Fill the last buffer but dont mark it ready, or else the
620 * device will think that the queue is full */
621 if (num_allocated == num_to_alloc)
622 break;
623
624 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
625 num_allocated++;
626 vmxnet3_cmd_ring_adv_next2fill(ring);
627 }
628
629 netdev_dbg(adapter->netdev,
630 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
631 num_allocated, ring->next2fill, ring->next2comp);
632
633 /* so that the device can distinguish a full ring and an empty ring */
634 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
635
636 return num_allocated;
637 }
638
639
640 static void
641 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
642 struct vmxnet3_rx_buf_info *rbi)
643 {
644 struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
645 skb_shinfo(skb)->nr_frags;
646
647 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
648
649 __skb_frag_set_page(frag, rbi->page);
650 frag->page_offset = 0;
651 skb_frag_size_set(frag, rcd->len);
652 skb->data_len += rcd->len;
653 skb->truesize += PAGE_SIZE;
654 skb_shinfo(skb)->nr_frags++;
655 }
656
657
658 static void
659 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
660 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
661 struct vmxnet3_adapter *adapter)
662 {
663 u32 dw2, len;
664 unsigned long buf_offset;
665 int i;
666 union Vmxnet3_GenericDesc *gdesc;
667 struct vmxnet3_tx_buf_info *tbi = NULL;
668
669 BUG_ON(ctx->copy_size > skb_headlen(skb));
670
671 /* use the previous gen bit for the SOP desc */
672 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
673
674 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
675 gdesc = ctx->sop_txd; /* both loops below can be skipped */
676
677 /* no need to map the buffer if headers are copied */
678 if (ctx->copy_size) {
679 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
680 tq->tx_ring.next2fill *
681 sizeof(struct Vmxnet3_TxDataDesc));
682 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
683 ctx->sop_txd->dword[3] = 0;
684
685 tbi = tq->buf_info + tq->tx_ring.next2fill;
686 tbi->map_type = VMXNET3_MAP_NONE;
687
688 netdev_dbg(adapter->netdev,
689 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
690 tq->tx_ring.next2fill,
691 le64_to_cpu(ctx->sop_txd->txd.addr),
692 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
693 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
694
695 /* use the right gen for non-SOP desc */
696 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
697 }
698
699 /* linear part can use multiple tx desc if it's big */
700 len = skb_headlen(skb) - ctx->copy_size;
701 buf_offset = ctx->copy_size;
702 while (len) {
703 u32 buf_size;
704
705 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
706 buf_size = len;
707 dw2 |= len;
708 } else {
709 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
710 /* spec says that for TxDesc.len, 0 == 2^14 */
711 }
712
713 tbi = tq->buf_info + tq->tx_ring.next2fill;
714 tbi->map_type = VMXNET3_MAP_SINGLE;
715 tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
716 skb->data + buf_offset, buf_size,
717 PCI_DMA_TODEVICE);
718
719 tbi->len = buf_size;
720
721 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
722 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
723
724 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
725 gdesc->dword[2] = cpu_to_le32(dw2);
726 gdesc->dword[3] = 0;
727
728 netdev_dbg(adapter->netdev,
729 "txd[%u]: 0x%Lx 0x%x 0x%x\n",
730 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
731 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
732 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
733 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
734
735 len -= buf_size;
736 buf_offset += buf_size;
737 }
738
739 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
740 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
741 u32 buf_size;
742
743 buf_offset = 0;
744 len = skb_frag_size(frag);
745 while (len) {
746 tbi = tq->buf_info + tq->tx_ring.next2fill;
747 if (len < VMXNET3_MAX_TX_BUF_SIZE) {
748 buf_size = len;
749 dw2 |= len;
750 } else {
751 buf_size = VMXNET3_MAX_TX_BUF_SIZE;
752 /* spec says that for TxDesc.len, 0 == 2^14 */
753 }
754 tbi->map_type = VMXNET3_MAP_PAGE;
755 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
756 buf_offset, buf_size,
757 DMA_TO_DEVICE);
758
759 tbi->len = buf_size;
760
761 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
762 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
763
764 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
765 gdesc->dword[2] = cpu_to_le32(dw2);
766 gdesc->dword[3] = 0;
767
768 netdev_dbg(adapter->netdev,
769 "txd[%u]: 0x%llu %u %u\n",
770 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
771 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
772 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
773 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
774
775 len -= buf_size;
776 buf_offset += buf_size;
777 }
778 }
779
780 ctx->eop_txd = gdesc;
781
782 /* set the last buf_info for the pkt */
783 tbi->skb = skb;
784 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
785 }
786
787
788 /* Init all tx queues */
789 static void
790 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
791 {
792 int i;
793
794 for (i = 0; i < adapter->num_tx_queues; i++)
795 vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
796 }
797
798
799 /*
800 * parse and copy relevant protocol headers:
801 * For a tso pkt, relevant headers are L2/3/4 including options
802 * For a pkt requesting csum offloading, they are L2/3 and may include L4
803 * if it's a TCP/UDP pkt
804 *
805 * Returns:
806 * -1: error happens during parsing
807 * 0: protocol headers parsed, but too big to be copied
808 * 1: protocol headers parsed and copied
809 *
810 * Other effects:
811 * 1. related *ctx fields are updated.
812 * 2. ctx->copy_size is # of bytes copied
813 * 3. the portion copied is guaranteed to be in the linear part
814 *
815 */
816 static int
817 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
818 struct vmxnet3_tx_ctx *ctx,
819 struct vmxnet3_adapter *adapter)
820 {
821 struct Vmxnet3_TxDataDesc *tdd;
822
823 if (ctx->mss) { /* TSO */
824 ctx->eth_ip_hdr_size = skb_transport_offset(skb);
825 ctx->l4_hdr_size = tcp_hdrlen(skb);
826 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
827 } else {
828 if (skb->ip_summed == CHECKSUM_PARTIAL) {
829 ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
830
831 if (ctx->ipv4) {
832 const struct iphdr *iph = ip_hdr(skb);
833
834 if (iph->protocol == IPPROTO_TCP)
835 ctx->l4_hdr_size = tcp_hdrlen(skb);
836 else if (iph->protocol == IPPROTO_UDP)
837 ctx->l4_hdr_size = sizeof(struct udphdr);
838 else
839 ctx->l4_hdr_size = 0;
840 } else {
841 /* for simplicity, don't copy L4 headers */
842 ctx->l4_hdr_size = 0;
843 }
844 ctx->copy_size = min(ctx->eth_ip_hdr_size +
845 ctx->l4_hdr_size, skb->len);
846 } else {
847 ctx->eth_ip_hdr_size = 0;
848 ctx->l4_hdr_size = 0;
849 /* copy as much as allowed */
850 ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
851 , skb_headlen(skb));
852 }
853
854 /* make sure headers are accessible directly */
855 if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
856 goto err;
857 }
858
859 if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
860 tq->stats.oversized_hdr++;
861 ctx->copy_size = 0;
862 return 0;
863 }
864
865 tdd = tq->data_ring.base + tq->tx_ring.next2fill;
866
867 memcpy(tdd->data, skb->data, ctx->copy_size);
868 netdev_dbg(adapter->netdev,
869 "copy %u bytes to dataRing[%u]\n",
870 ctx->copy_size, tq->tx_ring.next2fill);
871 return 1;
872
873 err:
874 return -1;
875 }
876
877
878 static void
879 vmxnet3_prepare_tso(struct sk_buff *skb,
880 struct vmxnet3_tx_ctx *ctx)
881 {
882 struct tcphdr *tcph = tcp_hdr(skb);
883
884 if (ctx->ipv4) {
885 struct iphdr *iph = ip_hdr(skb);
886
887 iph->check = 0;
888 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
889 IPPROTO_TCP, 0);
890 } else {
891 struct ipv6hdr *iph = ipv6_hdr(skb);
892
893 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
894 IPPROTO_TCP, 0);
895 }
896 }
897
898 static int txd_estimate(const struct sk_buff *skb)
899 {
900 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
901 int i;
902
903 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
904 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
905
906 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
907 }
908 return count;
909 }
910
911 /*
912 * Transmits a pkt thru a given tq
913 * Returns:
914 * NETDEV_TX_OK: descriptors are setup successfully
915 * NETDEV_TX_OK: error occurred, the pkt is dropped
916 * NETDEV_TX_BUSY: tx ring is full, queue is stopped
917 *
918 * Side-effects:
919 * 1. tx ring may be changed
920 * 2. tq stats may be updated accordingly
921 * 3. shared->txNumDeferred may be updated
922 */
923
924 static int
925 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
926 struct vmxnet3_adapter *adapter, struct net_device *netdev)
927 {
928 int ret;
929 u32 count;
930 unsigned long flags;
931 struct vmxnet3_tx_ctx ctx;
932 union Vmxnet3_GenericDesc *gdesc;
933 #ifdef __BIG_ENDIAN_BITFIELD
934 /* Use temporary descriptor to avoid touching bits multiple times */
935 union Vmxnet3_GenericDesc tempTxDesc;
936 #endif
937
938 count = txd_estimate(skb);
939
940 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
941
942 ctx.mss = skb_shinfo(skb)->gso_size;
943 if (ctx.mss) {
944 if (skb_header_cloned(skb)) {
945 if (unlikely(pskb_expand_head(skb, 0, 0,
946 GFP_ATOMIC) != 0)) {
947 tq->stats.drop_tso++;
948 goto drop_pkt;
949 }
950 tq->stats.copy_skb_header++;
951 }
952 vmxnet3_prepare_tso(skb, &ctx);
953 } else {
954 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
955
956 /* non-tso pkts must not use more than
957 * VMXNET3_MAX_TXD_PER_PKT entries
958 */
959 if (skb_linearize(skb) != 0) {
960 tq->stats.drop_too_many_frags++;
961 goto drop_pkt;
962 }
963 tq->stats.linearized++;
964
965 /* recalculate the # of descriptors to use */
966 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
967 }
968 }
969
970 spin_lock_irqsave(&tq->tx_lock, flags);
971
972 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
973 tq->stats.tx_ring_full++;
974 netdev_dbg(adapter->netdev,
975 "tx queue stopped on %s, next2comp %u"
976 " next2fill %u\n", adapter->netdev->name,
977 tq->tx_ring.next2comp, tq->tx_ring.next2fill);
978
979 vmxnet3_tq_stop(tq, adapter);
980 spin_unlock_irqrestore(&tq->tx_lock, flags);
981 return NETDEV_TX_BUSY;
982 }
983
984
985 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
986 if (ret >= 0) {
987 BUG_ON(ret <= 0 && ctx.copy_size != 0);
988 /* hdrs parsed, check against other limits */
989 if (ctx.mss) {
990 if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
991 VMXNET3_MAX_TX_BUF_SIZE)) {
992 goto hdr_too_big;
993 }
994 } else {
995 if (skb->ip_summed == CHECKSUM_PARTIAL) {
996 if (unlikely(ctx.eth_ip_hdr_size +
997 skb->csum_offset >
998 VMXNET3_MAX_CSUM_OFFSET)) {
999 goto hdr_too_big;
1000 }
1001 }
1002 }
1003 } else {
1004 tq->stats.drop_hdr_inspect_err++;
1005 goto unlock_drop_pkt;
1006 }
1007
1008 /* fill tx descs related to addr & len */
1009 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
1010
1011 /* setup the EOP desc */
1012 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1013
1014 /* setup the SOP desc */
1015 #ifdef __BIG_ENDIAN_BITFIELD
1016 gdesc = &tempTxDesc;
1017 gdesc->dword[2] = ctx.sop_txd->dword[2];
1018 gdesc->dword[3] = ctx.sop_txd->dword[3];
1019 #else
1020 gdesc = ctx.sop_txd;
1021 #endif
1022 if (ctx.mss) {
1023 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
1024 gdesc->txd.om = VMXNET3_OM_TSO;
1025 gdesc->txd.msscof = ctx.mss;
1026 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
1027 gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
1028 } else {
1029 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1030 gdesc->txd.hlen = ctx.eth_ip_hdr_size;
1031 gdesc->txd.om = VMXNET3_OM_CSUM;
1032 gdesc->txd.msscof = ctx.eth_ip_hdr_size +
1033 skb->csum_offset;
1034 } else {
1035 gdesc->txd.om = 0;
1036 gdesc->txd.msscof = 0;
1037 }
1038 le32_add_cpu(&tq->shared->txNumDeferred, 1);
1039 }
1040
1041 if (vlan_tx_tag_present(skb)) {
1042 gdesc->txd.ti = 1;
1043 gdesc->txd.tci = vlan_tx_tag_get(skb);
1044 }
1045
1046 /* finally flips the GEN bit of the SOP desc. */
1047 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1048 VMXNET3_TXD_GEN);
1049 #ifdef __BIG_ENDIAN_BITFIELD
1050 /* Finished updating in bitfields of Tx Desc, so write them in original
1051 * place.
1052 */
1053 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1054 (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1055 gdesc = ctx.sop_txd;
1056 #endif
1057 netdev_dbg(adapter->netdev,
1058 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1059 (u32)(ctx.sop_txd -
1060 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1061 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1062
1063 spin_unlock_irqrestore(&tq->tx_lock, flags);
1064
1065 if (le32_to_cpu(tq->shared->txNumDeferred) >=
1066 le32_to_cpu(tq->shared->txThreshold)) {
1067 tq->shared->txNumDeferred = 0;
1068 VMXNET3_WRITE_BAR0_REG(adapter,
1069 VMXNET3_REG_TXPROD + tq->qid * 8,
1070 tq->tx_ring.next2fill);
1071 }
1072
1073 return NETDEV_TX_OK;
1074
1075 hdr_too_big:
1076 tq->stats.drop_oversized_hdr++;
1077 unlock_drop_pkt:
1078 spin_unlock_irqrestore(&tq->tx_lock, flags);
1079 drop_pkt:
1080 tq->stats.drop_total++;
1081 dev_kfree_skb(skb);
1082 return NETDEV_TX_OK;
1083 }
1084
1085
1086 static netdev_tx_t
1087 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1088 {
1089 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1090
1091 BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1092 return vmxnet3_tq_xmit(skb,
1093 &adapter->tx_queue[skb->queue_mapping],
1094 adapter, netdev);
1095 }
1096
1097
1098 static void
1099 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1100 struct sk_buff *skb,
1101 union Vmxnet3_GenericDesc *gdesc)
1102 {
1103 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1104 /* typical case: TCP/UDP over IP and both csums are correct */
1105 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
1106 VMXNET3_RCD_CSUM_OK) {
1107 skb->ip_summed = CHECKSUM_UNNECESSARY;
1108 BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
1109 BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));
1110 BUG_ON(gdesc->rcd.frg);
1111 } else {
1112 if (gdesc->rcd.csum) {
1113 skb->csum = htons(gdesc->rcd.csum);
1114 skb->ip_summed = CHECKSUM_PARTIAL;
1115 } else {
1116 skb_checksum_none_assert(skb);
1117 }
1118 }
1119 } else {
1120 skb_checksum_none_assert(skb);
1121 }
1122 }
1123
1124
1125 static void
1126 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1127 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)
1128 {
1129 rq->stats.drop_err++;
1130 if (!rcd->fcs)
1131 rq->stats.drop_fcs++;
1132
1133 rq->stats.drop_total++;
1134
1135 /*
1136 * We do not unmap and chain the rx buffer to the skb.
1137 * We basically pretend this buffer is not used and will be recycled
1138 * by vmxnet3_rq_alloc_rx_buf()
1139 */
1140
1141 /*
1142 * ctx->skb may be NULL if this is the first and the only one
1143 * desc for the pkt
1144 */
1145 if (ctx->skb)
1146 dev_kfree_skb_irq(ctx->skb);
1147
1148 ctx->skb = NULL;
1149 }
1150
1151
1152 static int
1153 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1154 struct vmxnet3_adapter *adapter, int quota)
1155 {
1156 static const u32 rxprod_reg[2] = {
1157 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
1158 };
1159 u32 num_rxd = 0;
1160 bool skip_page_frags = false;
1161 struct Vmxnet3_RxCompDesc *rcd;
1162 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1163 #ifdef __BIG_ENDIAN_BITFIELD
1164 struct Vmxnet3_RxDesc rxCmdDesc;
1165 struct Vmxnet3_RxCompDesc rxComp;
1166 #endif
1167 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1168 &rxComp);
1169 while (rcd->gen == rq->comp_ring.gen) {
1170 struct vmxnet3_rx_buf_info *rbi;
1171 struct sk_buff *skb, *new_skb = NULL;
1172 struct page *new_page = NULL;
1173 int num_to_alloc;
1174 struct Vmxnet3_RxDesc *rxd;
1175 u32 idx, ring_idx;
1176 struct vmxnet3_cmd_ring *ring = NULL;
1177 if (num_rxd >= quota) {
1178 /* we may stop even before we see the EOP desc of
1179 * the current pkt
1180 */
1181 break;
1182 }
1183 num_rxd++;
1184 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
1185 idx = rcd->rxdIdx;
1186 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
1187 ring = rq->rx_ring + ring_idx;
1188 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1189 &rxCmdDesc);
1190 rbi = rq->buf_info[ring_idx] + idx;
1191
1192 BUG_ON(rxd->addr != rbi->dma_addr ||
1193 rxd->len != rbi->len);
1194
1195 if (unlikely(rcd->eop && rcd->err)) {
1196 vmxnet3_rx_error(rq, rcd, ctx, adapter);
1197 goto rcd_done;
1198 }
1199
1200 if (rcd->sop) { /* first buf of the pkt */
1201 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1202 rcd->rqID != rq->qid);
1203
1204 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1205 BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1206
1207 if (unlikely(rcd->len == 0)) {
1208 /* Pretend the rx buffer is skipped. */
1209 BUG_ON(!(rcd->sop && rcd->eop));
1210 netdev_dbg(adapter->netdev,
1211 "rxRing[%u][%u] 0 length\n",
1212 ring_idx, idx);
1213 goto rcd_done;
1214 }
1215
1216 skip_page_frags = false;
1217 ctx->skb = rbi->skb;
1218 new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1219 rbi->len);
1220 if (new_skb == NULL) {
1221 /* Skb allocation failed, do not handover this
1222 * skb to stack. Reuse it. Drop the existing pkt
1223 */
1224 rq->stats.rx_buf_alloc_failure++;
1225 ctx->skb = NULL;
1226 rq->stats.drop_total++;
1227 skip_page_frags = true;
1228 goto rcd_done;
1229 }
1230
1231 dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
1232 rbi->len,
1233 PCI_DMA_FROMDEVICE);
1234
1235 #ifdef VMXNET3_RSS
1236 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1237 (adapter->netdev->features & NETIF_F_RXHASH))
1238 ctx->skb->rxhash = le32_to_cpu(rcd->rssHash);
1239 #endif
1240 skb_put(ctx->skb, rcd->len);
1241
1242 /* Immediate refill */
1243 rbi->skb = new_skb;
1244 rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
1245 rbi->skb->data, rbi->len,
1246 PCI_DMA_FROMDEVICE);
1247 rxd->addr = cpu_to_le64(rbi->dma_addr);
1248 rxd->len = rbi->len;
1249
1250 } else {
1251 BUG_ON(ctx->skb == NULL && !skip_page_frags);
1252
1253 /* non SOP buffer must be type 1 in most cases */
1254 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1255 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1256
1257 /* If an sop buffer was dropped, skip all
1258 * following non-sop fragments. They will be reused.
1259 */
1260 if (skip_page_frags)
1261 goto rcd_done;
1262
1263 new_page = alloc_page(GFP_ATOMIC);
1264 if (unlikely(new_page == NULL)) {
1265 /* Replacement page frag could not be allocated.
1266 * Reuse this page. Drop the pkt and free the
1267 * skb which contained this page as a frag. Skip
1268 * processing all the following non-sop frags.
1269 */
1270 rq->stats.rx_buf_alloc_failure++;
1271 dev_kfree_skb(ctx->skb);
1272 ctx->skb = NULL;
1273 skip_page_frags = true;
1274 goto rcd_done;
1275 }
1276
1277 if (rcd->len) {
1278 dma_unmap_page(&adapter->pdev->dev,
1279 rbi->dma_addr, rbi->len,
1280 PCI_DMA_FROMDEVICE);
1281
1282 vmxnet3_append_frag(ctx->skb, rcd, rbi);
1283 }
1284
1285 /* Immediate refill */
1286 rbi->page = new_page;
1287 rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
1288 rbi->page,
1289 0, PAGE_SIZE,
1290 PCI_DMA_FROMDEVICE);
1291 rxd->addr = cpu_to_le64(rbi->dma_addr);
1292 rxd->len = rbi->len;
1293 }
1294
1295
1296 skb = ctx->skb;
1297 if (rcd->eop) {
1298 skb->len += skb->data_len;
1299
1300 vmxnet3_rx_csum(adapter, skb,
1301 (union Vmxnet3_GenericDesc *)rcd);
1302 skb->protocol = eth_type_trans(skb, adapter->netdev);
1303
1304 if (unlikely(rcd->ts))
1305 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1306
1307 if (adapter->netdev->features & NETIF_F_LRO)
1308 netif_receive_skb(skb);
1309 else
1310 napi_gro_receive(&rq->napi, skb);
1311
1312 ctx->skb = NULL;
1313 }
1314
1315 rcd_done:
1316 /* device may have skipped some rx descs */
1317 ring->next2comp = idx;
1318 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1319 ring = rq->rx_ring + ring_idx;
1320 while (num_to_alloc) {
1321 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1322 &rxCmdDesc);
1323 BUG_ON(!rxd->addr);
1324
1325 /* Recv desc is ready to be used by the device */
1326 rxd->gen = ring->gen;
1327 vmxnet3_cmd_ring_adv_next2fill(ring);
1328 num_to_alloc--;
1329 }
1330
1331 /* if needed, update the register */
1332 if (unlikely(rq->shared->updateRxProd)) {
1333 VMXNET3_WRITE_BAR0_REG(adapter,
1334 rxprod_reg[ring_idx] + rq->qid * 8,
1335 ring->next2fill);
1336 }
1337
1338 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1339 vmxnet3_getRxComp(rcd,
1340 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1341 }
1342
1343 return num_rxd;
1344 }
1345
1346
1347 static void
1348 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1349 struct vmxnet3_adapter *adapter)
1350 {
1351 u32 i, ring_idx;
1352 struct Vmxnet3_RxDesc *rxd;
1353
1354 for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1355 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1356 #ifdef __BIG_ENDIAN_BITFIELD
1357 struct Vmxnet3_RxDesc rxDesc;
1358 #endif
1359 vmxnet3_getRxDesc(rxd,
1360 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1361
1362 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1363 rq->buf_info[ring_idx][i].skb) {
1364 dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1365 rxd->len, PCI_DMA_FROMDEVICE);
1366 dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1367 rq->buf_info[ring_idx][i].skb = NULL;
1368 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1369 rq->buf_info[ring_idx][i].page) {
1370 dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1371 rxd->len, PCI_DMA_FROMDEVICE);
1372 put_page(rq->buf_info[ring_idx][i].page);
1373 rq->buf_info[ring_idx][i].page = NULL;
1374 }
1375 }
1376
1377 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1378 rq->rx_ring[ring_idx].next2fill =
1379 rq->rx_ring[ring_idx].next2comp = 0;
1380 }
1381
1382 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1383 rq->comp_ring.next2proc = 0;
1384 }
1385
1386
1387 static void
1388 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1389 {
1390 int i;
1391
1392 for (i = 0; i < adapter->num_rx_queues; i++)
1393 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1394 }
1395
1396
1397 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1398 struct vmxnet3_adapter *adapter)
1399 {
1400 int i;
1401 int j;
1402
1403 /* all rx buffers must have already been freed */
1404 for (i = 0; i < 2; i++) {
1405 if (rq->buf_info[i]) {
1406 for (j = 0; j < rq->rx_ring[i].size; j++)
1407 BUG_ON(rq->buf_info[i][j].page != NULL);
1408 }
1409 }
1410
1411
1412 for (i = 0; i < 2; i++) {
1413 if (rq->rx_ring[i].base) {
1414 dma_free_coherent(&adapter->pdev->dev,
1415 rq->rx_ring[i].size
1416 * sizeof(struct Vmxnet3_RxDesc),
1417 rq->rx_ring[i].base,
1418 rq->rx_ring[i].basePA);
1419 rq->rx_ring[i].base = NULL;
1420 }
1421 rq->buf_info[i] = NULL;
1422 }
1423
1424 if (rq->comp_ring.base) {
1425 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1426 * sizeof(struct Vmxnet3_RxCompDesc),
1427 rq->comp_ring.base, rq->comp_ring.basePA);
1428 rq->comp_ring.base = NULL;
1429 }
1430
1431 if (rq->buf_info[0]) {
1432 size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
1433 (rq->rx_ring[0].size + rq->rx_ring[1].size);
1434 dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
1435 rq->buf_info_pa);
1436 }
1437 }
1438
1439
1440 static int
1441 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1442 struct vmxnet3_adapter *adapter)
1443 {
1444 int i;
1445
1446 /* initialize buf_info */
1447 for (i = 0; i < rq->rx_ring[0].size; i++) {
1448
1449 /* 1st buf for a pkt is skbuff */
1450 if (i % adapter->rx_buf_per_pkt == 0) {
1451 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1452 rq->buf_info[0][i].len = adapter->skb_buf_size;
1453 } else { /* subsequent bufs for a pkt is frag */
1454 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1455 rq->buf_info[0][i].len = PAGE_SIZE;
1456 }
1457 }
1458 for (i = 0; i < rq->rx_ring[1].size; i++) {
1459 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1460 rq->buf_info[1][i].len = PAGE_SIZE;
1461 }
1462
1463 /* reset internal state and allocate buffers for both rings */
1464 for (i = 0; i < 2; i++) {
1465 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1466
1467 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1468 sizeof(struct Vmxnet3_RxDesc));
1469 rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1470 }
1471 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1472 adapter) == 0) {
1473 /* at least has 1 rx buffer for the 1st ring */
1474 return -ENOMEM;
1475 }
1476 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1477
1478 /* reset the comp ring */
1479 rq->comp_ring.next2proc = 0;
1480 memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1481 sizeof(struct Vmxnet3_RxCompDesc));
1482 rq->comp_ring.gen = VMXNET3_INIT_GEN;
1483
1484 /* reset rxctx */
1485 rq->rx_ctx.skb = NULL;
1486
1487 /* stats are not reset */
1488 return 0;
1489 }
1490
1491
1492 static int
1493 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1494 {
1495 int i, err = 0;
1496
1497 for (i = 0; i < adapter->num_rx_queues; i++) {
1498 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1499 if (unlikely(err)) {
1500 dev_err(&adapter->netdev->dev, "%s: failed to "
1501 "initialize rx queue%i\n",
1502 adapter->netdev->name, i);
1503 break;
1504 }
1505 }
1506 return err;
1507
1508 }
1509
1510
1511 static int
1512 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1513 {
1514 int i;
1515 size_t sz;
1516 struct vmxnet3_rx_buf_info *bi;
1517
1518 for (i = 0; i < 2; i++) {
1519
1520 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1521 rq->rx_ring[i].base = dma_alloc_coherent(
1522 &adapter->pdev->dev, sz,
1523 &rq->rx_ring[i].basePA,
1524 GFP_KERNEL);
1525 if (!rq->rx_ring[i].base) {
1526 netdev_err(adapter->netdev,
1527 "failed to allocate rx ring %d\n", i);
1528 goto err;
1529 }
1530 }
1531
1532 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1533 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1534 &rq->comp_ring.basePA,
1535 GFP_KERNEL);
1536 if (!rq->comp_ring.base) {
1537 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1538 goto err;
1539 }
1540
1541 sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
1542 rq->rx_ring[1].size);
1543 bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
1544 GFP_KERNEL);
1545 if (!bi)
1546 goto err;
1547
1548 rq->buf_info[0] = bi;
1549 rq->buf_info[1] = bi + rq->rx_ring[0].size;
1550
1551 return 0;
1552
1553 err:
1554 vmxnet3_rq_destroy(rq, adapter);
1555 return -ENOMEM;
1556 }
1557
1558
1559 static int
1560 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1561 {
1562 int i, err = 0;
1563
1564 for (i = 0; i < adapter->num_rx_queues; i++) {
1565 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
1566 if (unlikely(err)) {
1567 dev_err(&adapter->netdev->dev,
1568 "%s: failed to create rx queue%i\n",
1569 adapter->netdev->name, i);
1570 goto err_out;
1571 }
1572 }
1573 return err;
1574 err_out:
1575 vmxnet3_rq_destroy_all(adapter);
1576 return err;
1577
1578 }
1579
1580 /* Multiple queue aware polling function for tx and rx */
1581
1582 static int
1583 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
1584 {
1585 int rcd_done = 0, i;
1586 if (unlikely(adapter->shared->ecr))
1587 vmxnet3_process_events(adapter);
1588 for (i = 0; i < adapter->num_tx_queues; i++)
1589 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
1590
1591 for (i = 0; i < adapter->num_rx_queues; i++)
1592 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
1593 adapter, budget);
1594 return rcd_done;
1595 }
1596
1597
1598 static int
1599 vmxnet3_poll(struct napi_struct *napi, int budget)
1600 {
1601 struct vmxnet3_rx_queue *rx_queue = container_of(napi,
1602 struct vmxnet3_rx_queue, napi);
1603 int rxd_done;
1604
1605 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
1606
1607 if (rxd_done < budget) {
1608 napi_complete(napi);
1609 vmxnet3_enable_all_intrs(rx_queue->adapter);
1610 }
1611 return rxd_done;
1612 }
1613
1614 /*
1615 * NAPI polling function for MSI-X mode with multiple Rx queues
1616 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
1617 */
1618
1619 static int
1620 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
1621 {
1622 struct vmxnet3_rx_queue *rq = container_of(napi,
1623 struct vmxnet3_rx_queue, napi);
1624 struct vmxnet3_adapter *adapter = rq->adapter;
1625 int rxd_done;
1626
1627 /* When sharing interrupt with corresponding tx queue, process
1628 * tx completions in that queue as well
1629 */
1630 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1631 struct vmxnet3_tx_queue *tq =
1632 &adapter->tx_queue[rq - adapter->rx_queue];
1633 vmxnet3_tq_tx_complete(tq, adapter);
1634 }
1635
1636 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
1637
1638 if (rxd_done < budget) {
1639 napi_complete(napi);
1640 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
1641 }
1642 return rxd_done;
1643 }
1644
1645
1646 #ifdef CONFIG_PCI_MSI
1647
1648 /*
1649 * Handle completion interrupts on tx queues
1650 * Returns whether or not the intr is handled
1651 */
1652
1653 static irqreturn_t
1654 vmxnet3_msix_tx(int irq, void *data)
1655 {
1656 struct vmxnet3_tx_queue *tq = data;
1657 struct vmxnet3_adapter *adapter = tq->adapter;
1658
1659 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1660 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
1661
1662 /* Handle the case where only one irq is allocate for all tx queues */
1663 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1664 int i;
1665 for (i = 0; i < adapter->num_tx_queues; i++) {
1666 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
1667 vmxnet3_tq_tx_complete(txq, adapter);
1668 }
1669 } else {
1670 vmxnet3_tq_tx_complete(tq, adapter);
1671 }
1672 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
1673
1674 return IRQ_HANDLED;
1675 }
1676
1677
1678 /*
1679 * Handle completion interrupts on rx queues. Returns whether or not the
1680 * intr is handled
1681 */
1682
1683 static irqreturn_t
1684 vmxnet3_msix_rx(int irq, void *data)
1685 {
1686 struct vmxnet3_rx_queue *rq = data;
1687 struct vmxnet3_adapter *adapter = rq->adapter;
1688
1689 /* disable intr if needed */
1690 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1691 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
1692 napi_schedule(&rq->napi);
1693
1694 return IRQ_HANDLED;
1695 }
1696
1697 /*
1698 *----------------------------------------------------------------------------
1699 *
1700 * vmxnet3_msix_event --
1701 *
1702 * vmxnet3 msix event intr handler
1703 *
1704 * Result:
1705 * whether or not the intr is handled
1706 *
1707 *----------------------------------------------------------------------------
1708 */
1709
1710 static irqreturn_t
1711 vmxnet3_msix_event(int irq, void *data)
1712 {
1713 struct net_device *dev = data;
1714 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1715
1716 /* disable intr if needed */
1717 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1718 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
1719
1720 if (adapter->shared->ecr)
1721 vmxnet3_process_events(adapter);
1722
1723 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
1724
1725 return IRQ_HANDLED;
1726 }
1727
1728 #endif /* CONFIG_PCI_MSI */
1729
1730
1731 /* Interrupt handler for vmxnet3 */
1732 static irqreturn_t
1733 vmxnet3_intr(int irq, void *dev_id)
1734 {
1735 struct net_device *dev = dev_id;
1736 struct vmxnet3_adapter *adapter = netdev_priv(dev);
1737
1738 if (adapter->intr.type == VMXNET3_IT_INTX) {
1739 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
1740 if (unlikely(icr == 0))
1741 /* not ours */
1742 return IRQ_NONE;
1743 }
1744
1745
1746 /* disable intr if needed */
1747 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1748 vmxnet3_disable_all_intrs(adapter);
1749
1750 napi_schedule(&adapter->rx_queue[0].napi);
1751
1752 return IRQ_HANDLED;
1753 }
1754
1755 #ifdef CONFIG_NET_POLL_CONTROLLER
1756
1757 /* netpoll callback. */
1758 static void
1759 vmxnet3_netpoll(struct net_device *netdev)
1760 {
1761 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1762
1763 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
1764 vmxnet3_disable_all_intrs(adapter);
1765
1766 vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
1767 vmxnet3_enable_all_intrs(adapter);
1768
1769 }
1770 #endif /* CONFIG_NET_POLL_CONTROLLER */
1771
1772 static int
1773 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
1774 {
1775 struct vmxnet3_intr *intr = &adapter->intr;
1776 int err = 0, i;
1777 int vector = 0;
1778
1779 #ifdef CONFIG_PCI_MSI
1780 if (adapter->intr.type == VMXNET3_IT_MSIX) {
1781 for (i = 0; i < adapter->num_tx_queues; i++) {
1782 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1783 sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
1784 adapter->netdev->name, vector);
1785 err = request_irq(
1786 intr->msix_entries[vector].vector,
1787 vmxnet3_msix_tx, 0,
1788 adapter->tx_queue[i].name,
1789 &adapter->tx_queue[i]);
1790 } else {
1791 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
1792 adapter->netdev->name, vector);
1793 }
1794 if (err) {
1795 dev_err(&adapter->netdev->dev,
1796 "Failed to request irq for MSIX, %s, "
1797 "error %d\n",
1798 adapter->tx_queue[i].name, err);
1799 return err;
1800 }
1801
1802 /* Handle the case where only 1 MSIx was allocated for
1803 * all tx queues */
1804 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
1805 for (; i < adapter->num_tx_queues; i++)
1806 adapter->tx_queue[i].comp_ring.intr_idx
1807 = vector;
1808 vector++;
1809 break;
1810 } else {
1811 adapter->tx_queue[i].comp_ring.intr_idx
1812 = vector++;
1813 }
1814 }
1815 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
1816 vector = 0;
1817
1818 for (i = 0; i < adapter->num_rx_queues; i++) {
1819 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
1820 sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
1821 adapter->netdev->name, vector);
1822 else
1823 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
1824 adapter->netdev->name, vector);
1825 err = request_irq(intr->msix_entries[vector].vector,
1826 vmxnet3_msix_rx, 0,
1827 adapter->rx_queue[i].name,
1828 &(adapter->rx_queue[i]));
1829 if (err) {
1830 netdev_err(adapter->netdev,
1831 "Failed to request irq for MSIX, "
1832 "%s, error %d\n",
1833 adapter->rx_queue[i].name, err);
1834 return err;
1835 }
1836
1837 adapter->rx_queue[i].comp_ring.intr_idx = vector++;
1838 }
1839
1840 sprintf(intr->event_msi_vector_name, "%s-event-%d",
1841 adapter->netdev->name, vector);
1842 err = request_irq(intr->msix_entries[vector].vector,
1843 vmxnet3_msix_event, 0,
1844 intr->event_msi_vector_name, adapter->netdev);
1845 intr->event_intr_idx = vector;
1846
1847 } else if (intr->type == VMXNET3_IT_MSI) {
1848 adapter->num_rx_queues = 1;
1849 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
1850 adapter->netdev->name, adapter->netdev);
1851 } else {
1852 #endif
1853 adapter->num_rx_queues = 1;
1854 err = request_irq(adapter->pdev->irq, vmxnet3_intr,
1855 IRQF_SHARED, adapter->netdev->name,
1856 adapter->netdev);
1857 #ifdef CONFIG_PCI_MSI
1858 }
1859 #endif
1860 intr->num_intrs = vector + 1;
1861 if (err) {
1862 netdev_err(adapter->netdev,
1863 "Failed to request irq (intr type:%d), error %d\n",
1864 intr->type, err);
1865 } else {
1866 /* Number of rx queues will not change after this */
1867 for (i = 0; i < adapter->num_rx_queues; i++) {
1868 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1869 rq->qid = i;
1870 rq->qid2 = i + adapter->num_rx_queues;
1871 }
1872
1873
1874
1875 /* init our intr settings */
1876 for (i = 0; i < intr->num_intrs; i++)
1877 intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
1878 if (adapter->intr.type != VMXNET3_IT_MSIX) {
1879 adapter->intr.event_intr_idx = 0;
1880 for (i = 0; i < adapter->num_tx_queues; i++)
1881 adapter->tx_queue[i].comp_ring.intr_idx = 0;
1882 adapter->rx_queue[0].comp_ring.intr_idx = 0;
1883 }
1884
1885 netdev_info(adapter->netdev,
1886 "intr type %u, mode %u, %u vectors allocated\n",
1887 intr->type, intr->mask_mode, intr->num_intrs);
1888 }
1889
1890 return err;
1891 }
1892
1893
1894 static void
1895 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
1896 {
1897 struct vmxnet3_intr *intr = &adapter->intr;
1898 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
1899
1900 switch (intr->type) {
1901 #ifdef CONFIG_PCI_MSI
1902 case VMXNET3_IT_MSIX:
1903 {
1904 int i, vector = 0;
1905
1906 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
1907 for (i = 0; i < adapter->num_tx_queues; i++) {
1908 free_irq(intr->msix_entries[vector++].vector,
1909 &(adapter->tx_queue[i]));
1910 if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
1911 break;
1912 }
1913 }
1914
1915 for (i = 0; i < adapter->num_rx_queues; i++) {
1916 free_irq(intr->msix_entries[vector++].vector,
1917 &(adapter->rx_queue[i]));
1918 }
1919
1920 free_irq(intr->msix_entries[vector].vector,
1921 adapter->netdev);
1922 BUG_ON(vector >= intr->num_intrs);
1923 break;
1924 }
1925 #endif
1926 case VMXNET3_IT_MSI:
1927 free_irq(adapter->pdev->irq, adapter->netdev);
1928 break;
1929 case VMXNET3_IT_INTX:
1930 free_irq(adapter->pdev->irq, adapter->netdev);
1931 break;
1932 default:
1933 BUG();
1934 }
1935 }
1936
1937
1938 static void
1939 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
1940 {
1941 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1942 u16 vid;
1943
1944 /* allow untagged pkts */
1945 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
1946
1947 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1948 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1949 }
1950
1951
1952 static int
1953 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1954 {
1955 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1956
1957 if (!(netdev->flags & IFF_PROMISC)) {
1958 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1959 unsigned long flags;
1960
1961 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
1962 spin_lock_irqsave(&adapter->cmd_lock, flags);
1963 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1964 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1965 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1966 }
1967
1968 set_bit(vid, adapter->active_vlans);
1969
1970 return 0;
1971 }
1972
1973
1974 static int
1975 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1976 {
1977 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1978
1979 if (!(netdev->flags & IFF_PROMISC)) {
1980 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
1981 unsigned long flags;
1982
1983 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
1984 spin_lock_irqsave(&adapter->cmd_lock, flags);
1985 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1986 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
1987 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1988 }
1989
1990 clear_bit(vid, adapter->active_vlans);
1991
1992 return 0;
1993 }
1994
1995
1996 static u8 *
1997 vmxnet3_copy_mc(struct net_device *netdev)
1998 {
1999 u8 *buf = NULL;
2000 u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2001
2002 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2003 if (sz <= 0xffff) {
2004 /* We may be called with BH disabled */
2005 buf = kmalloc(sz, GFP_ATOMIC);
2006 if (buf) {
2007 struct netdev_hw_addr *ha;
2008 int i = 0;
2009
2010 netdev_for_each_mc_addr(ha, netdev)
2011 memcpy(buf + i++ * ETH_ALEN, ha->addr,
2012 ETH_ALEN);
2013 }
2014 }
2015 return buf;
2016 }
2017
2018
2019 static void
2020 vmxnet3_set_mc(struct net_device *netdev)
2021 {
2022 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2023 unsigned long flags;
2024 struct Vmxnet3_RxFilterConf *rxConf =
2025 &adapter->shared->devRead.rxFilterConf;
2026 u8 *new_table = NULL;
2027 dma_addr_t new_table_pa = 0;
2028 u32 new_mode = VMXNET3_RXM_UCAST;
2029
2030 if (netdev->flags & IFF_PROMISC) {
2031 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2032 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2033
2034 new_mode |= VMXNET3_RXM_PROMISC;
2035 } else {
2036 vmxnet3_restore_vlan(adapter);
2037 }
2038
2039 if (netdev->flags & IFF_BROADCAST)
2040 new_mode |= VMXNET3_RXM_BCAST;
2041
2042 if (netdev->flags & IFF_ALLMULTI)
2043 new_mode |= VMXNET3_RXM_ALL_MULTI;
2044 else
2045 if (!netdev_mc_empty(netdev)) {
2046 new_table = vmxnet3_copy_mc(netdev);
2047 if (new_table) {
2048 new_mode |= VMXNET3_RXM_MCAST;
2049 rxConf->mfTableLen = cpu_to_le16(
2050 netdev_mc_count(netdev) * ETH_ALEN);
2051 new_table_pa = dma_map_single(
2052 &adapter->pdev->dev,
2053 new_table,
2054 rxConf->mfTableLen,
2055 PCI_DMA_TODEVICE);
2056 rxConf->mfTablePA = cpu_to_le64(new_table_pa);
2057 } else {
2058 netdev_info(netdev, "failed to copy mcast list"
2059 ", setting ALL_MULTI\n");
2060 new_mode |= VMXNET3_RXM_ALL_MULTI;
2061 }
2062 }
2063
2064
2065 if (!(new_mode & VMXNET3_RXM_MCAST)) {
2066 rxConf->mfTableLen = 0;
2067 rxConf->mfTablePA = 0;
2068 }
2069
2070 spin_lock_irqsave(&adapter->cmd_lock, flags);
2071 if (new_mode != rxConf->rxMode) {
2072 rxConf->rxMode = cpu_to_le32(new_mode);
2073 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2074 VMXNET3_CMD_UPDATE_RX_MODE);
2075 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2076 VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2077 }
2078
2079 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2080 VMXNET3_CMD_UPDATE_MAC_FILTERS);
2081 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2082
2083 if (new_table) {
2084 dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2085 rxConf->mfTableLen, PCI_DMA_TODEVICE);
2086 kfree(new_table);
2087 }
2088 }
2089
2090 void
2091 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2092 {
2093 int i;
2094
2095 for (i = 0; i < adapter->num_rx_queues; i++)
2096 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2097 }
2098
2099
2100 /*
2101 * Set up driver_shared based on settings in adapter.
2102 */
2103
2104 static void
2105 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2106 {
2107 struct Vmxnet3_DriverShared *shared = adapter->shared;
2108 struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2109 struct Vmxnet3_TxQueueConf *tqc;
2110 struct Vmxnet3_RxQueueConf *rqc;
2111 int i;
2112
2113 memset(shared, 0, sizeof(*shared));
2114
2115 /* driver settings */
2116 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2117 devRead->misc.driverInfo.version = cpu_to_le32(
2118 VMXNET3_DRIVER_VERSION_NUM);
2119 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2120 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2121 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2122 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2123 *((u32 *)&devRead->misc.driverInfo.gos));
2124 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2125 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2126
2127 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2128 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2129
2130 /* set up feature flags */
2131 if (adapter->netdev->features & NETIF_F_RXCSUM)
2132 devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2133
2134 if (adapter->netdev->features & NETIF_F_LRO) {
2135 devRead->misc.uptFeatures |= UPT1_F_LRO;
2136 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2137 }
2138 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2139 devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2140
2141 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2142 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2143 devRead->misc.queueDescLen = cpu_to_le32(
2144 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2145 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2146
2147 /* tx queue settings */
2148 devRead->misc.numTxQueues = adapter->num_tx_queues;
2149 for (i = 0; i < adapter->num_tx_queues; i++) {
2150 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2151 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2152 tqc = &adapter->tqd_start[i].conf;
2153 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);
2154 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2155 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2156 tqc->ddPA = cpu_to_le64(tq->buf_info_pa);
2157 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);
2158 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);
2159 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);
2160 tqc->ddLen = cpu_to_le32(
2161 sizeof(struct vmxnet3_tx_buf_info) *
2162 tqc->txRingSize);
2163 tqc->intrIdx = tq->comp_ring.intr_idx;
2164 }
2165
2166 /* rx queue settings */
2167 devRead->misc.numRxQueues = adapter->num_rx_queues;
2168 for (i = 0; i < adapter->num_rx_queues; i++) {
2169 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2170 rqc = &adapter->rqd_start[i].conf;
2171 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2172 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2173 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);
2174 rqc->ddPA = cpu_to_le64(rq->buf_info_pa);
2175 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);
2176 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);
2177 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);
2178 rqc->ddLen = cpu_to_le32(
2179 sizeof(struct vmxnet3_rx_buf_info) *
2180 (rqc->rxRingSize[0] +
2181 rqc->rxRingSize[1]));
2182 rqc->intrIdx = rq->comp_ring.intr_idx;
2183 }
2184
2185 #ifdef VMXNET3_RSS
2186 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2187
2188 if (adapter->rss) {
2189 struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2190 static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
2191 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
2192 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
2193 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
2194 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
2195 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
2196 };
2197
2198 devRead->misc.uptFeatures |= UPT1_F_RSS;
2199 devRead->misc.numRxQueues = adapter->num_rx_queues;
2200 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2201 UPT1_RSS_HASH_TYPE_IPV4 |
2202 UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2203 UPT1_RSS_HASH_TYPE_IPV6;
2204 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2205 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2206 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2207 memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
2208
2209 for (i = 0; i < rssConf->indTableSize; i++)
2210 rssConf->indTable[i] = ethtool_rxfh_indir_default(
2211 i, adapter->num_rx_queues);
2212
2213 devRead->rssConfDesc.confVer = 1;
2214 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2215 devRead->rssConfDesc.confPA =
2216 cpu_to_le64(adapter->rss_conf_pa);
2217 }
2218
2219 #endif /* VMXNET3_RSS */
2220
2221 /* intr settings */
2222 devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2223 VMXNET3_IMM_AUTO;
2224 devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2225 for (i = 0; i < adapter->intr.num_intrs; i++)
2226 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2227
2228 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2229 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2230
2231 /* rx filter settings */
2232 devRead->rxFilterConf.rxMode = 0;
2233 vmxnet3_restore_vlan(adapter);
2234 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2235
2236 /* the rest are already zeroed */
2237 }
2238
2239
2240 int
2241 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2242 {
2243 int err, i;
2244 u32 ret;
2245 unsigned long flags;
2246
2247 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2248 " ring sizes %u %u %u\n", adapter->netdev->name,
2249 adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2250 adapter->tx_queue[0].tx_ring.size,
2251 adapter->rx_queue[0].rx_ring[0].size,
2252 adapter->rx_queue[0].rx_ring[1].size);
2253
2254 vmxnet3_tq_init_all(adapter);
2255 err = vmxnet3_rq_init_all(adapter);
2256 if (err) {
2257 netdev_err(adapter->netdev,
2258 "Failed to init rx queue error %d\n", err);
2259 goto rq_err;
2260 }
2261
2262 err = vmxnet3_request_irqs(adapter);
2263 if (err) {
2264 netdev_err(adapter->netdev,
2265 "Failed to setup irq for error %d\n", err);
2266 goto irq_err;
2267 }
2268
2269 vmxnet3_setup_driver_shared(adapter);
2270
2271 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2272 adapter->shared_pa));
2273 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2274 adapter->shared_pa));
2275 spin_lock_irqsave(&adapter->cmd_lock, flags);
2276 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2277 VMXNET3_CMD_ACTIVATE_DEV);
2278 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2279 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2280
2281 if (ret != 0) {
2282 netdev_err(adapter->netdev,
2283 "Failed to activate dev: error %u\n", ret);
2284 err = -EINVAL;
2285 goto activate_err;
2286 }
2287
2288 for (i = 0; i < adapter->num_rx_queues; i++) {
2289 VMXNET3_WRITE_BAR0_REG(adapter,
2290 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
2291 adapter->rx_queue[i].rx_ring[0].next2fill);
2292 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
2293 (i * VMXNET3_REG_ALIGN)),
2294 adapter->rx_queue[i].rx_ring[1].next2fill);
2295 }
2296
2297 /* Apply the rx filter settins last. */
2298 vmxnet3_set_mc(adapter->netdev);
2299
2300 /*
2301 * Check link state when first activating device. It will start the
2302 * tx queue if the link is up.
2303 */
2304 vmxnet3_check_link(adapter, true);
2305 for (i = 0; i < adapter->num_rx_queues; i++)
2306 napi_enable(&adapter->rx_queue[i].napi);
2307 vmxnet3_enable_all_intrs(adapter);
2308 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2309 return 0;
2310
2311 activate_err:
2312 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2313 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2314 vmxnet3_free_irqs(adapter);
2315 irq_err:
2316 rq_err:
2317 /* free up buffers we allocated */
2318 vmxnet3_rq_cleanup_all(adapter);
2319 return err;
2320 }
2321
2322
2323 void
2324 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2325 {
2326 unsigned long flags;
2327 spin_lock_irqsave(&adapter->cmd_lock, flags);
2328 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2329 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2330 }
2331
2332
2333 int
2334 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2335 {
2336 int i;
2337 unsigned long flags;
2338 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2339 return 0;
2340
2341
2342 spin_lock_irqsave(&adapter->cmd_lock, flags);
2343 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2344 VMXNET3_CMD_QUIESCE_DEV);
2345 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2346 vmxnet3_disable_all_intrs(adapter);
2347
2348 for (i = 0; i < adapter->num_rx_queues; i++)
2349 napi_disable(&adapter->rx_queue[i].napi);
2350 netif_tx_disable(adapter->netdev);
2351 adapter->link_speed = 0;
2352 netif_carrier_off(adapter->netdev);
2353
2354 vmxnet3_tq_cleanup_all(adapter);
2355 vmxnet3_rq_cleanup_all(adapter);
2356 vmxnet3_free_irqs(adapter);
2357 return 0;
2358 }
2359
2360
2361 static void
2362 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2363 {
2364 u32 tmp;
2365
2366 tmp = *(u32 *)mac;
2367 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2368
2369 tmp = (mac[5] << 8) | mac[4];
2370 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2371 }
2372
2373
2374 static int
2375 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2376 {
2377 struct sockaddr *addr = p;
2378 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2379
2380 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2381 vmxnet3_write_mac_addr(adapter, addr->sa_data);
2382
2383 return 0;
2384 }
2385
2386
2387 /* ==================== initialization and cleanup routines ============ */
2388
2389 static int
2390 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
2391 {
2392 int err;
2393 unsigned long mmio_start, mmio_len;
2394 struct pci_dev *pdev = adapter->pdev;
2395
2396 err = pci_enable_device(pdev);
2397 if (err) {
2398 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2399 return err;
2400 }
2401
2402 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
2403 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
2404 dev_err(&pdev->dev,
2405 "pci_set_consistent_dma_mask failed\n");
2406 err = -EIO;
2407 goto err_set_mask;
2408 }
2409 *dma64 = true;
2410 } else {
2411 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
2412 dev_err(&pdev->dev,
2413 "pci_set_dma_mask failed\n");
2414 err = -EIO;
2415 goto err_set_mask;
2416 }
2417 *dma64 = false;
2418 }
2419
2420 err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2421 vmxnet3_driver_name);
2422 if (err) {
2423 dev_err(&pdev->dev,
2424 "Failed to request region for adapter: error %d\n", err);
2425 goto err_set_mask;
2426 }
2427
2428 pci_set_master(pdev);
2429
2430 mmio_start = pci_resource_start(pdev, 0);
2431 mmio_len = pci_resource_len(pdev, 0);
2432 adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2433 if (!adapter->hw_addr0) {
2434 dev_err(&pdev->dev, "Failed to map bar0\n");
2435 err = -EIO;
2436 goto err_ioremap;
2437 }
2438
2439 mmio_start = pci_resource_start(pdev, 1);
2440 mmio_len = pci_resource_len(pdev, 1);
2441 adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
2442 if (!adapter->hw_addr1) {
2443 dev_err(&pdev->dev, "Failed to map bar1\n");
2444 err = -EIO;
2445 goto err_bar1;
2446 }
2447 return 0;
2448
2449 err_bar1:
2450 iounmap(adapter->hw_addr0);
2451 err_ioremap:
2452 pci_release_selected_regions(pdev, (1 << 2) - 1);
2453 err_set_mask:
2454 pci_disable_device(pdev);
2455 return err;
2456 }
2457
2458
2459 static void
2460 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
2461 {
2462 BUG_ON(!adapter->pdev);
2463
2464 iounmap(adapter->hw_addr0);
2465 iounmap(adapter->hw_addr1);
2466 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
2467 pci_disable_device(adapter->pdev);
2468 }
2469
2470
2471 static void
2472 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
2473 {
2474 size_t sz, i, ring0_size, ring1_size, comp_size;
2475 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];
2476
2477
2478 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
2479 VMXNET3_MAX_ETH_HDR_SIZE) {
2480 adapter->skb_buf_size = adapter->netdev->mtu +
2481 VMXNET3_MAX_ETH_HDR_SIZE;
2482 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
2483 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
2484
2485 adapter->rx_buf_per_pkt = 1;
2486 } else {
2487 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
2488 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
2489 VMXNET3_MAX_ETH_HDR_SIZE;
2490 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
2491 }
2492
2493 /*
2494 * for simplicity, force the ring0 size to be a multiple of
2495 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
2496 */
2497 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
2498 ring0_size = adapter->rx_queue[0].rx_ring[0].size;
2499 ring0_size = (ring0_size + sz - 1) / sz * sz;
2500 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
2501 sz * sz);
2502 ring1_size = adapter->rx_queue[0].rx_ring[1].size;
2503 comp_size = ring0_size + ring1_size;
2504
2505 for (i = 0; i < adapter->num_rx_queues; i++) {
2506 rq = &adapter->rx_queue[i];
2507 rq->rx_ring[0].size = ring0_size;
2508 rq->rx_ring[1].size = ring1_size;
2509 rq->comp_ring.size = comp_size;
2510 }
2511 }
2512
2513
2514 int
2515 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
2516 u32 rx_ring_size, u32 rx_ring2_size)
2517 {
2518 int err = 0, i;
2519
2520 for (i = 0; i < adapter->num_tx_queues; i++) {
2521 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
2522 tq->tx_ring.size = tx_ring_size;
2523 tq->data_ring.size = tx_ring_size;
2524 tq->comp_ring.size = tx_ring_size;
2525 tq->shared = &adapter->tqd_start[i].ctrl;
2526 tq->stopped = true;
2527 tq->adapter = adapter;
2528 tq->qid = i;
2529 err = vmxnet3_tq_create(tq, adapter);
2530 /*
2531 * Too late to change num_tx_queues. We cannot do away with
2532 * lesser number of queues than what we asked for
2533 */
2534 if (err)
2535 goto queue_err;
2536 }
2537
2538 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
2539 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
2540 vmxnet3_adjust_rx_ring_size(adapter);
2541 for (i = 0; i < adapter->num_rx_queues; i++) {
2542 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2543 /* qid and qid2 for rx queues will be assigned later when num
2544 * of rx queues is finalized after allocating intrs */
2545 rq->shared = &adapter->rqd_start[i].ctrl;
2546 rq->adapter = adapter;
2547 err = vmxnet3_rq_create(rq, adapter);
2548 if (err) {
2549 if (i == 0) {
2550 netdev_err(adapter->netdev,
2551 "Could not allocate any rx queues. "
2552 "Aborting.\n");
2553 goto queue_err;
2554 } else {
2555 netdev_info(adapter->netdev,
2556 "Number of rx queues changed "
2557 "to : %d.\n", i);
2558 adapter->num_rx_queues = i;
2559 err = 0;
2560 break;
2561 }
2562 }
2563 }
2564 return err;
2565 queue_err:
2566 vmxnet3_tq_destroy_all(adapter);
2567 return err;
2568 }
2569
2570 static int
2571 vmxnet3_open(struct net_device *netdev)
2572 {
2573 struct vmxnet3_adapter *adapter;
2574 int err, i;
2575
2576 adapter = netdev_priv(netdev);
2577
2578 for (i = 0; i < adapter->num_tx_queues; i++)
2579 spin_lock_init(&adapter->tx_queue[i].tx_lock);
2580
2581 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
2582 VMXNET3_DEF_RX_RING_SIZE,
2583 VMXNET3_DEF_RX_RING_SIZE);
2584 if (err)
2585 goto queue_err;
2586
2587 err = vmxnet3_activate_dev(adapter);
2588 if (err)
2589 goto activate_err;
2590
2591 return 0;
2592
2593 activate_err:
2594 vmxnet3_rq_destroy_all(adapter);
2595 vmxnet3_tq_destroy_all(adapter);
2596 queue_err:
2597 return err;
2598 }
2599
2600
2601 static int
2602 vmxnet3_close(struct net_device *netdev)
2603 {
2604 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2605
2606 /*
2607 * Reset_work may be in the middle of resetting the device, wait for its
2608 * completion.
2609 */
2610 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2611 msleep(1);
2612
2613 vmxnet3_quiesce_dev(adapter);
2614
2615 vmxnet3_rq_destroy_all(adapter);
2616 vmxnet3_tq_destroy_all(adapter);
2617
2618 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2619
2620
2621 return 0;
2622 }
2623
2624
2625 void
2626 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
2627 {
2628 int i;
2629
2630 /*
2631 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
2632 * vmxnet3_close() will deadlock.
2633 */
2634 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
2635
2636 /* we need to enable NAPI, otherwise dev_close will deadlock */
2637 for (i = 0; i < adapter->num_rx_queues; i++)
2638 napi_enable(&adapter->rx_queue[i].napi);
2639 dev_close(adapter->netdev);
2640 }
2641
2642
2643 static int
2644 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
2645 {
2646 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2647 int err = 0;
2648
2649 if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
2650 return -EINVAL;
2651
2652 netdev->mtu = new_mtu;
2653
2654 /*
2655 * Reset_work may be in the middle of resetting the device, wait for its
2656 * completion.
2657 */
2658 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2659 msleep(1);
2660
2661 if (netif_running(netdev)) {
2662 vmxnet3_quiesce_dev(adapter);
2663 vmxnet3_reset_dev(adapter);
2664
2665 /* we need to re-create the rx queue based on the new mtu */
2666 vmxnet3_rq_destroy_all(adapter);
2667 vmxnet3_adjust_rx_ring_size(adapter);
2668 err = vmxnet3_rq_create_all(adapter);
2669 if (err) {
2670 netdev_err(netdev,
2671 "failed to re-create rx queues, "
2672 " error %d. Closing it.\n", err);
2673 goto out;
2674 }
2675
2676 err = vmxnet3_activate_dev(adapter);
2677 if (err) {
2678 netdev_err(netdev,
2679 "failed to re-activate, error %d. "
2680 "Closing it\n", err);
2681 goto out;
2682 }
2683 }
2684
2685 out:
2686 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2687 if (err)
2688 vmxnet3_force_close(adapter);
2689
2690 return err;
2691 }
2692
2693
2694 static void
2695 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
2696 {
2697 struct net_device *netdev = adapter->netdev;
2698
2699 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
2700 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
2701 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
2702 NETIF_F_LRO;
2703 if (dma64)
2704 netdev->hw_features |= NETIF_F_HIGHDMA;
2705 netdev->vlan_features = netdev->hw_features &
2706 ~(NETIF_F_HW_VLAN_CTAG_TX |
2707 NETIF_F_HW_VLAN_CTAG_RX);
2708 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
2709 }
2710
2711
2712 static void
2713 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
2714 {
2715 u32 tmp;
2716
2717 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
2718 *(u32 *)mac = tmp;
2719
2720 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
2721 mac[4] = tmp & 0xff;
2722 mac[5] = (tmp >> 8) & 0xff;
2723 }
2724
2725 #ifdef CONFIG_PCI_MSI
2726
2727 /*
2728 * Enable MSIx vectors.
2729 * Returns :
2730 * 0 on successful enabling of required vectors,
2731 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
2732 * could be enabled.
2733 * number of vectors which can be enabled otherwise (this number is smaller
2734 * than VMXNET3_LINUX_MIN_MSIX_VECT)
2735 */
2736
2737 static int
2738 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
2739 int vectors)
2740 {
2741 int err = 0, vector_threshold;
2742 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
2743
2744 while (vectors >= vector_threshold) {
2745 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
2746 vectors);
2747 if (!err) {
2748 adapter->intr.num_intrs = vectors;
2749 return 0;
2750 } else if (err < 0) {
2751 dev_err(&adapter->netdev->dev,
2752 "Failed to enable MSI-X, error: %d\n", err);
2753 vectors = 0;
2754 } else if (err < vector_threshold) {
2755 break;
2756 } else {
2757 /* If fails to enable required number of MSI-x vectors
2758 * try enabling minimum number of vectors required.
2759 */
2760 dev_err(&adapter->netdev->dev,
2761 "Failed to enable %d MSI-X, trying %d instead\n",
2762 vectors, vector_threshold);
2763 vectors = vector_threshold;
2764 }
2765 }
2766
2767 dev_info(&adapter->pdev->dev,
2768 "Number of MSI-X interrupts which can be allocated "
2769 "is lower than min threshold required.\n");
2770 return err;
2771 }
2772
2773
2774 #endif /* CONFIG_PCI_MSI */
2775
2776 static void
2777 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
2778 {
2779 u32 cfg;
2780 unsigned long flags;
2781
2782 /* intr settings */
2783 spin_lock_irqsave(&adapter->cmd_lock, flags);
2784 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2785 VMXNET3_CMD_GET_CONF_INTR);
2786 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2787 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2788 adapter->intr.type = cfg & 0x3;
2789 adapter->intr.mask_mode = (cfg >> 2) & 0x3;
2790
2791 if (adapter->intr.type == VMXNET3_IT_AUTO) {
2792 adapter->intr.type = VMXNET3_IT_MSIX;
2793 }
2794
2795 #ifdef CONFIG_PCI_MSI
2796 if (adapter->intr.type == VMXNET3_IT_MSIX) {
2797 int vector, err = 0;
2798
2799 adapter->intr.num_intrs = (adapter->share_intr ==
2800 VMXNET3_INTR_TXSHARE) ? 1 :
2801 adapter->num_tx_queues;
2802 adapter->intr.num_intrs += (adapter->share_intr ==
2803 VMXNET3_INTR_BUDDYSHARE) ? 0 :
2804 adapter->num_rx_queues;
2805 adapter->intr.num_intrs += 1; /* for link event */
2806
2807 adapter->intr.num_intrs = (adapter->intr.num_intrs >
2808 VMXNET3_LINUX_MIN_MSIX_VECT
2809 ? adapter->intr.num_intrs :
2810 VMXNET3_LINUX_MIN_MSIX_VECT);
2811
2812 for (vector = 0; vector < adapter->intr.num_intrs; vector++)
2813 adapter->intr.msix_entries[vector].entry = vector;
2814
2815 err = vmxnet3_acquire_msix_vectors(adapter,
2816 adapter->intr.num_intrs);
2817 /* If we cannot allocate one MSIx vector per queue
2818 * then limit the number of rx queues to 1
2819 */
2820 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
2821 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
2822 || adapter->num_rx_queues != 1) {
2823 adapter->share_intr = VMXNET3_INTR_TXSHARE;
2824 netdev_err(adapter->netdev,
2825 "Number of rx queues : 1\n");
2826 adapter->num_rx_queues = 1;
2827 adapter->intr.num_intrs =
2828 VMXNET3_LINUX_MIN_MSIX_VECT;
2829 }
2830 return;
2831 }
2832 if (!err)
2833 return;
2834
2835 /* If we cannot allocate MSIx vectors use only one rx queue */
2836 dev_info(&adapter->pdev->dev,
2837 "Failed to enable MSI-X, error %d. "
2838 "Limiting #rx queues to 1, try MSI.\n", err);
2839
2840 adapter->intr.type = VMXNET3_IT_MSI;
2841 }
2842
2843 if (adapter->intr.type == VMXNET3_IT_MSI) {
2844 int err;
2845 err = pci_enable_msi(adapter->pdev);
2846 if (!err) {
2847 adapter->num_rx_queues = 1;
2848 adapter->intr.num_intrs = 1;
2849 return;
2850 }
2851 }
2852 #endif /* CONFIG_PCI_MSI */
2853
2854 adapter->num_rx_queues = 1;
2855 dev_info(&adapter->netdev->dev,
2856 "Using INTx interrupt, #Rx queues: 1.\n");
2857 adapter->intr.type = VMXNET3_IT_INTX;
2858
2859 /* INT-X related setting */
2860 adapter->intr.num_intrs = 1;
2861 }
2862
2863
2864 static void
2865 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
2866 {
2867 if (adapter->intr.type == VMXNET3_IT_MSIX)
2868 pci_disable_msix(adapter->pdev);
2869 else if (adapter->intr.type == VMXNET3_IT_MSI)
2870 pci_disable_msi(adapter->pdev);
2871 else
2872 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
2873 }
2874
2875
2876 static void
2877 vmxnet3_tx_timeout(struct net_device *netdev)
2878 {
2879 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2880 adapter->tx_timeout_count++;
2881
2882 netdev_err(adapter->netdev, "tx hang\n");
2883 schedule_work(&adapter->work);
2884 netif_wake_queue(adapter->netdev);
2885 }
2886
2887
2888 static void
2889 vmxnet3_reset_work(struct work_struct *data)
2890 {
2891 struct vmxnet3_adapter *adapter;
2892
2893 adapter = container_of(data, struct vmxnet3_adapter, work);
2894
2895 /* if another thread is resetting the device, no need to proceed */
2896 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
2897 return;
2898
2899 /* if the device is closed, we must leave it alone */
2900 rtnl_lock();
2901 if (netif_running(adapter->netdev)) {
2902 netdev_notice(adapter->netdev, "resetting\n");
2903 vmxnet3_quiesce_dev(adapter);
2904 vmxnet3_reset_dev(adapter);
2905 vmxnet3_activate_dev(adapter);
2906 } else {
2907 netdev_info(adapter->netdev, "already closed\n");
2908 }
2909 rtnl_unlock();
2910
2911 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
2912 }
2913
2914
2915 static int
2916 vmxnet3_probe_device(struct pci_dev *pdev,
2917 const struct pci_device_id *id)
2918 {
2919 static const struct net_device_ops vmxnet3_netdev_ops = {
2920 .ndo_open = vmxnet3_open,
2921 .ndo_stop = vmxnet3_close,
2922 .ndo_start_xmit = vmxnet3_xmit_frame,
2923 .ndo_set_mac_address = vmxnet3_set_mac_addr,
2924 .ndo_change_mtu = vmxnet3_change_mtu,
2925 .ndo_set_features = vmxnet3_set_features,
2926 .ndo_get_stats64 = vmxnet3_get_stats64,
2927 .ndo_tx_timeout = vmxnet3_tx_timeout,
2928 .ndo_set_rx_mode = vmxnet3_set_mc,
2929 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
2930 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
2931 #ifdef CONFIG_NET_POLL_CONTROLLER
2932 .ndo_poll_controller = vmxnet3_netpoll,
2933 #endif
2934 };
2935 int err;
2936 bool dma64 = false; /* stupid gcc */
2937 u32 ver;
2938 struct net_device *netdev;
2939 struct vmxnet3_adapter *adapter;
2940 u8 mac[ETH_ALEN];
2941 int size;
2942 int num_tx_queues;
2943 int num_rx_queues;
2944
2945 if (!pci_msi_enabled())
2946 enable_mq = 0;
2947
2948 #ifdef VMXNET3_RSS
2949 if (enable_mq)
2950 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
2951 (int)num_online_cpus());
2952 else
2953 #endif
2954 num_rx_queues = 1;
2955 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
2956
2957 if (enable_mq)
2958 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
2959 (int)num_online_cpus());
2960 else
2961 num_tx_queues = 1;
2962
2963 num_tx_queues = rounddown_pow_of_two(num_tx_queues);
2964 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
2965 max(num_tx_queues, num_rx_queues));
2966 dev_info(&pdev->dev,
2967 "# of Tx queues : %d, # of Rx queues : %d\n",
2968 num_tx_queues, num_rx_queues);
2969
2970 if (!netdev)
2971 return -ENOMEM;
2972
2973 pci_set_drvdata(pdev, netdev);
2974 adapter = netdev_priv(netdev);
2975 adapter->netdev = netdev;
2976 adapter->pdev = pdev;
2977
2978 spin_lock_init(&adapter->cmd_lock);
2979 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
2980 sizeof(struct vmxnet3_adapter),
2981 PCI_DMA_TODEVICE);
2982 adapter->shared = dma_alloc_coherent(
2983 &adapter->pdev->dev,
2984 sizeof(struct Vmxnet3_DriverShared),
2985 &adapter->shared_pa, GFP_KERNEL);
2986 if (!adapter->shared) {
2987 dev_err(&pdev->dev, "Failed to allocate memory\n");
2988 err = -ENOMEM;
2989 goto err_alloc_shared;
2990 }
2991
2992 adapter->num_rx_queues = num_rx_queues;
2993 adapter->num_tx_queues = num_tx_queues;
2994 adapter->rx_buf_per_pkt = 1;
2995
2996 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
2997 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
2998 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
2999 &adapter->queue_desc_pa,
3000 GFP_KERNEL);
3001
3002 if (!adapter->tqd_start) {
3003 dev_err(&pdev->dev, "Failed to allocate memory\n");
3004 err = -ENOMEM;
3005 goto err_alloc_queue_desc;
3006 }
3007 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3008 adapter->num_tx_queues);
3009
3010 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3011 sizeof(struct Vmxnet3_PMConf),
3012 &adapter->pm_conf_pa,
3013 GFP_KERNEL);
3014 if (adapter->pm_conf == NULL) {
3015 err = -ENOMEM;
3016 goto err_alloc_pm;
3017 }
3018
3019 #ifdef VMXNET3_RSS
3020
3021 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3022 sizeof(struct UPT1_RSSConf),
3023 &adapter->rss_conf_pa,
3024 GFP_KERNEL);
3025 if (adapter->rss_conf == NULL) {
3026 err = -ENOMEM;
3027 goto err_alloc_rss;
3028 }
3029 #endif /* VMXNET3_RSS */
3030
3031 err = vmxnet3_alloc_pci_resources(adapter, &dma64);
3032 if (err < 0)
3033 goto err_alloc_pci;
3034
3035 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3036 if (ver & 1) {
3037 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
3038 } else {
3039 dev_err(&pdev->dev,
3040 "Incompatible h/w version (0x%x) for adapter\n", ver);
3041 err = -EBUSY;
3042 goto err_ver;
3043 }
3044
3045 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3046 if (ver & 1) {
3047 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3048 } else {
3049 dev_err(&pdev->dev,
3050 "Incompatible upt version (0x%x) for adapter\n", ver);
3051 err = -EBUSY;
3052 goto err_ver;
3053 }
3054
3055 SET_NETDEV_DEV(netdev, &pdev->dev);
3056 vmxnet3_declare_features(adapter, dma64);
3057
3058 if (adapter->num_tx_queues == adapter->num_rx_queues)
3059 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3060 else
3061 adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3062
3063 vmxnet3_alloc_intr_resources(adapter);
3064
3065 #ifdef VMXNET3_RSS
3066 if (adapter->num_rx_queues > 1 &&
3067 adapter->intr.type == VMXNET3_IT_MSIX) {
3068 adapter->rss = true;
3069 netdev->hw_features |= NETIF_F_RXHASH;
3070 netdev->features |= NETIF_F_RXHASH;
3071 dev_dbg(&pdev->dev, "RSS is enabled.\n");
3072 } else {
3073 adapter->rss = false;
3074 }
3075 #endif
3076
3077 vmxnet3_read_mac_addr(adapter, mac);
3078 memcpy(netdev->dev_addr, mac, netdev->addr_len);
3079
3080 netdev->netdev_ops = &vmxnet3_netdev_ops;
3081 vmxnet3_set_ethtool_ops(netdev);
3082 netdev->watchdog_timeo = 5 * HZ;
3083
3084 INIT_WORK(&adapter->work, vmxnet3_reset_work);
3085 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3086
3087 if (adapter->intr.type == VMXNET3_IT_MSIX) {
3088 int i;
3089 for (i = 0; i < adapter->num_rx_queues; i++) {
3090 netif_napi_add(adapter->netdev,
3091 &adapter->rx_queue[i].napi,
3092 vmxnet3_poll_rx_only, 64);
3093 }
3094 } else {
3095 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3096 vmxnet3_poll, 64);
3097 }
3098
3099 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3100 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3101
3102 netif_carrier_off(netdev);
3103 err = register_netdev(netdev);
3104
3105 if (err) {
3106 dev_err(&pdev->dev, "Failed to register adapter\n");
3107 goto err_register;
3108 }
3109
3110 vmxnet3_check_link(adapter, false);
3111 return 0;
3112
3113 err_register:
3114 vmxnet3_free_intr_resources(adapter);
3115 err_ver:
3116 vmxnet3_free_pci_resources(adapter);
3117 err_alloc_pci:
3118 #ifdef VMXNET3_RSS
3119 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3120 adapter->rss_conf, adapter->rss_conf_pa);
3121 err_alloc_rss:
3122 #endif
3123 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3124 adapter->pm_conf, adapter->pm_conf_pa);
3125 err_alloc_pm:
3126 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3127 adapter->queue_desc_pa);
3128 err_alloc_queue_desc:
3129 dma_free_coherent(&adapter->pdev->dev,
3130 sizeof(struct Vmxnet3_DriverShared),
3131 adapter->shared, adapter->shared_pa);
3132 err_alloc_shared:
3133 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3134 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3135 pci_set_drvdata(pdev, NULL);
3136 free_netdev(netdev);
3137 return err;
3138 }
3139
3140
3141 static void
3142 vmxnet3_remove_device(struct pci_dev *pdev)
3143 {
3144 struct net_device *netdev = pci_get_drvdata(pdev);
3145 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3146 int size = 0;
3147 int num_rx_queues;
3148
3149 #ifdef VMXNET3_RSS
3150 if (enable_mq)
3151 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3152 (int)num_online_cpus());
3153 else
3154 #endif
3155 num_rx_queues = 1;
3156 num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3157
3158 cancel_work_sync(&adapter->work);
3159
3160 unregister_netdev(netdev);
3161
3162 vmxnet3_free_intr_resources(adapter);
3163 vmxnet3_free_pci_resources(adapter);
3164 #ifdef VMXNET3_RSS
3165 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3166 adapter->rss_conf, adapter->rss_conf_pa);
3167 #endif
3168 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3169 adapter->pm_conf, adapter->pm_conf_pa);
3170
3171 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3172 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3173 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3174 adapter->queue_desc_pa);
3175 dma_free_coherent(&adapter->pdev->dev,
3176 sizeof(struct Vmxnet3_DriverShared),
3177 adapter->shared, adapter->shared_pa);
3178 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3179 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
3180 free_netdev(netdev);
3181 }
3182
3183
3184 #ifdef CONFIG_PM
3185
3186 static int
3187 vmxnet3_suspend(struct device *device)
3188 {
3189 struct pci_dev *pdev = to_pci_dev(device);
3190 struct net_device *netdev = pci_get_drvdata(pdev);
3191 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3192 struct Vmxnet3_PMConf *pmConf;
3193 struct ethhdr *ehdr;
3194 struct arphdr *ahdr;
3195 u8 *arpreq;
3196 struct in_device *in_dev;
3197 struct in_ifaddr *ifa;
3198 unsigned long flags;
3199 int i = 0;
3200
3201 if (!netif_running(netdev))
3202 return 0;
3203
3204 for (i = 0; i < adapter->num_rx_queues; i++)
3205 napi_disable(&adapter->rx_queue[i].napi);
3206
3207 vmxnet3_disable_all_intrs(adapter);
3208 vmxnet3_free_irqs(adapter);
3209 vmxnet3_free_intr_resources(adapter);
3210
3211 netif_device_detach(netdev);
3212 netif_tx_stop_all_queues(netdev);
3213
3214 /* Create wake-up filters. */
3215 pmConf = adapter->pm_conf;
3216 memset(pmConf, 0, sizeof(*pmConf));
3217
3218 if (adapter->wol & WAKE_UCAST) {
3219 pmConf->filters[i].patternSize = ETH_ALEN;
3220 pmConf->filters[i].maskSize = 1;
3221 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
3222 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
3223
3224 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3225 i++;
3226 }
3227
3228 if (adapter->wol & WAKE_ARP) {
3229 in_dev = in_dev_get(netdev);
3230 if (!in_dev)
3231 goto skip_arp;
3232
3233 ifa = (struct in_ifaddr *)in_dev->ifa_list;
3234 if (!ifa)
3235 goto skip_arp;
3236
3237 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
3238 sizeof(struct arphdr) + /* ARP header */
3239 2 * ETH_ALEN + /* 2 Ethernet addresses*/
3240 2 * sizeof(u32); /*2 IPv4 addresses */
3241 pmConf->filters[i].maskSize =
3242 (pmConf->filters[i].patternSize - 1) / 8 + 1;
3243
3244 /* ETH_P_ARP in Ethernet header. */
3245 ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
3246 ehdr->h_proto = htons(ETH_P_ARP);
3247
3248 /* ARPOP_REQUEST in ARP header. */
3249 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
3250 ahdr->ar_op = htons(ARPOP_REQUEST);
3251 arpreq = (u8 *)(ahdr + 1);
3252
3253 /* The Unicast IPv4 address in 'tip' field. */
3254 arpreq += 2 * ETH_ALEN + sizeof(u32);
3255 *(u32 *)arpreq = ifa->ifa_address;
3256
3257 /* The mask for the relevant bits. */
3258 pmConf->filters[i].mask[0] = 0x00;
3259 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
3260 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
3261 pmConf->filters[i].mask[3] = 0x00;
3262 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
3263 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
3264 in_dev_put(in_dev);
3265
3266 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
3267 i++;
3268 }
3269
3270 skip_arp:
3271 if (adapter->wol & WAKE_MAGIC)
3272 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
3273
3274 pmConf->numFilters = i;
3275
3276 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3277 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3278 *pmConf));
3279 adapter->shared->devRead.pmConfDesc.confPA =
3280 cpu_to_le64(adapter->pm_conf_pa);
3281
3282 spin_lock_irqsave(&adapter->cmd_lock, flags);
3283 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3284 VMXNET3_CMD_UPDATE_PMCFG);
3285 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3286
3287 pci_save_state(pdev);
3288 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
3289 adapter->wol);
3290 pci_disable_device(pdev);
3291 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
3292
3293 return 0;
3294 }
3295
3296
3297 static int
3298 vmxnet3_resume(struct device *device)
3299 {
3300 int err, i = 0;
3301 unsigned long flags;
3302 struct pci_dev *pdev = to_pci_dev(device);
3303 struct net_device *netdev = pci_get_drvdata(pdev);
3304 struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3305 struct Vmxnet3_PMConf *pmConf;
3306
3307 if (!netif_running(netdev))
3308 return 0;
3309
3310 /* Destroy wake-up filters. */
3311 pmConf = adapter->pm_conf;
3312 memset(pmConf, 0, sizeof(*pmConf));
3313
3314 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
3315 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
3316 *pmConf));
3317 adapter->shared->devRead.pmConfDesc.confPA =
3318 cpu_to_le64(adapter->pm_conf_pa);
3319
3320 netif_device_attach(netdev);
3321 pci_set_power_state(pdev, PCI_D0);
3322 pci_restore_state(pdev);
3323 err = pci_enable_device_mem(pdev);
3324 if (err != 0)
3325 return err;
3326
3327 pci_enable_wake(pdev, PCI_D0, 0);
3328
3329 spin_lock_irqsave(&adapter->cmd_lock, flags);
3330 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3331 VMXNET3_CMD_UPDATE_PMCFG);
3332 spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3333 vmxnet3_alloc_intr_resources(adapter);
3334 vmxnet3_request_irqs(adapter);
3335 for (i = 0; i < adapter->num_rx_queues; i++)
3336 napi_enable(&adapter->rx_queue[i].napi);
3337 vmxnet3_enable_all_intrs(adapter);
3338
3339 return 0;
3340 }
3341
3342 static const struct dev_pm_ops vmxnet3_pm_ops = {
3343 .suspend = vmxnet3_suspend,
3344 .resume = vmxnet3_resume,
3345 };
3346 #endif
3347
3348 static struct pci_driver vmxnet3_driver = {
3349 .name = vmxnet3_driver_name,
3350 .id_table = vmxnet3_pciid_table,
3351 .probe = vmxnet3_probe_device,
3352 .remove = vmxnet3_remove_device,
3353 #ifdef CONFIG_PM
3354 .driver.pm = &vmxnet3_pm_ops,
3355 #endif
3356 };
3357
3358
3359 static int __init
3360 vmxnet3_init_module(void)
3361 {
3362 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
3363 VMXNET3_DRIVER_VERSION_REPORT);
3364 return pci_register_driver(&vmxnet3_driver);
3365 }
3366
3367 module_init(vmxnet3_init_module);
3368
3369
3370 static void
3371 vmxnet3_exit_module(void)
3372 {
3373 pci_unregister_driver(&vmxnet3_driver);
3374 }
3375
3376 module_exit(vmxnet3_exit_module);
3377
3378 MODULE_AUTHOR("VMware, Inc.");
3379 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
3380 MODULE_LICENSE("GPL v2");
3381 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
Linux with added FPC-III support