1 /****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2011 Solarflare Communications Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
11 #include <linux/socket.h>
13 #include <linux/slab.h>
15 #include <linux/tcp.h>
16 #include <linux/udp.h>
18 #include <net/checksum.h>
19 #include "net_driver.h"
23 #include "workarounds.h"
25 /* Number of RX descriptors pushed at once. */
26 #define EFX_RX_BATCH 8
28 /* Maximum size of a buffer sharing a page */
29 #define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state))
31 /* Size of buffer allocated for skb header area. */
32 #define EFX_SKB_HEADERS 64u
35 * rx_alloc_method - RX buffer allocation method
37 * This driver supports two methods for allocating and using RX buffers:
38 * each RX buffer may be backed by an skb or by an order-n page.
40 * When GRO is in use then the second method has a lower overhead,
41 * since we don't have to allocate then free skbs on reassembled frames.
44 * - RX_ALLOC_METHOD_AUTO = 0
45 * - RX_ALLOC_METHOD_SKB = 1
46 * - RX_ALLOC_METHOD_PAGE = 2
48 * The heuristic for %RX_ALLOC_METHOD_AUTO is a simple hysteresis count
49 * controlled by the parameters below.
51 * - Since pushing and popping descriptors are separated by the rx_queue
52 * size, so the watermarks should be ~rxd_size.
53 * - The performance win by using page-based allocation for GRO is less
54 * than the performance hit of using page-based allocation of non-GRO,
55 * so the watermarks should reflect this.
57 * Per channel we maintain a single variable, updated by each channel:
59 * rx_alloc_level += (gro_performed ? RX_ALLOC_FACTOR_GRO :
60 * RX_ALLOC_FACTOR_SKB)
61 * Per NAPI poll interval, we constrain rx_alloc_level to 0..MAX (which
62 * limits the hysteresis), and update the allocation strategy:
64 * rx_alloc_method = (rx_alloc_level > RX_ALLOC_LEVEL_GRO ?
65 * RX_ALLOC_METHOD_PAGE : RX_ALLOC_METHOD_SKB)
67 static int rx_alloc_method
= RX_ALLOC_METHOD_AUTO
;
69 #define RX_ALLOC_LEVEL_GRO 0x2000
70 #define RX_ALLOC_LEVEL_MAX 0x3000
71 #define RX_ALLOC_FACTOR_GRO 1
72 #define RX_ALLOC_FACTOR_SKB (-2)
74 /* This is the percentage fill level below which new RX descriptors
75 * will be added to the RX descriptor ring.
77 static unsigned int rx_refill_threshold
= 90;
79 /* This is the percentage fill level to which an RX queue will be refilled
80 * when the "RX refill threshold" is reached.
82 static unsigned int rx_refill_limit
= 95;
85 * RX maximum head room required.
87 * This must be at least 1 to prevent overflow and at least 2 to allow
90 #define EFX_RXD_HEAD_ROOM 2
92 /* Offset of ethernet header within page */
93 static inline unsigned int efx_rx_buf_offset(struct efx_nic
*efx
,
94 struct efx_rx_buffer
*buf
)
96 /* Offset is always within one page, so we don't need to consider
99 return (((__force
unsigned long) buf
->dma_addr
& (PAGE_SIZE
- 1)) +
100 efx
->type
->rx_buffer_hash_size
);
102 static inline unsigned int efx_rx_buf_size(struct efx_nic
*efx
)
104 return PAGE_SIZE
<< efx
->rx_buffer_order
;
107 static u8
*efx_rx_buf_eh(struct efx_nic
*efx
, struct efx_rx_buffer
*buf
)
110 return page_address(buf
->u
.page
) + efx_rx_buf_offset(efx
, buf
);
112 return ((u8
*)buf
->u
.skb
->data
+
113 efx
->type
->rx_buffer_hash_size
);
116 static inline u32
efx_rx_buf_hash(const u8
*eh
)
118 /* The ethernet header is always directly after any hash. */
119 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0
120 return __le32_to_cpup((const __le32
*)(eh
- 4));
122 const u8
*data
= eh
- 4;
123 return ((u32
)data
[0] |
131 * efx_init_rx_buffers_skb - create EFX_RX_BATCH skb-based RX buffers
133 * @rx_queue: Efx RX queue
135 * This allocates EFX_RX_BATCH skbs, maps them for DMA, and populates a
136 * struct efx_rx_buffer for each one. Return a negative error code or 0
137 * on success. May fail having only inserted fewer than EFX_RX_BATCH
140 static int efx_init_rx_buffers_skb(struct efx_rx_queue
*rx_queue
)
142 struct efx_nic
*efx
= rx_queue
->efx
;
143 struct net_device
*net_dev
= efx
->net_dev
;
144 struct efx_rx_buffer
*rx_buf
;
146 int skb_len
= efx
->rx_buffer_len
;
147 unsigned index
, count
;
149 for (count
= 0; count
< EFX_RX_BATCH
; ++count
) {
150 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
151 rx_buf
= efx_rx_buffer(rx_queue
, index
);
153 rx_buf
->u
.skb
= skb
= netdev_alloc_skb(net_dev
, skb_len
);
157 /* Adjust the SKB for padding and checksum */
158 skb_reserve(skb
, NET_IP_ALIGN
);
159 rx_buf
->len
= skb_len
- NET_IP_ALIGN
;
160 rx_buf
->is_page
= false;
161 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
163 rx_buf
->dma_addr
= pci_map_single(efx
->pci_dev
,
164 skb
->data
, rx_buf
->len
,
166 if (unlikely(pci_dma_mapping_error(efx
->pci_dev
,
167 rx_buf
->dma_addr
))) {
168 dev_kfree_skb_any(skb
);
169 rx_buf
->u
.skb
= NULL
;
173 ++rx_queue
->added_count
;
174 ++rx_queue
->alloc_skb_count
;
181 * efx_init_rx_buffers_page - create EFX_RX_BATCH page-based RX buffers
183 * @rx_queue: Efx RX queue
185 * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA,
186 * and populates struct efx_rx_buffers for each one. Return a negative error
187 * code or 0 on success. If a single page can be split between two buffers,
188 * then the page will either be inserted fully, or not at at all.
190 static int efx_init_rx_buffers_page(struct efx_rx_queue
*rx_queue
)
192 struct efx_nic
*efx
= rx_queue
->efx
;
193 struct efx_rx_buffer
*rx_buf
;
196 struct efx_rx_page_state
*state
;
198 unsigned index
, count
;
200 /* We can split a page between two buffers */
201 BUILD_BUG_ON(EFX_RX_BATCH
& 1);
203 for (count
= 0; count
< EFX_RX_BATCH
; ++count
) {
204 page
= alloc_pages(__GFP_COLD
| __GFP_COMP
| GFP_ATOMIC
,
205 efx
->rx_buffer_order
);
206 if (unlikely(page
== NULL
))
208 dma_addr
= pci_map_page(efx
->pci_dev
, page
, 0,
209 efx_rx_buf_size(efx
),
211 if (unlikely(pci_dma_mapping_error(efx
->pci_dev
, dma_addr
))) {
212 __free_pages(page
, efx
->rx_buffer_order
);
215 page_addr
= page_address(page
);
218 state
->dma_addr
= dma_addr
;
220 page_addr
+= sizeof(struct efx_rx_page_state
);
221 dma_addr
+= sizeof(struct efx_rx_page_state
);
224 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
225 rx_buf
= efx_rx_buffer(rx_queue
, index
);
226 rx_buf
->dma_addr
= dma_addr
+ EFX_PAGE_IP_ALIGN
;
227 rx_buf
->u
.page
= page
;
228 rx_buf
->len
= efx
->rx_buffer_len
- EFX_PAGE_IP_ALIGN
;
229 rx_buf
->is_page
= true;
230 ++rx_queue
->added_count
;
231 ++rx_queue
->alloc_page_count
;
234 if ((~count
& 1) && (efx
->rx_buffer_len
<= EFX_RX_HALF_PAGE
)) {
235 /* Use the second half of the page */
237 dma_addr
+= (PAGE_SIZE
>> 1);
238 page_addr
+= (PAGE_SIZE
>> 1);
247 static void efx_unmap_rx_buffer(struct efx_nic
*efx
,
248 struct efx_rx_buffer
*rx_buf
)
250 if (rx_buf
->is_page
&& rx_buf
->u
.page
) {
251 struct efx_rx_page_state
*state
;
253 state
= page_address(rx_buf
->u
.page
);
254 if (--state
->refcnt
== 0) {
255 pci_unmap_page(efx
->pci_dev
,
257 efx_rx_buf_size(efx
),
260 } else if (!rx_buf
->is_page
&& rx_buf
->u
.skb
) {
261 pci_unmap_single(efx
->pci_dev
, rx_buf
->dma_addr
,
262 rx_buf
->len
, PCI_DMA_FROMDEVICE
);
266 static void efx_free_rx_buffer(struct efx_nic
*efx
,
267 struct efx_rx_buffer
*rx_buf
)
269 if (rx_buf
->is_page
&& rx_buf
->u
.page
) {
270 __free_pages(rx_buf
->u
.page
, efx
->rx_buffer_order
);
271 rx_buf
->u
.page
= NULL
;
272 } else if (!rx_buf
->is_page
&& rx_buf
->u
.skb
) {
273 dev_kfree_skb_any(rx_buf
->u
.skb
);
274 rx_buf
->u
.skb
= NULL
;
278 static void efx_fini_rx_buffer(struct efx_rx_queue
*rx_queue
,
279 struct efx_rx_buffer
*rx_buf
)
281 efx_unmap_rx_buffer(rx_queue
->efx
, rx_buf
);
282 efx_free_rx_buffer(rx_queue
->efx
, rx_buf
);
285 /* Attempt to resurrect the other receive buffer that used to share this page,
286 * which had previously been passed up to the kernel and freed. */
287 static void efx_resurrect_rx_buffer(struct efx_rx_queue
*rx_queue
,
288 struct efx_rx_buffer
*rx_buf
)
290 struct efx_rx_page_state
*state
= page_address(rx_buf
->u
.page
);
291 struct efx_rx_buffer
*new_buf
;
292 unsigned fill_level
, index
;
294 /* +1 because efx_rx_packet() incremented removed_count. +1 because
295 * we'd like to insert an additional descriptor whilst leaving
296 * EFX_RXD_HEAD_ROOM for the non-recycle path */
297 fill_level
= (rx_queue
->added_count
- rx_queue
->removed_count
+ 2);
298 if (unlikely(fill_level
> rx_queue
->max_fill
)) {
299 /* We could place "state" on a list, and drain the list in
300 * efx_fast_push_rx_descriptors(). For now, this will do. */
305 get_page(rx_buf
->u
.page
);
307 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
308 new_buf
= efx_rx_buffer(rx_queue
, index
);
309 new_buf
->dma_addr
= rx_buf
->dma_addr
^ (PAGE_SIZE
>> 1);
310 new_buf
->u
.page
= rx_buf
->u
.page
;
311 new_buf
->len
= rx_buf
->len
;
312 new_buf
->is_page
= true;
313 ++rx_queue
->added_count
;
316 /* Recycle the given rx buffer directly back into the rx_queue. There is
317 * always room to add this buffer, because we've just popped a buffer. */
318 static void efx_recycle_rx_buffer(struct efx_channel
*channel
,
319 struct efx_rx_buffer
*rx_buf
)
321 struct efx_nic
*efx
= channel
->efx
;
322 struct efx_rx_queue
*rx_queue
= efx_channel_get_rx_queue(channel
);
323 struct efx_rx_buffer
*new_buf
;
326 if (rx_buf
->is_page
&& efx
->rx_buffer_len
<= EFX_RX_HALF_PAGE
&&
327 page_count(rx_buf
->u
.page
) == 1)
328 efx_resurrect_rx_buffer(rx_queue
, rx_buf
);
330 index
= rx_queue
->added_count
& rx_queue
->ptr_mask
;
331 new_buf
= efx_rx_buffer(rx_queue
, index
);
333 memcpy(new_buf
, rx_buf
, sizeof(*new_buf
));
334 rx_buf
->u
.page
= NULL
;
335 ++rx_queue
->added_count
;
339 * efx_fast_push_rx_descriptors - push new RX descriptors quickly
340 * @rx_queue: RX descriptor queue
341 * This will aim to fill the RX descriptor queue up to
342 * @rx_queue->@fast_fill_limit. If there is insufficient atomic
343 * memory to do so, a slow fill will be scheduled.
345 * The caller must provide serialisation (none is used here). In practise,
346 * this means this function must run from the NAPI handler, or be called
347 * when NAPI is disabled.
349 void efx_fast_push_rx_descriptors(struct efx_rx_queue
*rx_queue
)
351 struct efx_channel
*channel
= efx_rx_queue_channel(rx_queue
);
355 /* Calculate current fill level, and exit if we don't need to fill */
356 fill_level
= (rx_queue
->added_count
- rx_queue
->removed_count
);
357 EFX_BUG_ON_PARANOID(fill_level
> rx_queue
->efx
->rxq_entries
);
358 if (fill_level
>= rx_queue
->fast_fill_trigger
)
361 /* Record minimum fill level */
362 if (unlikely(fill_level
< rx_queue
->min_fill
)) {
364 rx_queue
->min_fill
= fill_level
;
367 space
= rx_queue
->fast_fill_limit
- fill_level
;
368 if (space
< EFX_RX_BATCH
)
371 netif_vdbg(rx_queue
->efx
, rx_status
, rx_queue
->efx
->net_dev
,
372 "RX queue %d fast-filling descriptor ring from"
373 " level %d to level %d using %s allocation\n",
374 efx_rx_queue_index(rx_queue
), fill_level
,
375 rx_queue
->fast_fill_limit
,
376 channel
->rx_alloc_push_pages
? "page" : "skb");
379 if (channel
->rx_alloc_push_pages
)
380 rc
= efx_init_rx_buffers_page(rx_queue
);
382 rc
= efx_init_rx_buffers_skb(rx_queue
);
384 /* Ensure that we don't leave the rx queue empty */
385 if (rx_queue
->added_count
== rx_queue
->removed_count
)
386 efx_schedule_slow_fill(rx_queue
);
389 } while ((space
-= EFX_RX_BATCH
) >= EFX_RX_BATCH
);
391 netif_vdbg(rx_queue
->efx
, rx_status
, rx_queue
->efx
->net_dev
,
392 "RX queue %d fast-filled descriptor ring "
393 "to level %d\n", efx_rx_queue_index(rx_queue
),
394 rx_queue
->added_count
- rx_queue
->removed_count
);
397 if (rx_queue
->notified_count
!= rx_queue
->added_count
)
398 efx_nic_notify_rx_desc(rx_queue
);
401 void efx_rx_slow_fill(unsigned long context
)
403 struct efx_rx_queue
*rx_queue
= (struct efx_rx_queue
*)context
;
404 struct efx_channel
*channel
= efx_rx_queue_channel(rx_queue
);
406 /* Post an event to cause NAPI to run and refill the queue */
407 efx_nic_generate_fill_event(channel
);
408 ++rx_queue
->slow_fill_count
;
411 static void efx_rx_packet__check_len(struct efx_rx_queue
*rx_queue
,
412 struct efx_rx_buffer
*rx_buf
,
413 int len
, bool *discard
,
416 struct efx_nic
*efx
= rx_queue
->efx
;
417 unsigned max_len
= rx_buf
->len
- efx
->type
->rx_buffer_padding
;
419 if (likely(len
<= max_len
))
422 /* The packet must be discarded, but this is only a fatal error
423 * if the caller indicated it was
427 if ((len
> rx_buf
->len
) && EFX_WORKAROUND_8071(efx
)) {
429 netif_err(efx
, rx_err
, efx
->net_dev
,
430 " RX queue %d seriously overlength "
431 "RX event (0x%x > 0x%x+0x%x). Leaking\n",
432 efx_rx_queue_index(rx_queue
), len
, max_len
,
433 efx
->type
->rx_buffer_padding
);
434 /* If this buffer was skb-allocated, then the meta
435 * data at the end of the skb will be trashed. So
436 * we have no choice but to leak the fragment.
438 *leak_packet
= !rx_buf
->is_page
;
439 efx_schedule_reset(efx
, RESET_TYPE_RX_RECOVERY
);
442 netif_err(efx
, rx_err
, efx
->net_dev
,
443 " RX queue %d overlength RX event "
445 efx_rx_queue_index(rx_queue
), len
, max_len
);
448 efx_rx_queue_channel(rx_queue
)->n_rx_overlength
++;
451 /* Pass a received packet up through the generic GRO stack
453 * Handles driverlink veto, and passes the fragment up via
454 * the appropriate GRO method
456 static void efx_rx_packet_gro(struct efx_channel
*channel
,
457 struct efx_rx_buffer
*rx_buf
,
458 const u8
*eh
, bool checksummed
)
460 struct napi_struct
*napi
= &channel
->napi_str
;
461 gro_result_t gro_result
;
463 /* Pass the skb/page into the GRO engine */
464 if (rx_buf
->is_page
) {
465 struct efx_nic
*efx
= channel
->efx
;
466 struct page
*page
= rx_buf
->u
.page
;
469 rx_buf
->u
.page
= NULL
;
471 skb
= napi_get_frags(napi
);
477 if (efx
->net_dev
->features
& NETIF_F_RXHASH
)
478 skb
->rxhash
= efx_rx_buf_hash(eh
);
480 skb_shinfo(skb
)->frags
[0].page
= page
;
481 skb_shinfo(skb
)->frags
[0].page_offset
=
482 efx_rx_buf_offset(efx
, rx_buf
);
483 skb_shinfo(skb
)->frags
[0].size
= rx_buf
->len
;
484 skb_shinfo(skb
)->nr_frags
= 1;
486 skb
->len
= rx_buf
->len
;
487 skb
->data_len
= rx_buf
->len
;
488 skb
->truesize
+= rx_buf
->len
;
490 checksummed
? CHECKSUM_UNNECESSARY
: CHECKSUM_NONE
;
492 skb_record_rx_queue(skb
, channel
->channel
);
494 gro_result
= napi_gro_frags(napi
);
496 struct sk_buff
*skb
= rx_buf
->u
.skb
;
498 EFX_BUG_ON_PARANOID(!checksummed
);
499 rx_buf
->u
.skb
= NULL
;
501 gro_result
= napi_gro_receive(napi
, skb
);
504 if (gro_result
== GRO_NORMAL
) {
505 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_SKB
;
506 } else if (gro_result
!= GRO_DROP
) {
507 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_GRO
;
508 channel
->irq_mod_score
+= 2;
512 void efx_rx_packet(struct efx_rx_queue
*rx_queue
, unsigned int index
,
513 unsigned int len
, bool checksummed
, bool discard
)
515 struct efx_nic
*efx
= rx_queue
->efx
;
516 struct efx_channel
*channel
= efx_rx_queue_channel(rx_queue
);
517 struct efx_rx_buffer
*rx_buf
;
518 bool leak_packet
= false;
520 rx_buf
= efx_rx_buffer(rx_queue
, index
);
522 /* This allows the refill path to post another buffer.
523 * EFX_RXD_HEAD_ROOM ensures that the slot we are using
524 * isn't overwritten yet.
526 rx_queue
->removed_count
++;
528 /* Validate the length encoded in the event vs the descriptor pushed */
529 efx_rx_packet__check_len(rx_queue
, rx_buf
, len
,
530 &discard
, &leak_packet
);
532 netif_vdbg(efx
, rx_status
, efx
->net_dev
,
533 "RX queue %d received id %x at %llx+%x %s%s\n",
534 efx_rx_queue_index(rx_queue
), index
,
535 (unsigned long long)rx_buf
->dma_addr
, len
,
536 (checksummed
? " [SUMMED]" : ""),
537 (discard
? " [DISCARD]" : ""));
539 /* Discard packet, if instructed to do so */
540 if (unlikely(discard
)) {
541 if (unlikely(leak_packet
))
542 channel
->n_skbuff_leaks
++;
544 efx_recycle_rx_buffer(channel
, rx_buf
);
546 /* Don't hold off the previous receive */
551 /* Release card resources - assumes all RX buffers consumed in-order
554 efx_unmap_rx_buffer(efx
, rx_buf
);
556 /* Prefetch nice and early so data will (hopefully) be in cache by
557 * the time we look at it.
559 prefetch(efx_rx_buf_eh(efx
, rx_buf
));
561 /* Pipeline receives so that we give time for packet headers to be
562 * prefetched into cache.
564 rx_buf
->len
= len
- efx
->type
->rx_buffer_hash_size
;
567 __efx_rx_packet(channel
,
568 channel
->rx_pkt
, channel
->rx_pkt_csummed
);
569 channel
->rx_pkt
= rx_buf
;
570 channel
->rx_pkt_csummed
= checksummed
;
573 /* Handle a received packet. Second half: Touches packet payload. */
574 void __efx_rx_packet(struct efx_channel
*channel
,
575 struct efx_rx_buffer
*rx_buf
, bool checksummed
)
577 struct efx_nic
*efx
= channel
->efx
;
579 u8
*eh
= efx_rx_buf_eh(efx
, rx_buf
);
581 /* If we're in loopback test, then pass the packet directly to the
582 * loopback layer, and free the rx_buf here
584 if (unlikely(efx
->loopback_selftest
)) {
585 efx_loopback_rx_packet(efx
, eh
, rx_buf
->len
);
586 efx_free_rx_buffer(efx
, rx_buf
);
590 if (!rx_buf
->is_page
) {
593 prefetch(skb_shinfo(skb
));
595 skb_reserve(skb
, efx
->type
->rx_buffer_hash_size
);
596 skb_put(skb
, rx_buf
->len
);
598 if (efx
->net_dev
->features
& NETIF_F_RXHASH
)
599 skb
->rxhash
= efx_rx_buf_hash(eh
);
601 /* Move past the ethernet header. rx_buf->data still points
602 * at the ethernet header */
603 skb
->protocol
= eth_type_trans(skb
, efx
->net_dev
);
605 skb_record_rx_queue(skb
, channel
->channel
);
608 if (unlikely(!(efx
->net_dev
->features
& NETIF_F_RXCSUM
)))
611 if (likely(checksummed
|| rx_buf
->is_page
)) {
612 efx_rx_packet_gro(channel
, rx_buf
, eh
, checksummed
);
616 /* We now own the SKB */
618 rx_buf
->u
.skb
= NULL
;
620 /* Set the SKB flags */
621 skb_checksum_none_assert(skb
);
623 /* Pass the packet up */
624 netif_receive_skb(skb
);
626 /* Update allocation strategy method */
627 channel
->rx_alloc_level
+= RX_ALLOC_FACTOR_SKB
;
630 void efx_rx_strategy(struct efx_channel
*channel
)
632 enum efx_rx_alloc_method method
= rx_alloc_method
;
634 /* Only makes sense to use page based allocation if GRO is enabled */
635 if (!(channel
->efx
->net_dev
->features
& NETIF_F_GRO
)) {
636 method
= RX_ALLOC_METHOD_SKB
;
637 } else if (method
== RX_ALLOC_METHOD_AUTO
) {
638 /* Constrain the rx_alloc_level */
639 if (channel
->rx_alloc_level
< 0)
640 channel
->rx_alloc_level
= 0;
641 else if (channel
->rx_alloc_level
> RX_ALLOC_LEVEL_MAX
)
642 channel
->rx_alloc_level
= RX_ALLOC_LEVEL_MAX
;
644 /* Decide on the allocation method */
645 method
= ((channel
->rx_alloc_level
> RX_ALLOC_LEVEL_GRO
) ?
646 RX_ALLOC_METHOD_PAGE
: RX_ALLOC_METHOD_SKB
);
649 /* Push the option */
650 channel
->rx_alloc_push_pages
= (method
== RX_ALLOC_METHOD_PAGE
);
653 int efx_probe_rx_queue(struct efx_rx_queue
*rx_queue
)
655 struct efx_nic
*efx
= rx_queue
->efx
;
656 unsigned int entries
;
659 /* Create the smallest power-of-two aligned ring */
660 entries
= max(roundup_pow_of_two(efx
->rxq_entries
), EFX_MIN_DMAQ_SIZE
);
661 EFX_BUG_ON_PARANOID(entries
> EFX_MAX_DMAQ_SIZE
);
662 rx_queue
->ptr_mask
= entries
- 1;
664 netif_dbg(efx
, probe
, efx
->net_dev
,
665 "creating RX queue %d size %#x mask %#x\n",
666 efx_rx_queue_index(rx_queue
), efx
->rxq_entries
,
669 /* Allocate RX buffers */
670 rx_queue
->buffer
= kzalloc(entries
* sizeof(*rx_queue
->buffer
),
672 if (!rx_queue
->buffer
)
675 rc
= efx_nic_probe_rx(rx_queue
);
677 kfree(rx_queue
->buffer
);
678 rx_queue
->buffer
= NULL
;
683 void efx_init_rx_queue(struct efx_rx_queue
*rx_queue
)
685 struct efx_nic
*efx
= rx_queue
->efx
;
686 unsigned int max_fill
, trigger
, limit
;
688 netif_dbg(rx_queue
->efx
, drv
, rx_queue
->efx
->net_dev
,
689 "initialising RX queue %d\n", efx_rx_queue_index(rx_queue
));
691 /* Initialise ptr fields */
692 rx_queue
->added_count
= 0;
693 rx_queue
->notified_count
= 0;
694 rx_queue
->removed_count
= 0;
695 rx_queue
->min_fill
= -1U;
697 /* Initialise limit fields */
698 max_fill
= efx
->rxq_entries
- EFX_RXD_HEAD_ROOM
;
699 trigger
= max_fill
* min(rx_refill_threshold
, 100U) / 100U;
700 limit
= max_fill
* min(rx_refill_limit
, 100U) / 100U;
702 rx_queue
->max_fill
= max_fill
;
703 rx_queue
->fast_fill_trigger
= trigger
;
704 rx_queue
->fast_fill_limit
= limit
;
706 /* Set up RX descriptor ring */
707 efx_nic_init_rx(rx_queue
);
710 void efx_fini_rx_queue(struct efx_rx_queue
*rx_queue
)
713 struct efx_rx_buffer
*rx_buf
;
715 netif_dbg(rx_queue
->efx
, drv
, rx_queue
->efx
->net_dev
,
716 "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue
));
718 del_timer_sync(&rx_queue
->slow_fill
);
719 efx_nic_fini_rx(rx_queue
);
721 /* Release RX buffers NB start at index 0 not current HW ptr */
722 if (rx_queue
->buffer
) {
723 for (i
= 0; i
<= rx_queue
->ptr_mask
; i
++) {
724 rx_buf
= efx_rx_buffer(rx_queue
, i
);
725 efx_fini_rx_buffer(rx_queue
, rx_buf
);
730 void efx_remove_rx_queue(struct efx_rx_queue
*rx_queue
)
732 netif_dbg(rx_queue
->efx
, drv
, rx_queue
->efx
->net_dev
,
733 "destroying RX queue %d\n", efx_rx_queue_index(rx_queue
));
735 efx_nic_remove_rx(rx_queue
);
737 kfree(rx_queue
->buffer
);
738 rx_queue
->buffer
= NULL
;
742 module_param(rx_alloc_method
, int, 0644);
743 MODULE_PARM_DESC(rx_alloc_method
, "Allocation method used for RX buffers");
745 module_param(rx_refill_threshold
, uint
, 0444);
746 MODULE_PARM_DESC(rx_refill_threshold
,
747 "RX descriptor ring fast/slow fill threshold (%)");