1 // SPDX-License-Identifier: ISC
3 * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
4 * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
7 #include <linux/etherdevice.h>
8 #include <net/ieee80211_radiotap.h>
9 #include <linux/if_arp.h>
10 #include <linux/moduleparam.h>
12 #include <linux/ipv6.h>
13 #include <linux/if_vlan.h>
15 #include <linux/prefetch.h>
21 #include "txrx_edma.h"
24 module_param(rx_align_2
, bool, 0444);
25 MODULE_PARM_DESC(rx_align_2
, " align Rx buffers on 4*n+2, default - no");
28 module_param(rx_large_buf
, bool, 0444);
29 MODULE_PARM_DESC(rx_large_buf
, " allocate 8KB RX buffers, default - no");
31 /* Drop Tx packets in case Tx ring is full */
32 bool drop_if_ring_full
;
34 static inline uint
wil_rx_snaplen(void)
36 return rx_align_2
? 6 : 0;
39 /* wil_ring_wmark_low - low watermark for available descriptor space */
40 static inline int wil_ring_wmark_low(struct wil_ring
*ring
)
42 return ring
->size
/ 8;
45 /* wil_ring_wmark_high - high watermark for available descriptor space */
46 static inline int wil_ring_wmark_high(struct wil_ring
*ring
)
48 return ring
->size
/ 4;
51 /* returns true if num avail descriptors is lower than wmark_low */
52 static inline int wil_ring_avail_low(struct wil_ring
*ring
)
54 return wil_ring_avail_tx(ring
) < wil_ring_wmark_low(ring
);
57 /* returns true if num avail descriptors is higher than wmark_high */
58 static inline int wil_ring_avail_high(struct wil_ring
*ring
)
60 return wil_ring_avail_tx(ring
) > wil_ring_wmark_high(ring
);
63 /* returns true when all tx vrings are empty */
64 bool wil_is_tx_idle(struct wil6210_priv
*wil
)
67 unsigned long data_comp_to
;
68 int min_ring_id
= wil_get_min_tx_ring_id(wil
);
70 for (i
= min_ring_id
; i
< WIL6210_MAX_TX_RINGS
; i
++) {
71 struct wil_ring
*vring
= &wil
->ring_tx
[i
];
72 int vring_index
= vring
- wil
->ring_tx
;
73 struct wil_ring_tx_data
*txdata
=
74 &wil
->ring_tx_data
[vring_index
];
76 spin_lock(&txdata
->lock
);
78 if (!vring
->va
|| !txdata
->enabled
) {
79 spin_unlock(&txdata
->lock
);
83 data_comp_to
= jiffies
+ msecs_to_jiffies(
84 WIL_DATA_COMPLETION_TO_MS
);
85 if (test_bit(wil_status_napi_en
, wil
->status
)) {
86 while (!wil_ring_is_empty(vring
)) {
87 if (time_after(jiffies
, data_comp_to
)) {
89 "TO waiting for idle tx\n");
90 spin_unlock(&txdata
->lock
);
93 wil_dbg_ratelimited(wil
,
94 "tx vring is not empty -> NAPI\n");
95 spin_unlock(&txdata
->lock
);
96 napi_synchronize(&wil
->napi_tx
);
98 spin_lock(&txdata
->lock
);
99 if (!vring
->va
|| !txdata
->enabled
)
104 spin_unlock(&txdata
->lock
);
110 static int wil_vring_alloc(struct wil6210_priv
*wil
, struct wil_ring
*vring
)
112 struct device
*dev
= wil_to_dev(wil
);
113 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
116 wil_dbg_misc(wil
, "vring_alloc:\n");
118 BUILD_BUG_ON(sizeof(vring
->va
[0]) != 32);
122 vring
->ctx
= kcalloc(vring
->size
, sizeof(vring
->ctx
[0]), GFP_KERNEL
);
128 /* vring->va should be aligned on its size rounded up to power of 2
129 * This is granted by the dma_alloc_coherent.
131 * HW has limitation that all vrings addresses must share the same
132 * upper 16 msb bits part of 48 bits address. To workaround that,
133 * if we are using more than 32 bit addresses switch to 32 bit
134 * allocation before allocating vring memory.
136 * There's no check for the return value of dma_set_mask_and_coherent,
137 * since we assume if we were able to set the mask during
138 * initialization in this system it will not fail if we set it again
140 if (wil
->dma_addr_size
> 32)
141 dma_set_mask_and_coherent(dev
, DMA_BIT_MASK(32));
143 vring
->va
= dma_alloc_coherent(dev
, sz
, &vring
->pa
, GFP_KERNEL
);
150 if (wil
->dma_addr_size
> 32)
151 dma_set_mask_and_coherent(dev
,
152 DMA_BIT_MASK(wil
->dma_addr_size
));
154 /* initially, all descriptors are SW owned
155 * For Tx and Rx, ownership bit is at the same location, thus
158 for (i
= 0; i
< vring
->size
; i
++) {
159 volatile struct vring_tx_desc
*_d
=
160 &vring
->va
[i
].tx
.legacy
;
162 _d
->dma
.status
= TX_DMA_STATUS_DU
;
165 wil_dbg_misc(wil
, "vring[%d] 0x%p:%pad 0x%p\n", vring
->size
,
166 vring
->va
, &vring
->pa
, vring
->ctx
);
171 static void wil_txdesc_unmap(struct device
*dev
, union wil_tx_desc
*desc
,
174 struct vring_tx_desc
*d
= &desc
->legacy
;
175 dma_addr_t pa
= wil_desc_addr(&d
->dma
.addr
);
176 u16 dmalen
= le16_to_cpu(d
->dma
.length
);
178 switch (ctx
->mapped_as
) {
179 case wil_mapped_as_single
:
180 dma_unmap_single(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
182 case wil_mapped_as_page
:
183 dma_unmap_page(dev
, pa
, dmalen
, DMA_TO_DEVICE
);
190 static void wil_vring_free(struct wil6210_priv
*wil
, struct wil_ring
*vring
)
192 struct device
*dev
= wil_to_dev(wil
);
193 size_t sz
= vring
->size
* sizeof(vring
->va
[0]);
195 lockdep_assert_held(&wil
->mutex
);
197 int vring_index
= vring
- wil
->ring_tx
;
199 wil_dbg_misc(wil
, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",
200 vring_index
, vring
->size
, vring
->va
,
201 &vring
->pa
, vring
->ctx
);
203 wil_dbg_misc(wil
, "free Rx vring [%d] 0x%p:%pad 0x%p\n",
204 vring
->size
, vring
->va
,
205 &vring
->pa
, vring
->ctx
);
208 while (!wil_ring_is_empty(vring
)) {
214 struct vring_tx_desc dd
, *d
= &dd
;
215 volatile struct vring_tx_desc
*_d
=
216 &vring
->va
[vring
->swtail
].tx
.legacy
;
218 ctx
= &vring
->ctx
[vring
->swtail
];
221 "ctx(%d) was already completed\n",
223 vring
->swtail
= wil_ring_next_tail(vring
);
227 wil_txdesc_unmap(dev
, (union wil_tx_desc
*)d
, ctx
);
229 dev_kfree_skb_any(ctx
->skb
);
230 vring
->swtail
= wil_ring_next_tail(vring
);
232 struct vring_rx_desc dd
, *d
= &dd
;
233 volatile struct vring_rx_desc
*_d
=
234 &vring
->va
[vring
->swhead
].rx
.legacy
;
236 ctx
= &vring
->ctx
[vring
->swhead
];
238 pa
= wil_desc_addr(&d
->dma
.addr
);
239 dmalen
= le16_to_cpu(d
->dma
.length
);
240 dma_unmap_single(dev
, pa
, dmalen
, DMA_FROM_DEVICE
);
242 wil_ring_advance_head(vring
, 1);
245 dma_free_coherent(dev
, sz
, (void *)vring
->va
, vring
->pa
);
252 /* Allocate one skb for Rx VRING
254 * Safe to call from IRQ
256 static int wil_vring_alloc_skb(struct wil6210_priv
*wil
, struct wil_ring
*vring
,
259 struct device
*dev
= wil_to_dev(wil
);
260 unsigned int sz
= wil
->rx_buf_len
+ ETH_HLEN
+ wil_rx_snaplen();
261 struct vring_rx_desc dd
, *d
= &dd
;
262 volatile struct vring_rx_desc
*_d
= &vring
->va
[i
].rx
.legacy
;
264 struct sk_buff
*skb
= dev_alloc_skb(sz
+ headroom
);
269 skb_reserve(skb
, headroom
);
273 * Make sure that the network stack calculates checksum for packets
274 * which failed the HW checksum calculation
276 skb
->ip_summed
= CHECKSUM_NONE
;
278 pa
= dma_map_single(dev
, skb
->data
, skb
->len
, DMA_FROM_DEVICE
);
279 if (unlikely(dma_mapping_error(dev
, pa
))) {
284 d
->dma
.d0
= RX_DMA_D0_CMD_DMA_RT
| RX_DMA_D0_CMD_DMA_IT
;
285 wil_desc_addr_set(&d
->dma
.addr
, pa
);
286 /* ip_length don't care */
288 /* error don't care */
289 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
290 d
->dma
.length
= cpu_to_le16(sz
);
292 vring
->ctx
[i
].skb
= skb
;
297 /* Adds radiotap header
299 * Any error indicated as "Bad FCS"
301 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of:
302 * - Rx descriptor: 32 bytes
305 static void wil_rx_add_radiotap_header(struct wil6210_priv
*wil
,
308 struct wil6210_rtap
{
309 struct ieee80211_radiotap_header_fixed rthdr
;
310 /* fields should be in the order of bits in rthdr.it_present */
314 __le16 chnl_freq
__aligned(2);
321 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
322 struct wil6210_rtap
*rtap
;
323 int rtap_len
= sizeof(struct wil6210_rtap
);
324 struct ieee80211_channel
*ch
= wil
->monitor_chandef
.chan
;
326 if (skb_headroom(skb
) < rtap_len
&&
327 pskb_expand_head(skb
, rtap_len
, 0, GFP_ATOMIC
)) {
328 wil_err(wil
, "Unable to expand headroom to %d\n", rtap_len
);
332 rtap
= skb_push(skb
, rtap_len
);
333 memset(rtap
, 0, rtap_len
);
335 rtap
->rthdr
.it_version
= PKTHDR_RADIOTAP_VERSION
;
336 rtap
->rthdr
.it_len
= cpu_to_le16(rtap_len
);
337 rtap
->rthdr
.it_present
= cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS
) |
338 (1 << IEEE80211_RADIOTAP_CHANNEL
) |
339 (1 << IEEE80211_RADIOTAP_MCS
));
340 if (d
->dma
.status
& RX_DMA_STATUS_ERROR
)
341 rtap
->flags
|= IEEE80211_RADIOTAP_F_BADFCS
;
343 rtap
->chnl_freq
= cpu_to_le16(ch
? ch
->center_freq
: 58320);
344 rtap
->chnl_flags
= cpu_to_le16(0);
346 rtap
->mcs_present
= IEEE80211_RADIOTAP_MCS_HAVE_MCS
;
348 rtap
->mcs_index
= wil_rxdesc_mcs(d
);
351 static bool wil_is_rx_idle(struct wil6210_priv
*wil
)
353 struct vring_rx_desc
*_d
;
354 struct wil_ring
*ring
= &wil
->ring_rx
;
356 _d
= (struct vring_rx_desc
*)&ring
->va
[ring
->swhead
].rx
.legacy
;
357 if (_d
->dma
.status
& RX_DMA_STATUS_DU
)
363 static int wil_rx_get_cid_by_skb(struct wil6210_priv
*wil
, struct sk_buff
*skb
)
365 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
366 int mid
= wil_rxdesc_mid(d
);
367 struct wil6210_vif
*vif
= wil
->vifs
[mid
];
368 /* cid from DMA descriptor is limited to 3 bits.
369 * In case of cid>=8, the value would be cid modulo 8 and we need to
370 * find real cid by locating the transmitter (ta) inside sta array
372 int cid
= wil_rxdesc_cid(d
);
373 unsigned int snaplen
= wil_rx_snaplen();
374 struct ieee80211_hdr_3addr
*hdr
;
379 /* in monitor mode there are no connections */
380 if (vif
->wdev
.iftype
== NL80211_IFTYPE_MONITOR
)
383 ftype
= wil_rxdesc_ftype(d
) << 2;
384 if (likely(ftype
== IEEE80211_FTYPE_DATA
)) {
385 if (unlikely(skb
->len
< ETH_HLEN
+ snaplen
)) {
386 wil_err_ratelimited(wil
,
387 "Short data frame, len = %d\n",
391 ta
= wil_skb_get_sa(skb
);
393 if (unlikely(skb
->len
< sizeof(struct ieee80211_hdr_3addr
))) {
394 wil_err_ratelimited(wil
, "Short frame, len = %d\n",
398 hdr
= (void *)skb
->data
;
402 if (wil
->max_assoc_sta
<= WIL6210_RX_DESC_MAX_CID
)
405 /* assuming no concurrency between AP interfaces and STA interfaces.
406 * multista is used only in P2P_GO or AP mode. In other modes return
407 * cid from the rx descriptor
409 if (vif
->wdev
.iftype
!= NL80211_IFTYPE_P2P_GO
&&
410 vif
->wdev
.iftype
!= NL80211_IFTYPE_AP
)
413 /* For Rx packets cid from rx descriptor is limited to 3 bits (0..7),
414 * to find the real cid, compare transmitter address with the stored
415 * stations mac address in the driver sta array
417 for (i
= cid
; i
< wil
->max_assoc_sta
; i
+= WIL6210_RX_DESC_MAX_CID
) {
418 if (wil
->sta
[i
].status
!= wil_sta_unused
&&
419 ether_addr_equal(wil
->sta
[i
].addr
, ta
)) {
424 if (i
>= wil
->max_assoc_sta
) {
425 wil_err_ratelimited(wil
, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n",
426 ta
, vif
->wdev
.iftype
, ftype
, skb
->len
);
433 /* reap 1 frame from @swhead
435 * Rx descriptor copied to skb->cb
437 * Safe to call from IRQ
439 static struct sk_buff
*wil_vring_reap_rx(struct wil6210_priv
*wil
,
440 struct wil_ring
*vring
)
442 struct device
*dev
= wil_to_dev(wil
);
443 struct wil6210_vif
*vif
;
444 struct net_device
*ndev
;
445 volatile struct vring_rx_desc
*_d
;
446 struct vring_rx_desc
*d
;
449 unsigned int snaplen
= wil_rx_snaplen();
450 unsigned int sz
= wil
->rx_buf_len
+ ETH_HLEN
+ snaplen
;
455 struct wil_net_stats
*stats
;
457 BUILD_BUG_ON(sizeof(struct skb_rx_info
) > sizeof(skb
->cb
));
460 if (unlikely(wil_ring_is_empty(vring
)))
463 i
= (int)vring
->swhead
;
464 _d
= &vring
->va
[i
].rx
.legacy
;
465 if (unlikely(!(_d
->dma
.status
& RX_DMA_STATUS_DU
))) {
466 /* it is not error, we just reached end of Rx done area */
470 skb
= vring
->ctx
[i
].skb
;
471 vring
->ctx
[i
].skb
= NULL
;
472 wil_ring_advance_head(vring
, 1);
474 wil_err(wil
, "No Rx skb at [%d]\n", i
);
477 d
= wil_skb_rxdesc(skb
);
479 pa
= wil_desc_addr(&d
->dma
.addr
);
481 dma_unmap_single(dev
, pa
, sz
, DMA_FROM_DEVICE
);
482 dmalen
= le16_to_cpu(d
->dma
.length
);
484 trace_wil6210_rx(i
, d
);
485 wil_dbg_txrx(wil
, "Rx[%3d] : %d bytes\n", i
, dmalen
);
486 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE
, 32, 4,
487 (const void *)d
, sizeof(*d
), false);
489 mid
= wil_rxdesc_mid(d
);
490 vif
= wil
->vifs
[mid
];
492 if (unlikely(!vif
)) {
493 wil_dbg_txrx(wil
, "skipped RX descriptor with invalid mid %d",
498 ndev
= vif_to_ndev(vif
);
499 if (unlikely(dmalen
> sz
)) {
500 wil_err_ratelimited(wil
, "Rx size too large: %d bytes!\n",
505 skb_trim(skb
, dmalen
);
509 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
510 skb
->data
, skb_headlen(skb
), false);
512 cid
= wil_rx_get_cid_by_skb(wil
, skb
);
513 if (cid
== -ENOENT
) {
517 wil_skb_set_cid(skb
, (u8
)cid
);
518 stats
= &wil
->sta
[cid
].stats
;
520 stats
->last_mcs_rx
= wil_rxdesc_mcs(d
);
521 if (stats
->last_mcs_rx
< ARRAY_SIZE(stats
->rx_per_mcs
))
522 stats
->rx_per_mcs
[stats
->last_mcs_rx
]++;
524 /* use radiotap header only if required */
525 if (ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
)
526 wil_rx_add_radiotap_header(wil
, skb
);
528 /* no extra checks if in sniffer mode */
529 if (ndev
->type
!= ARPHRD_ETHER
)
531 /* Non-data frames may be delivered through Rx DMA channel (ex: BAR)
532 * Driver should recognize it by frame type, that is found
533 * in Rx descriptor. If type is not data, it is 802.11 frame as is
535 ftype
= wil_rxdesc_ftype(d
) << 2;
536 if (unlikely(ftype
!= IEEE80211_FTYPE_DATA
)) {
537 u8 fc1
= wil_rxdesc_fc1(d
);
538 int tid
= wil_rxdesc_tid(d
);
539 u16 seq
= wil_rxdesc_seq(d
);
542 "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
543 fc1
, mid
, cid
, tid
, seq
);
544 stats
->rx_non_data_frame
++;
545 if (wil_is_back_req(fc1
)) {
547 "BAR: MID %d CID %d TID %d Seq 0x%03x\n",
549 wil_rx_bar(wil
, vif
, cid
, tid
, seq
);
551 /* print again all info. One can enable only this
552 * without overhead for printing every Rx frame
555 "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n",
556 fc1
, mid
, cid
, tid
, seq
);
557 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE
, 32, 4,
558 (const void *)d
, sizeof(*d
), false);
559 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET
, 16, 1,
560 skb
->data
, skb_headlen(skb
), false);
566 /* L4 IDENT is on when HW calculated checksum, check status
567 * and in case of error drop the packet
568 * higher stack layers will handle retransmission (if required)
570 if (likely(d
->dma
.status
& RX_DMA_STATUS_L4I
)) {
571 /* L4 protocol identified, csum calculated */
572 if (likely((d
->dma
.error
& RX_DMA_ERROR_L4_ERR
) == 0))
573 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
574 /* If HW reports bad checksum, let IP stack re-check it
575 * For example, HW don't understand Microsoft IP stack that
576 * mis-calculates TCP checksum - if it should be 0x0,
577 * it writes 0xffff in violation of RFC 1624
580 stats
->rx_csum_err
++;
585 * +-------+-------+---------+------------+------+
586 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA |
587 * +-------+-------+---------+------------+------+
588 * Need to remove SNAP, shifting SA and DA forward
590 memmove(skb
->data
+ snaplen
, skb
->data
, 2 * ETH_ALEN
);
591 skb_pull(skb
, snaplen
);
597 /* allocate and fill up to @count buffers in rx ring
598 * buffers posted at @swtail
599 * Note: we have a single RX queue for servicing all VIFs, but we
600 * allocate skbs with headroom according to main interface only. This
601 * means it will not work with monitor interface together with other VIFs.
602 * Currently we only support monitor interface on its own without other VIFs,
603 * and we will need to fix this code once we add support.
605 static int wil_rx_refill(struct wil6210_priv
*wil
, int count
)
607 struct net_device
*ndev
= wil
->main_ndev
;
608 struct wil_ring
*v
= &wil
->ring_rx
;
611 int headroom
= ndev
->type
== ARPHRD_IEEE80211_RADIOTAP
?
612 WIL6210_RTAP_SIZE
: 0;
614 for (; next_tail
= wil_ring_next_tail(v
),
615 (next_tail
!= v
->swhead
) && (count
-- > 0);
616 v
->swtail
= next_tail
) {
617 rc
= wil_vring_alloc_skb(wil
, v
, v
->swtail
, headroom
);
619 wil_err_ratelimited(wil
, "Error %d in rx refill[%d]\n",
625 /* make sure all writes to descriptors (shared memory) are done before
626 * committing them to HW
630 wil_w(wil
, v
->hwtail
, v
->swtail
);
636 * reverse_memcmp - Compare two areas of memory, in reverse order
637 * @cs: One area of memory
638 * @ct: Another area of memory
639 * @count: The size of the area.
641 * Cut'n'paste from original memcmp (see lib/string.c)
642 * with minimal modifications
644 int reverse_memcmp(const void *cs
, const void *ct
, size_t count
)
646 const unsigned char *su1
, *su2
;
649 for (su1
= cs
+ count
- 1, su2
= ct
+ count
- 1; count
> 0;
650 --su1
, --su2
, count
--) {
658 static int wil_rx_crypto_check(struct wil6210_priv
*wil
, struct sk_buff
*skb
)
660 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
661 int cid
= wil_skb_get_cid(skb
);
662 int tid
= wil_rxdesc_tid(d
);
663 int key_id
= wil_rxdesc_key_id(d
);
664 int mc
= wil_rxdesc_mcast(d
);
665 struct wil_sta_info
*s
= &wil
->sta
[cid
];
666 struct wil_tid_crypto_rx
*c
= mc
? &s
->group_crypto_rx
:
667 &s
->tid_crypto_rx
[tid
];
668 struct wil_tid_crypto_rx_single
*cc
= &c
->key_id
[key_id
];
669 const u8
*pn
= (u8
*)&d
->mac
.pn
;
672 wil_err_ratelimited(wil
,
673 "Key missing. CID %d TID %d MCast %d KEY_ID %d\n",
674 cid
, tid
, mc
, key_id
);
678 if (reverse_memcmp(pn
, cc
->pn
, IEEE80211_GCMP_PN_LEN
) <= 0) {
679 wil_err_ratelimited(wil
,
680 "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n",
681 cid
, tid
, mc
, key_id
, pn
, cc
->pn
);
684 memcpy(cc
->pn
, pn
, IEEE80211_GCMP_PN_LEN
);
689 static int wil_rx_error_check(struct wil6210_priv
*wil
, struct sk_buff
*skb
,
690 struct wil_net_stats
*stats
)
692 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
694 if ((d
->dma
.status
& RX_DMA_STATUS_ERROR
) &&
695 (d
->dma
.error
& RX_DMA_ERROR_MIC
)) {
696 stats
->rx_mic_error
++;
697 wil_dbg_txrx(wil
, "MIC error, dropping packet\n");
704 static void wil_get_netif_rx_params(struct sk_buff
*skb
, int *cid
,
707 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
709 *cid
= wil_skb_get_cid(skb
);
710 *security
= wil_rxdesc_security(d
);
714 * Check if skb is ptk eapol key message
716 * returns a pointer to the start of the eapol key structure, NULL
717 * if frame is not PTK eapol key
719 static struct wil_eapol_key
*wil_is_ptk_eapol_key(struct wil6210_priv
*wil
,
723 const struct wil_1x_hdr
*hdr
;
724 struct wil_eapol_key
*key
;
728 if (!skb_mac_header_was_set(skb
)) {
729 wil_err(wil
, "mac header was not set\n");
733 len
-= skb_mac_offset(skb
);
735 if (len
< sizeof(struct ethhdr
) + sizeof(struct wil_1x_hdr
) +
736 sizeof(struct wil_eapol_key
))
739 buf
= skb_mac_header(skb
) + sizeof(struct ethhdr
);
741 hdr
= (const struct wil_1x_hdr
*)buf
;
742 if (hdr
->type
!= WIL_1X_TYPE_EAPOL_KEY
)
745 key
= (struct wil_eapol_key
*)(buf
+ sizeof(struct wil_1x_hdr
));
746 if (key
->type
!= WIL_EAPOL_KEY_TYPE_WPA
&&
747 key
->type
!= WIL_EAPOL_KEY_TYPE_RSN
)
750 key_info
= be16_to_cpu(key
->key_info
);
751 if (!(key_info
& WIL_KEY_INFO_KEY_TYPE
)) /* check if pairwise */
757 static bool wil_skb_is_eap_3(struct wil6210_priv
*wil
, struct sk_buff
*skb
)
759 struct wil_eapol_key
*key
;
762 key
= wil_is_ptk_eapol_key(wil
, skb
);
766 key_info
= be16_to_cpu(key
->key_info
);
767 if (key_info
& (WIL_KEY_INFO_MIC
|
768 WIL_KEY_INFO_ENCR_KEY_DATA
)) {
769 /* 3/4 of 4-Way Handshake */
770 wil_dbg_misc(wil
, "EAPOL key message 3\n");
773 /* 1/4 of 4-Way Handshake */
774 wil_dbg_misc(wil
, "EAPOL key message 1\n");
779 static bool wil_skb_is_eap_4(struct wil6210_priv
*wil
, struct sk_buff
*skb
)
781 struct wil_eapol_key
*key
;
784 key
= wil_is_ptk_eapol_key(wil
, skb
);
788 nonce
= (u32
*)key
->key_nonce
;
789 for (i
= 0; i
< WIL_EAP_NONCE_LEN
/ sizeof(u32
); i
++, nonce
++) {
792 wil_dbg_misc(wil
, "EAPOL key message 2\n");
796 wil_dbg_misc(wil
, "EAPOL key message 4\n");
801 void wil_enable_tx_key_worker(struct work_struct
*work
)
803 struct wil6210_vif
*vif
= container_of(work
,
804 struct wil6210_vif
, enable_tx_key_worker
);
805 struct wil6210_priv
*wil
= vif_to_wil(vif
);
809 if (vif
->ptk_rekey_state
!= WIL_REKEY_WAIT_M4_SENT
) {
810 wil_dbg_misc(wil
, "Invalid rekey state = %d\n",
811 vif
->ptk_rekey_state
);
816 cid
= wil_find_cid_by_idx(wil
, vif
->mid
, 0);
817 if (!wil_cid_valid(wil
, cid
)) {
818 wil_err(wil
, "Invalid cid = %d\n", cid
);
823 wil_dbg_misc(wil
, "Apply PTK key after eapol was sent out\n");
824 rc
= wmi_add_cipher_key(vif
, 0, wil
->sta
[cid
].addr
, 0, NULL
,
825 WMI_KEY_USE_APPLY_PTK
);
827 vif
->ptk_rekey_state
= WIL_REKEY_IDLE
;
831 wil_err(wil
, "Apply PTK key failed %d\n", rc
);
834 void wil_tx_complete_handle_eapol(struct wil6210_vif
*vif
, struct sk_buff
*skb
)
836 struct wil6210_priv
*wil
= vif_to_wil(vif
);
837 struct wireless_dev
*wdev
= vif_to_wdev(vif
);
840 if (wdev
->iftype
!= NL80211_IFTYPE_STATION
||
841 !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY
, wil
->fw_capabilities
))
844 /* check if skb is an EAP message 4/4 */
845 if (!wil_skb_is_eap_4(wil
, skb
))
848 spin_lock_bh(&wil
->eap_lock
);
849 switch (vif
->ptk_rekey_state
) {
851 /* ignore idle state, can happen due to M4 retransmission */
853 case WIL_REKEY_M3_RECEIVED
:
854 vif
->ptk_rekey_state
= WIL_REKEY_IDLE
;
856 case WIL_REKEY_WAIT_M4_SENT
:
860 wil_err(wil
, "Unknown rekey state = %d",
861 vif
->ptk_rekey_state
);
863 spin_unlock_bh(&wil
->eap_lock
);
866 q
= queue_work(wil
->wmi_wq
, &vif
->enable_tx_key_worker
);
867 wil_dbg_misc(wil
, "queue_work of enable_tx_key_worker -> %d\n",
872 static void wil_rx_handle_eapol(struct wil6210_vif
*vif
, struct sk_buff
*skb
)
874 struct wil6210_priv
*wil
= vif_to_wil(vif
);
875 struct wireless_dev
*wdev
= vif_to_wdev(vif
);
877 if (wdev
->iftype
!= NL80211_IFTYPE_STATION
||
878 !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY
, wil
->fw_capabilities
))
881 /* check if skb is a EAP message 3/4 */
882 if (!wil_skb_is_eap_3(wil
, skb
))
885 if (vif
->ptk_rekey_state
== WIL_REKEY_IDLE
)
886 vif
->ptk_rekey_state
= WIL_REKEY_M3_RECEIVED
;
890 * Pass Rx packet to the netif. Update statistics.
891 * Called in softirq context (NAPI poll).
893 void wil_netif_rx(struct sk_buff
*skb
, struct net_device
*ndev
, int cid
,
894 struct wil_net_stats
*stats
, bool gro
)
896 struct wil6210_vif
*vif
= ndev_to_vif(ndev
);
897 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
898 struct wireless_dev
*wdev
= vif_to_wdev(vif
);
899 unsigned int len
= skb
->len
;
900 u8
*sa
, *da
= wil_skb_get_da(skb
);
901 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication
902 * is not suitable, need to look at data
904 int mcast
= is_multicast_ether_addr(da
);
905 struct sk_buff
*xmit_skb
= NULL
;
907 if (wdev
->iftype
== NL80211_IFTYPE_STATION
) {
908 sa
= wil_skb_get_sa(skb
);
909 if (mcast
&& ether_addr_equal(sa
, ndev
->dev_addr
)) {
910 /* mcast packet looped back to us */
912 ndev
->stats
.rx_dropped
++;
914 wil_dbg_txrx(wil
, "Rx drop %d bytes\n", len
);
917 } else if (wdev
->iftype
== NL80211_IFTYPE_AP
&& !vif
->ap_isolate
) {
919 /* send multicast frames both to higher layers in
920 * local net stack and back to the wireless medium
922 xmit_skb
= skb_copy(skb
, GFP_ATOMIC
);
924 int xmit_cid
= wil_find_cid(wil
, vif
->mid
, da
);
927 /* The destination station is associated to
928 * this AP (in this VLAN), so send the frame
929 * directly to it and do not pass it to local
938 /* Send to wireless media and increase priority by 256 to
939 * keep the received priority instead of reclassifying
940 * the frame (see cfg80211_classify8021d).
942 xmit_skb
->dev
= ndev
;
943 xmit_skb
->priority
+= 256;
944 xmit_skb
->protocol
= htons(ETH_P_802_3
);
945 skb_reset_network_header(xmit_skb
);
946 skb_reset_mac_header(xmit_skb
);
947 wil_dbg_txrx(wil
, "Rx -> Tx %d bytes\n", len
);
948 dev_queue_xmit(xmit_skb
);
951 if (skb
) { /* deliver to local stack */
952 skb
->protocol
= eth_type_trans(skb
, ndev
);
955 if (skb
->protocol
== cpu_to_be16(ETH_P_PAE
))
956 wil_rx_handle_eapol(vif
, skb
);
959 napi_gro_receive(&wil
->napi_rx
, skb
);
963 ndev
->stats
.rx_packets
++;
965 ndev
->stats
.rx_bytes
+= len
;
966 stats
->rx_bytes
+= len
;
968 ndev
->stats
.multicast
++;
971 void wil_netif_rx_any(struct sk_buff
*skb
, struct net_device
*ndev
)
974 struct wil6210_priv
*wil
= ndev_to_wil(ndev
);
975 struct wil_net_stats
*stats
;
977 wil
->txrx_ops
.get_netif_rx_params(skb
, &cid
, &security
);
979 stats
= &wil
->sta
[cid
].stats
;
983 if (security
&& (wil
->txrx_ops
.rx_crypto_check(wil
, skb
) != 0)) {
984 wil_dbg_txrx(wil
, "Rx drop %d bytes\n", skb
->len
);
986 ndev
->stats
.rx_dropped
++;
992 /* check errors reported by HW and update statistics */
993 if (unlikely(wil
->txrx_ops
.rx_error_check(wil
, skb
, stats
))) {
998 wil_netif_rx(skb
, ndev
, cid
, stats
, true);
1001 /* Proceed all completed skb's from Rx VRING
1003 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled
1005 void wil_rx_handle(struct wil6210_priv
*wil
, int *quota
)
1007 struct net_device
*ndev
= wil
->main_ndev
;
1008 struct wireless_dev
*wdev
= ndev
->ieee80211_ptr
;
1009 struct wil_ring
*v
= &wil
->ring_rx
;
1010 struct sk_buff
*skb
;
1012 if (unlikely(!v
->va
)) {
1013 wil_err(wil
, "Rx IRQ while Rx not yet initialized\n");
1016 wil_dbg_txrx(wil
, "rx_handle\n");
1017 while ((*quota
> 0) && (NULL
!= (skb
= wil_vring_reap_rx(wil
, v
)))) {
1020 /* monitor is currently supported on main interface only */
1021 if (wdev
->iftype
== NL80211_IFTYPE_MONITOR
) {
1023 skb_reset_mac_header(skb
);
1024 skb
->ip_summed
= CHECKSUM_UNNECESSARY
;
1025 skb
->pkt_type
= PACKET_OTHERHOST
;
1026 skb
->protocol
= htons(ETH_P_802_2
);
1027 wil_netif_rx_any(skb
, ndev
);
1029 wil_rx_reorder(wil
, skb
);
1032 wil_rx_refill(wil
, v
->size
);
1035 static void wil_rx_buf_len_init(struct wil6210_priv
*wil
)
1037 wil
->rx_buf_len
= rx_large_buf
?
1038 WIL_MAX_ETH_MTU
: TXRX_BUF_LEN_DEFAULT
- WIL_MAX_MPDU_OVERHEAD
;
1039 if (mtu_max
> wil
->rx_buf_len
) {
1040 /* do not allow RX buffers to be smaller than mtu_max, for
1041 * backward compatibility (mtu_max parameter was also used
1042 * to support receiving large packets)
1044 wil_info(wil
, "Override RX buffer to mtu_max(%d)\n", mtu_max
);
1045 wil
->rx_buf_len
= mtu_max
;
1049 static int wil_rx_init(struct wil6210_priv
*wil
, uint order
)
1051 struct wil_ring
*vring
= &wil
->ring_rx
;
1054 wil_dbg_misc(wil
, "rx_init\n");
1057 wil_err(wil
, "Rx ring already allocated\n");
1061 wil_rx_buf_len_init(wil
);
1063 vring
->size
= 1 << order
;
1064 vring
->is_rx
= true;
1065 rc
= wil_vring_alloc(wil
, vring
);
1069 rc
= wmi_rx_chain_add(wil
, vring
);
1073 rc
= wil_rx_refill(wil
, vring
->size
);
1079 wil_vring_free(wil
, vring
);
1084 static void wil_rx_fini(struct wil6210_priv
*wil
)
1086 struct wil_ring
*vring
= &wil
->ring_rx
;
1088 wil_dbg_misc(wil
, "rx_fini\n");
1091 wil_vring_free(wil
, vring
);
1094 static int wil_tx_desc_map(union wil_tx_desc
*desc
, dma_addr_t pa
,
1095 u32 len
, int vring_index
)
1097 struct vring_tx_desc
*d
= &desc
->legacy
;
1099 wil_desc_addr_set(&d
->dma
.addr
, pa
);
1100 d
->dma
.ip_length
= 0;
1101 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/
1102 d
->dma
.b11
= 0/*14 | BIT(7)*/;
1104 d
->dma
.status
= 0; /* BIT(0) should be 0 for HW_OWNED */
1105 d
->dma
.length
= cpu_to_le16((u16
)len
);
1106 d
->dma
.d0
= (vring_index
<< DMA_CFG_DESC_TX_0_QID_POS
);
1110 d
->mac
.ucode_cmd
= 0;
1111 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */
1112 d
->mac
.d
[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS
) |
1113 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS
);
1118 void wil_tx_data_init(struct wil_ring_tx_data
*txdata
)
1120 spin_lock_bh(&txdata
->lock
);
1121 txdata
->dot1x_open
= false;
1122 txdata
->enabled
= 0;
1124 txdata
->last_idle
= 0;
1126 txdata
->agg_wsize
= 0;
1127 txdata
->agg_timeout
= 0;
1128 txdata
->agg_amsdu
= 0;
1129 txdata
->addba_in_progress
= false;
1130 txdata
->mid
= U8_MAX
;
1131 spin_unlock_bh(&txdata
->lock
);
1134 static int wil_vring_init_tx(struct wil6210_vif
*vif
, int id
, int size
,
1137 struct wil6210_priv
*wil
= vif_to_wil(vif
);
1139 struct wmi_vring_cfg_cmd cmd
= {
1140 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
1144 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
1145 .ring_size
= cpu_to_le16(size
),
1148 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
1153 .priority
= cpu_to_le16(0),
1154 .timeslot_us
= cpu_to_le16(0xfff),
1159 struct wmi_cmd_hdr wmi
;
1160 struct wmi_vring_cfg_done_event cmd
;
1161 } __packed reply
= {
1162 .cmd
= {.status
= WMI_FW_STATUS_FAILURE
},
1164 struct wil_ring
*vring
= &wil
->ring_tx
[id
];
1165 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[id
];
1167 if (cid
>= WIL6210_RX_DESC_MAX_CID
) {
1168 cmd
.vring_cfg
.cidxtid
= CIDXTID_EXTENDED_CID_TID
;
1169 cmd
.vring_cfg
.cid
= cid
;
1170 cmd
.vring_cfg
.tid
= tid
;
1172 cmd
.vring_cfg
.cidxtid
= mk_cidxtid(cid
, tid
);
1175 wil_dbg_misc(wil
, "vring_init_tx: max_mpdu_size %d\n",
1176 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
1177 lockdep_assert_held(&wil
->mutex
);
1180 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
1185 wil_tx_data_init(txdata
);
1186 vring
->is_rx
= false;
1188 rc
= wil_vring_alloc(wil
, vring
);
1192 wil
->ring2cid_tid
[id
][0] = cid
;
1193 wil
->ring2cid_tid
[id
][1] = tid
;
1195 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
1198 txdata
->dot1x_open
= true;
1199 rc
= wmi_call(wil
, WMI_VRING_CFG_CMDID
, vif
->mid
, &cmd
, sizeof(cmd
),
1200 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
),
1201 WIL_WMI_CALL_GENERAL_TO_MS
);
1205 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
1206 wil_err(wil
, "Tx config failed, status 0x%02x\n",
1212 spin_lock_bh(&txdata
->lock
);
1213 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
1214 txdata
->mid
= vif
->mid
;
1215 txdata
->enabled
= 1;
1216 spin_unlock_bh(&txdata
->lock
);
1218 if (txdata
->dot1x_open
&& (agg_wsize
>= 0))
1219 wil_addba_tx_request(wil
, id
, agg_wsize
);
1223 spin_lock_bh(&txdata
->lock
);
1224 txdata
->dot1x_open
= false;
1225 txdata
->enabled
= 0;
1226 spin_unlock_bh(&txdata
->lock
);
1227 wil_vring_free(wil
, vring
);
1228 wil
->ring2cid_tid
[id
][0] = wil
->max_assoc_sta
;
1229 wil
->ring2cid_tid
[id
][1] = 0;
1236 static int wil_tx_vring_modify(struct wil6210_vif
*vif
, int ring_id
, int cid
,
1239 struct wil6210_priv
*wil
= vif_to_wil(vif
);
1241 struct wmi_vring_cfg_cmd cmd
= {
1242 .action
= cpu_to_le32(WMI_VRING_CMD_MODIFY
),
1246 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
1250 .cidxtid
= mk_cidxtid(cid
, tid
),
1251 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
1256 .priority
= cpu_to_le16(0),
1257 .timeslot_us
= cpu_to_le16(0xfff),
1262 struct wmi_cmd_hdr wmi
;
1263 struct wmi_vring_cfg_done_event cmd
;
1264 } __packed reply
= {
1265 .cmd
= {.status
= WMI_FW_STATUS_FAILURE
},
1267 struct wil_ring
*vring
= &wil
->ring_tx
[ring_id
];
1268 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ring_id
];
1270 wil_dbg_misc(wil
, "vring_modify: ring %d cid %d tid %d\n", ring_id
,
1272 lockdep_assert_held(&wil
->mutex
);
1275 wil_err(wil
, "Tx ring [%d] not allocated\n", ring_id
);
1279 if (wil
->ring2cid_tid
[ring_id
][0] != cid
||
1280 wil
->ring2cid_tid
[ring_id
][1] != tid
) {
1281 wil_err(wil
, "ring info does not match cid=%u tid=%u\n",
1282 wil
->ring2cid_tid
[ring_id
][0],
1283 wil
->ring2cid_tid
[ring_id
][1]);
1286 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
1288 rc
= wmi_call(wil
, WMI_VRING_CFG_CMDID
, vif
->mid
, &cmd
, sizeof(cmd
),
1289 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
),
1290 WIL_WMI_CALL_GENERAL_TO_MS
);
1294 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
1295 wil_err(wil
, "Tx modify failed, status 0x%02x\n",
1301 /* set BA aggregation window size to 0 to force a new BA with the
1304 txdata
->agg_wsize
= 0;
1305 if (txdata
->dot1x_open
&& agg_wsize
>= 0)
1306 wil_addba_tx_request(wil
, ring_id
, agg_wsize
);
1310 spin_lock_bh(&txdata
->lock
);
1311 txdata
->dot1x_open
= false;
1312 txdata
->enabled
= 0;
1313 spin_unlock_bh(&txdata
->lock
);
1314 wil
->ring2cid_tid
[ring_id
][0] = wil
->max_assoc_sta
;
1315 wil
->ring2cid_tid
[ring_id
][1] = 0;
1319 int wil_vring_init_bcast(struct wil6210_vif
*vif
, int id
, int size
)
1321 struct wil6210_priv
*wil
= vif_to_wil(vif
);
1323 struct wmi_bcast_vring_cfg_cmd cmd
= {
1324 .action
= cpu_to_le32(WMI_VRING_CMD_ADD
),
1328 cpu_to_le16(wil_mtu2macbuf(mtu_max
)),
1329 .ring_size
= cpu_to_le16(size
),
1332 .encap_trans_type
= WMI_VRING_ENC_TYPE_802_3
,
1336 struct wmi_cmd_hdr wmi
;
1337 struct wmi_vring_cfg_done_event cmd
;
1338 } __packed reply
= {
1339 .cmd
= {.status
= WMI_FW_STATUS_FAILURE
},
1341 struct wil_ring
*vring
= &wil
->ring_tx
[id
];
1342 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[id
];
1344 wil_dbg_misc(wil
, "vring_init_bcast: max_mpdu_size %d\n",
1345 cmd
.vring_cfg
.tx_sw_ring
.max_mpdu_size
);
1346 lockdep_assert_held(&wil
->mutex
);
1349 wil_err(wil
, "Tx ring [%d] already allocated\n", id
);
1354 wil_tx_data_init(txdata
);
1355 vring
->is_rx
= false;
1357 rc
= wil_vring_alloc(wil
, vring
);
1361 wil
->ring2cid_tid
[id
][0] = wil
->max_assoc_sta
; /* CID */
1362 wil
->ring2cid_tid
[id
][1] = 0; /* TID */
1364 cmd
.vring_cfg
.tx_sw_ring
.ring_mem_base
= cpu_to_le64(vring
->pa
);
1367 txdata
->dot1x_open
= true;
1368 rc
= wmi_call(wil
, WMI_BCAST_VRING_CFG_CMDID
, vif
->mid
,
1370 WMI_VRING_CFG_DONE_EVENTID
, &reply
, sizeof(reply
),
1371 WIL_WMI_CALL_GENERAL_TO_MS
);
1375 if (reply
.cmd
.status
!= WMI_FW_STATUS_SUCCESS
) {
1376 wil_err(wil
, "Tx config failed, status 0x%02x\n",
1382 spin_lock_bh(&txdata
->lock
);
1383 vring
->hwtail
= le32_to_cpu(reply
.cmd
.tx_vring_tail_ptr
);
1384 txdata
->mid
= vif
->mid
;
1385 txdata
->enabled
= 1;
1386 spin_unlock_bh(&txdata
->lock
);
1390 spin_lock_bh(&txdata
->lock
);
1391 txdata
->enabled
= 0;
1392 txdata
->dot1x_open
= false;
1393 spin_unlock_bh(&txdata
->lock
);
1394 wil_vring_free(wil
, vring
);
1400 static struct wil_ring
*wil_find_tx_ucast(struct wil6210_priv
*wil
,
1401 struct wil6210_vif
*vif
,
1402 struct sk_buff
*skb
)
1405 const u8
*da
= wil_skb_get_da(skb
);
1406 int min_ring_id
= wil_get_min_tx_ring_id(wil
);
1408 cid
= wil_find_cid(wil
, vif
->mid
, da
);
1410 if (cid
< 0 || cid
>= wil
->max_assoc_sta
)
1413 /* TODO: fix for multiple TID */
1414 for (i
= min_ring_id
; i
< ARRAY_SIZE(wil
->ring2cid_tid
); i
++) {
1415 if (!wil
->ring_tx_data
[i
].dot1x_open
&&
1416 skb
->protocol
!= cpu_to_be16(ETH_P_PAE
))
1418 if (wil
->ring2cid_tid
[i
][0] == cid
) {
1419 struct wil_ring
*v
= &wil
->ring_tx
[i
];
1420 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[i
];
1422 wil_dbg_txrx(wil
, "find_tx_ucast: (%pM) -> [%d]\n",
1424 if (v
->va
&& txdata
->enabled
) {
1428 "find_tx_ucast: vring[%d] not valid\n",
1438 static int wil_tx_ring(struct wil6210_priv
*wil
, struct wil6210_vif
*vif
,
1439 struct wil_ring
*ring
, struct sk_buff
*skb
);
1441 static struct wil_ring
*wil_find_tx_ring_sta(struct wil6210_priv
*wil
,
1442 struct wil6210_vif
*vif
,
1443 struct sk_buff
*skb
)
1445 struct wil_ring
*ring
;
1448 struct wil_ring_tx_data
*txdata
;
1449 int min_ring_id
= wil_get_min_tx_ring_id(wil
);
1451 /* In the STA mode, it is expected to have only 1 VRING
1452 * for the AP we connected to.
1453 * find 1-st vring eligible for this skb and use it.
1455 for (i
= min_ring_id
; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1456 ring
= &wil
->ring_tx
[i
];
1457 txdata
= &wil
->ring_tx_data
[i
];
1458 if (!ring
->va
|| !txdata
->enabled
|| txdata
->mid
!= vif
->mid
)
1461 cid
= wil
->ring2cid_tid
[i
][0];
1462 if (cid
>= wil
->max_assoc_sta
) /* skip BCAST */
1465 if (!wil
->ring_tx_data
[i
].dot1x_open
&&
1466 skb
->protocol
!= cpu_to_be16(ETH_P_PAE
))
1469 wil_dbg_txrx(wil
, "Tx -> ring %d\n", i
);
1474 wil_dbg_txrx(wil
, "Tx while no rings active?\n");
1479 /* Use one of 2 strategies:
1481 * 1. New (real broadcast):
1482 * use dedicated broadcast vring
1483 * 2. Old (pseudo-DMS):
1484 * Find 1-st vring and return it;
1485 * duplicate skb and send it to other active vrings;
1486 * in all cases override dest address to unicast peer's address
1487 * Use old strategy when new is not supported yet:
1490 static struct wil_ring
*wil_find_tx_bcast_1(struct wil6210_priv
*wil
,
1491 struct wil6210_vif
*vif
,
1492 struct sk_buff
*skb
)
1495 struct wil_ring_tx_data
*txdata
;
1496 int i
= vif
->bcast_ring
;
1500 v
= &wil
->ring_tx
[i
];
1501 txdata
= &wil
->ring_tx_data
[i
];
1502 if (!v
->va
|| !txdata
->enabled
)
1504 if (!wil
->ring_tx_data
[i
].dot1x_open
&&
1505 skb
->protocol
!= cpu_to_be16(ETH_P_PAE
))
1511 /* apply multicast to unicast only for ARP and IP packets
1512 * (see NL80211_CMD_SET_MULTICAST_TO_UNICAST for more info)
1514 static bool wil_check_multicast_to_unicast(struct wil6210_priv
*wil
,
1515 struct sk_buff
*skb
)
1517 const struct ethhdr
*eth
= (void *)skb
->data
;
1518 const struct vlan_ethhdr
*ethvlan
= (void *)skb
->data
;
1521 if (!wil
->multicast_to_unicast
)
1524 /* multicast to unicast conversion only for some payload */
1525 ethertype
= eth
->h_proto
;
1526 if (ethertype
== htons(ETH_P_8021Q
) && skb
->len
>= VLAN_ETH_HLEN
)
1527 ethertype
= ethvlan
->h_vlan_encapsulated_proto
;
1528 switch (ethertype
) {
1529 case htons(ETH_P_ARP
):
1530 case htons(ETH_P_IP
):
1531 case htons(ETH_P_IPV6
):
1540 static void wil_set_da_for_vring(struct wil6210_priv
*wil
,
1541 struct sk_buff
*skb
, int vring_index
)
1543 u8
*da
= wil_skb_get_da(skb
);
1544 int cid
= wil
->ring2cid_tid
[vring_index
][0];
1546 ether_addr_copy(da
, wil
->sta
[cid
].addr
);
1549 static struct wil_ring
*wil_find_tx_bcast_2(struct wil6210_priv
*wil
,
1550 struct wil6210_vif
*vif
,
1551 struct sk_buff
*skb
)
1553 struct wil_ring
*v
, *v2
;
1554 struct sk_buff
*skb2
;
1557 const u8
*src
= wil_skb_get_sa(skb
);
1558 struct wil_ring_tx_data
*txdata
, *txdata2
;
1559 int min_ring_id
= wil_get_min_tx_ring_id(wil
);
1561 /* find 1-st vring eligible for data */
1562 for (i
= min_ring_id
; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1563 v
= &wil
->ring_tx
[i
];
1564 txdata
= &wil
->ring_tx_data
[i
];
1565 if (!v
->va
|| !txdata
->enabled
|| txdata
->mid
!= vif
->mid
)
1568 cid
= wil
->ring2cid_tid
[i
][0];
1569 if (cid
>= wil
->max_assoc_sta
) /* skip BCAST */
1571 if (!wil
->ring_tx_data
[i
].dot1x_open
&&
1572 skb
->protocol
!= cpu_to_be16(ETH_P_PAE
))
1575 /* don't Tx back to source when re-routing Rx->Tx at the AP */
1576 if (0 == memcmp(wil
->sta
[cid
].addr
, src
, ETH_ALEN
))
1582 wil_dbg_txrx(wil
, "Tx while no vrings active?\n");
1587 wil_dbg_txrx(wil
, "BCAST -> ring %d\n", i
);
1588 wil_set_da_for_vring(wil
, skb
, i
);
1590 /* find other active vrings and duplicate skb for each */
1591 for (i
++; i
< WIL6210_MAX_TX_RINGS
; i
++) {
1592 v2
= &wil
->ring_tx
[i
];
1593 txdata2
= &wil
->ring_tx_data
[i
];
1594 if (!v2
->va
|| txdata2
->mid
!= vif
->mid
)
1596 cid
= wil
->ring2cid_tid
[i
][0];
1597 if (cid
>= wil
->max_assoc_sta
) /* skip BCAST */
1599 if (!wil
->ring_tx_data
[i
].dot1x_open
&&
1600 skb
->protocol
!= cpu_to_be16(ETH_P_PAE
))
1603 if (0 == memcmp(wil
->sta
[cid
].addr
, src
, ETH_ALEN
))
1606 skb2
= skb_copy(skb
, GFP_ATOMIC
);
1608 wil_dbg_txrx(wil
, "BCAST DUP -> ring %d\n", i
);
1609 wil_set_da_for_vring(wil
, skb2
, i
);
1610 wil_tx_ring(wil
, vif
, v2
, skb2
);
1611 /* successful call to wil_tx_ring takes skb2 ref */
1612 dev_kfree_skb_any(skb2
);
1614 wil_err(wil
, "skb_copy failed\n");
1622 void wil_tx_desc_set_nr_frags(struct vring_tx_desc
*d
, int nr_frags
)
1624 d
->mac
.d
[2] |= (nr_frags
<< MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS
);
1627 /* Sets the descriptor @d up for csum and/or TSO offloading. The corresponding
1628 * @skb is used to obtain the protocol and headers length.
1629 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data,
1630 * 2 - middle, 3 - last descriptor.
1633 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc
*d
,
1634 struct sk_buff
*skb
,
1635 int tso_desc_type
, bool is_ipv4
,
1636 int tcp_hdr_len
, int skb_net_hdr_len
)
1638 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
1639 d
->dma
.b11
|= is_ipv4
<< DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
;
1641 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
1642 /* L4 header len: TCP header length */
1643 d
->dma
.d0
|= (tcp_hdr_len
& DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1645 /* Setup TSO: bit and desc type */
1646 d
->dma
.d0
|= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS
)) |
1647 (tso_desc_type
<< DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS
);
1648 d
->dma
.d0
|= (is_ipv4
<< DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS
);
1650 d
->dma
.ip_length
= skb_net_hdr_len
;
1651 /* Enable TCP/UDP checksum */
1652 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
1653 /* Calculate pseudo-header */
1654 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
1657 /* Sets the descriptor @d up for csum. The corresponding
1658 * @skb is used to obtain the protocol and headers length.
1659 * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6.
1660 * Note, if d==NULL, the function only returns the protocol result.
1662 * It is very similar to previous wil_tx_desc_offload_setup_tso. This
1663 * is "if unrolling" to optimize the critical path.
1666 static int wil_tx_desc_offload_setup(struct vring_tx_desc
*d
,
1667 struct sk_buff
*skb
){
1670 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1673 d
->dma
.b11
= ETH_HLEN
; /* MAC header length */
1675 switch (skb
->protocol
) {
1676 case cpu_to_be16(ETH_P_IP
):
1677 protocol
= ip_hdr(skb
)->protocol
;
1678 d
->dma
.b11
|= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS
);
1680 case cpu_to_be16(ETH_P_IPV6
):
1681 protocol
= ipv6_hdr(skb
)->nexthdr
;
1689 d
->dma
.d0
|= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS
);
1690 /* L4 header len: TCP header length */
1692 (tcp_hdrlen(skb
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1695 /* L4 header len: UDP header length */
1697 (sizeof(struct udphdr
) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK
);
1703 d
->dma
.ip_length
= skb_network_header_len(skb
);
1704 /* Enable TCP/UDP checksum */
1705 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS
);
1706 /* Calculate pseudo-header */
1707 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS
);
1712 static inline void wil_tx_last_desc(struct vring_tx_desc
*d
)
1714 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
) |
1715 BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
) |
1716 BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
1719 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc
*d
)
1721 d
->dma
.d0
|= wil_tso_type_lst
<<
1722 DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS
;
1725 static int __wil_tx_vring_tso(struct wil6210_priv
*wil
, struct wil6210_vif
*vif
,
1726 struct wil_ring
*vring
, struct sk_buff
*skb
)
1728 struct device
*dev
= wil_to_dev(wil
);
1730 /* point to descriptors in shared memory */
1731 volatile struct vring_tx_desc
*_desc
= NULL
, *_hdr_desc
,
1732 *_first_desc
= NULL
;
1734 /* pointers to shadow descriptors */
1735 struct vring_tx_desc desc_mem
, hdr_desc_mem
, first_desc_mem
,
1736 *d
= &hdr_desc_mem
, *hdr_desc
= &hdr_desc_mem
,
1737 *first_desc
= &first_desc_mem
;
1739 /* pointer to shadow descriptors' context */
1740 struct wil_ctx
*hdr_ctx
, *first_ctx
= NULL
;
1742 int descs_used
= 0; /* total number of used descriptors */
1743 int sg_desc_cnt
= 0; /* number of descriptors for current mss*/
1745 u32 swhead
= vring
->swhead
;
1746 int used
, avail
= wil_ring_avail_tx(vring
);
1747 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
1748 int min_desc_required
= nr_frags
+ 1;
1749 int mss
= skb_shinfo(skb
)->gso_size
; /* payload size w/o headers */
1750 int f
, len
, hdrlen
, headlen
;
1751 int vring_index
= vring
- wil
->ring_tx
;
1752 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[vring_index
];
1755 const skb_frag_t
*frag
= NULL
;
1758 int hdr_compensation_need
= true;
1759 int desc_tso_type
= wil_tso_type_first
;
1762 int skb_net_hdr_len
;
1766 wil_dbg_txrx(wil
, "tx_vring_tso: %d bytes to vring %d\n", skb
->len
,
1769 if (unlikely(!txdata
->enabled
))
1772 /* A typical page 4K is 3-4 payloads, we assume each fragment
1773 * is a full payload, that's how min_desc_required has been
1774 * calculated. In real we might need more or less descriptors,
1775 * this is the initial check only.
1777 if (unlikely(avail
< min_desc_required
)) {
1778 wil_err_ratelimited(wil
,
1779 "TSO: Tx ring[%2d] full. No space for %d fragments\n",
1780 vring_index
, min_desc_required
);
1784 /* Header Length = MAC header len + IP header len + TCP header len*/
1785 hdrlen
= skb_tcp_all_headers(skb
);
1787 gso_type
= skb_shinfo(skb
)->gso_type
& (SKB_GSO_TCPV6
| SKB_GSO_TCPV4
);
1790 /* TCP v4, zero out the IP length and IPv4 checksum fields
1791 * as required by the offloading doc
1793 ip_hdr(skb
)->tot_len
= 0;
1794 ip_hdr(skb
)->check
= 0;
1798 /* TCP v6, zero out the payload length */
1799 ipv6_hdr(skb
)->payload_len
= 0;
1803 /* other than TCPv4 or TCPv6 types are not supported for TSO.
1804 * It is also illegal for both to be set simultaneously
1809 if (skb
->ip_summed
!= CHECKSUM_PARTIAL
)
1812 /* tcp header length and skb network header length are fixed for all
1813 * packet's descriptors - read then once here
1815 tcp_hdr_len
= tcp_hdrlen(skb
);
1816 skb_net_hdr_len
= skb_network_header_len(skb
);
1818 _hdr_desc
= &vring
->va
[i
].tx
.legacy
;
1820 pa
= dma_map_single(dev
, skb
->data
, hdrlen
, DMA_TO_DEVICE
);
1821 if (unlikely(dma_mapping_error(dev
, pa
))) {
1822 wil_err(wil
, "TSO: Skb head DMA map error\n");
1826 wil
->txrx_ops
.tx_desc_map((union wil_tx_desc
*)hdr_desc
, pa
,
1827 hdrlen
, vring_index
);
1828 wil_tx_desc_offload_setup_tso(hdr_desc
, skb
, wil_tso_type_hdr
, is_ipv4
,
1829 tcp_hdr_len
, skb_net_hdr_len
);
1830 wil_tx_last_desc(hdr_desc
);
1832 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1833 hdr_ctx
= &vring
->ctx
[i
];
1836 headlen
= skb_headlen(skb
) - hdrlen
;
1838 for (f
= headlen
? -1 : 0; f
< nr_frags
; f
++) {
1841 wil_dbg_txrx(wil
, "TSO: process skb head, len %u\n",
1844 frag
= &skb_shinfo(skb
)->frags
[f
];
1845 len
= skb_frag_size(frag
);
1846 wil_dbg_txrx(wil
, "TSO: frag[%d]: len %u\n", f
, len
);
1851 "TSO: len %d, rem_data %d, descs_used %d\n",
1852 len
, rem_data
, descs_used
);
1854 if (descs_used
== avail
) {
1855 wil_err_ratelimited(wil
, "TSO: ring overflow\n");
1860 lenmss
= min_t(int, rem_data
, len
);
1861 i
= (swhead
+ descs_used
) % vring
->size
;
1862 wil_dbg_txrx(wil
, "TSO: lenmss %d, i %d\n", lenmss
, i
);
1865 pa
= skb_frag_dma_map(dev
, frag
,
1866 skb_frag_size(frag
) - len
,
1867 lenmss
, DMA_TO_DEVICE
);
1868 vring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
1870 pa
= dma_map_single(dev
,
1872 skb_headlen(skb
) - headlen
,
1875 vring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
1879 if (unlikely(dma_mapping_error(dev
, pa
))) {
1880 wil_err(wil
, "TSO: DMA map page error\n");
1884 _desc
= &vring
->va
[i
].tx
.legacy
;
1887 _first_desc
= _desc
;
1888 first_ctx
= &vring
->ctx
[i
];
1894 wil
->txrx_ops
.tx_desc_map((union wil_tx_desc
*)d
,
1895 pa
, lenmss
, vring_index
);
1896 wil_tx_desc_offload_setup_tso(d
, skb
, desc_tso_type
,
1897 is_ipv4
, tcp_hdr_len
,
1900 /* use tso_type_first only once */
1901 desc_tso_type
= wil_tso_type_mid
;
1903 descs_used
++; /* desc used so far */
1904 sg_desc_cnt
++; /* desc used for this segment */
1909 "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n",
1910 len
, rem_data
, descs_used
, sg_desc_cnt
);
1912 /* Close the segment if reached mss size or last frag*/
1913 if (rem_data
== 0 || (f
== nr_frags
- 1 && len
== 0)) {
1914 if (hdr_compensation_need
) {
1915 /* first segment include hdr desc for
1918 hdr_ctx
->nr_frags
= sg_desc_cnt
;
1919 wil_tx_desc_set_nr_frags(first_desc
,
1922 hdr_compensation_need
= false;
1924 wil_tx_desc_set_nr_frags(first_desc
,
1927 first_ctx
->nr_frags
= sg_desc_cnt
- 1;
1929 wil_tx_last_desc(d
);
1931 /* first descriptor may also be the last
1932 * for this mss - make sure not to copy
1935 if (first_desc
!= d
)
1936 *_first_desc
= *first_desc
;
1938 /*last descriptor will be copied at the end
1939 * of this TS processing
1941 if (f
< nr_frags
- 1 || len
> 0)
1947 } else if (first_desc
!= d
) /* update mid descriptor */
1955 /* first descriptor may also be the last.
1956 * in this case d pointer is invalid
1958 if (_first_desc
== _desc
)
1961 /* Last data descriptor */
1962 wil_set_tx_desc_last_tso(d
);
1965 /* Fill the total number of descriptors in first desc (hdr)*/
1966 wil_tx_desc_set_nr_frags(hdr_desc
, descs_used
);
1967 *_hdr_desc
= *hdr_desc
;
1969 /* hold reference to skb
1970 * to prevent skb release before accounting
1971 * in case of immediate "tx done"
1973 vring
->ctx
[i
].skb
= skb_get(skb
);
1975 /* performance monitoring */
1976 used
= wil_ring_used_tx(vring
);
1977 if (wil_val_in_range(wil
->ring_idle_trsh
,
1978 used
, used
+ descs_used
)) {
1979 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
1980 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
1981 vring_index
, used
, used
+ descs_used
);
1984 /* Make sure to advance the head only after descriptor update is done.
1985 * This will prevent a race condition where the completion thread
1986 * will see the DU bit set from previous run and will handle the
1987 * skb before it was completed.
1991 /* advance swhead */
1992 wil_ring_advance_head(vring
, descs_used
);
1993 wil_dbg_txrx(wil
, "TSO: Tx swhead %d -> %d\n", swhead
, vring
->swhead
);
1995 /* make sure all writes to descriptors (shared memory) are done before
1996 * committing them to HW
2000 if (wil
->tx_latency
)
2001 *(ktime_t
*)&skb
->cb
= ktime_get();
2003 memset(skb
->cb
, 0, sizeof(ktime_t
));
2005 wil_w(wil
, vring
->hwtail
, vring
->swhead
);
2009 while (descs_used
> 0) {
2010 struct wil_ctx
*ctx
;
2012 i
= (swhead
+ descs_used
- 1) % vring
->size
;
2013 d
= (struct vring_tx_desc
*)&vring
->va
[i
].tx
.legacy
;
2014 _desc
= &vring
->va
[i
].tx
.legacy
;
2016 _desc
->dma
.status
= TX_DMA_STATUS_DU
;
2017 ctx
= &vring
->ctx
[i
];
2018 wil_txdesc_unmap(dev
, (union wil_tx_desc
*)d
, ctx
);
2019 memset(ctx
, 0, sizeof(*ctx
));
2026 static int __wil_tx_ring(struct wil6210_priv
*wil
, struct wil6210_vif
*vif
,
2027 struct wil_ring
*ring
, struct sk_buff
*skb
)
2029 struct device
*dev
= wil_to_dev(wil
);
2030 struct vring_tx_desc dd
, *d
= &dd
;
2031 volatile struct vring_tx_desc
*_d
;
2032 u32 swhead
= ring
->swhead
;
2033 int avail
= wil_ring_avail_tx(ring
);
2034 int nr_frags
= skb_shinfo(skb
)->nr_frags
;
2036 int ring_index
= ring
- wil
->ring_tx
;
2037 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ring_index
];
2041 bool mcast
= (ring_index
== vif
->bcast_ring
);
2042 uint len
= skb_headlen(skb
);
2044 wil_dbg_txrx(wil
, "tx_ring: %d bytes to ring %d, nr_frags %d\n",
2045 skb
->len
, ring_index
, nr_frags
);
2047 if (unlikely(!txdata
->enabled
))
2050 if (unlikely(avail
< 1 + nr_frags
)) {
2051 wil_err_ratelimited(wil
,
2052 "Tx ring[%2d] full. No space for %d fragments\n",
2053 ring_index
, 1 + nr_frags
);
2056 _d
= &ring
->va
[i
].tx
.legacy
;
2058 pa
= dma_map_single(dev
, skb
->data
, skb_headlen(skb
), DMA_TO_DEVICE
);
2060 wil_dbg_txrx(wil
, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index
,
2061 skb_headlen(skb
), skb
->data
, &pa
);
2062 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET
, 16, 1,
2063 skb
->data
, skb_headlen(skb
), false);
2065 if (unlikely(dma_mapping_error(dev
, pa
)))
2067 ring
->ctx
[i
].mapped_as
= wil_mapped_as_single
;
2069 wil
->txrx_ops
.tx_desc_map((union wil_tx_desc
*)d
, pa
, len
,
2071 if (unlikely(mcast
)) {
2072 d
->mac
.d
[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS
); /* MCS 0 */
2073 if (unlikely(len
> WIL_BCAST_MCS0_LIMIT
)) /* set MCS 1 */
2074 d
->mac
.d
[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS
);
2076 /* Process TCP/UDP checksum offloading */
2077 if (unlikely(wil_tx_desc_offload_setup(d
, skb
))) {
2078 wil_err(wil
, "Tx[%2d] Failed to set cksum, drop packet\n",
2083 ring
->ctx
[i
].nr_frags
= nr_frags
;
2084 wil_tx_desc_set_nr_frags(d
, nr_frags
+ 1);
2086 /* middle segments */
2087 for (; f
< nr_frags
; f
++) {
2088 const skb_frag_t
*frag
= &skb_shinfo(skb
)->frags
[f
];
2089 int len
= skb_frag_size(frag
);
2092 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", ring_index
, i
);
2093 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
2094 (const void *)d
, sizeof(*d
), false);
2095 i
= (swhead
+ f
+ 1) % ring
->size
;
2096 _d
= &ring
->va
[i
].tx
.legacy
;
2097 pa
= skb_frag_dma_map(dev
, frag
, 0, skb_frag_size(frag
),
2099 if (unlikely(dma_mapping_error(dev
, pa
))) {
2100 wil_err(wil
, "Tx[%2d] failed to map fragment\n",
2104 ring
->ctx
[i
].mapped_as
= wil_mapped_as_page
;
2105 wil
->txrx_ops
.tx_desc_map((union wil_tx_desc
*)d
,
2106 pa
, len
, ring_index
);
2107 /* no need to check return code -
2108 * if it succeeded for 1-st descriptor,
2109 * it will succeed here too
2111 wil_tx_desc_offload_setup(d
, skb
);
2113 /* for the last seg only */
2114 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS
);
2115 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS
);
2116 d
->dma
.d0
|= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS
);
2118 wil_dbg_txrx(wil
, "Tx[%2d] desc[%4d]\n", ring_index
, i
);
2119 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE
, 32, 4,
2120 (const void *)d
, sizeof(*d
), false);
2122 /* hold reference to skb
2123 * to prevent skb release before accounting
2124 * in case of immediate "tx done"
2126 ring
->ctx
[i
].skb
= skb_get(skb
);
2128 /* performance monitoring */
2129 used
= wil_ring_used_tx(ring
);
2130 if (wil_val_in_range(wil
->ring_idle_trsh
,
2131 used
, used
+ nr_frags
+ 1)) {
2132 txdata
->idle
+= get_cycles() - txdata
->last_idle
;
2133 wil_dbg_txrx(wil
, "Ring[%2d] not idle %d -> %d\n",
2134 ring_index
, used
, used
+ nr_frags
+ 1);
2137 /* Make sure to advance the head only after descriptor update is done.
2138 * This will prevent a race condition where the completion thread
2139 * will see the DU bit set from previous run and will handle the
2140 * skb before it was completed.
2144 /* advance swhead */
2145 wil_ring_advance_head(ring
, nr_frags
+ 1);
2146 wil_dbg_txrx(wil
, "Tx[%2d] swhead %d -> %d\n", ring_index
, swhead
,
2148 trace_wil6210_tx(ring_index
, swhead
, skb
->len
, nr_frags
);
2150 /* make sure all writes to descriptors (shared memory) are done before
2151 * committing them to HW
2155 if (wil
->tx_latency
)
2156 *(ktime_t
*)&skb
->cb
= ktime_get();
2158 memset(skb
->cb
, 0, sizeof(ktime_t
));
2160 wil_w(wil
, ring
->hwtail
, ring
->swhead
);
2164 /* unmap what we have mapped */
2165 nr_frags
= f
+ 1; /* frags mapped + one for skb head */
2166 for (f
= 0; f
< nr_frags
; f
++) {
2167 struct wil_ctx
*ctx
;
2169 i
= (swhead
+ f
) % ring
->size
;
2170 ctx
= &ring
->ctx
[i
];
2171 _d
= &ring
->va
[i
].tx
.legacy
;
2173 _d
->dma
.status
= TX_DMA_STATUS_DU
;
2174 wil
->txrx_ops
.tx_desc_unmap(dev
,
2175 (union wil_tx_desc
*)d
,
2178 memset(ctx
, 0, sizeof(*ctx
));
2184 static int wil_tx_ring(struct wil6210_priv
*wil
, struct wil6210_vif
*vif
,
2185 struct wil_ring
*ring
, struct sk_buff
*skb
)
2187 int ring_index
= ring
- wil
->ring_tx
;
2188 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ring_index
];
2191 spin_lock(&txdata
->lock
);
2193 if (test_bit(wil_status_suspending
, wil
->status
) ||
2194 test_bit(wil_status_suspended
, wil
->status
) ||
2195 test_bit(wil_status_resuming
, wil
->status
)) {
2197 "suspend/resume in progress. drop packet\n");
2198 spin_unlock(&txdata
->lock
);
2202 rc
= (skb_is_gso(skb
) ? wil
->txrx_ops
.tx_ring_tso
: __wil_tx_ring
)
2203 (wil
, vif
, ring
, skb
);
2205 spin_unlock(&txdata
->lock
);
2210 /* Check status of tx vrings and stop/wake net queues if needed
2211 * It will start/stop net queues of a specific VIF net_device.
2213 * This function does one of two checks:
2214 * In case check_stop is true, will check if net queues need to be stopped. If
2215 * the conditions for stopping are met, netif_tx_stop_all_queues() is called.
2216 * In case check_stop is false, will check if net queues need to be waked. If
2217 * the conditions for waking are met, netif_tx_wake_all_queues() is called.
2218 * vring is the vring which is currently being modified by either adding
2219 * descriptors (tx) into it or removing descriptors (tx complete) from it. Can
2220 * be null when irrelevant (e.g. connect/disconnect events).
2222 * The implementation is to stop net queues if modified vring has low
2223 * descriptor availability. Wake if all vrings are not in low descriptor
2224 * availability and modified vring has high descriptor availability.
2226 static inline void __wil_update_net_queues(struct wil6210_priv
*wil
,
2227 struct wil6210_vif
*vif
,
2228 struct wil_ring
*ring
,
2232 int min_ring_id
= wil_get_min_tx_ring_id(wil
);
2238 wil_dbg_txrx(wil
, "vring %d, mid %d, check_stop=%d, stopped=%d",
2239 (int)(ring
- wil
->ring_tx
), vif
->mid
, check_stop
,
2240 vif
->net_queue_stopped
);
2242 wil_dbg_txrx(wil
, "check_stop=%d, mid=%d, stopped=%d",
2243 check_stop
, vif
->mid
, vif
->net_queue_stopped
);
2245 if (ring
&& drop_if_ring_full
)
2246 /* no need to stop/wake net queues */
2249 if (check_stop
== vif
->net_queue_stopped
)
2250 /* net queues already in desired state */
2254 if (!ring
|| unlikely(wil_ring_avail_low(ring
))) {
2255 /* not enough room in the vring */
2256 netif_tx_stop_all_queues(vif_to_ndev(vif
));
2257 vif
->net_queue_stopped
= true;
2258 wil_dbg_txrx(wil
, "netif_tx_stop called\n");
2263 /* Do not wake the queues in suspend flow */
2264 if (test_bit(wil_status_suspending
, wil
->status
) ||
2265 test_bit(wil_status_suspended
, wil
->status
))
2269 for (i
= min_ring_id
; i
< WIL6210_MAX_TX_RINGS
; i
++) {
2270 struct wil_ring
*cur_ring
= &wil
->ring_tx
[i
];
2271 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[i
];
2273 if (txdata
->mid
!= vif
->mid
|| !cur_ring
->va
||
2274 !txdata
->enabled
|| cur_ring
== ring
)
2277 if (wil_ring_avail_low(cur_ring
)) {
2278 wil_dbg_txrx(wil
, "ring %d full, can't wake\n",
2279 (int)(cur_ring
- wil
->ring_tx
));
2284 if (!ring
|| wil_ring_avail_high(ring
)) {
2285 /* enough room in the ring */
2286 wil_dbg_txrx(wil
, "calling netif_tx_wake\n");
2287 netif_tx_wake_all_queues(vif_to_ndev(vif
));
2288 vif
->net_queue_stopped
= false;
2292 void wil_update_net_queues(struct wil6210_priv
*wil
, struct wil6210_vif
*vif
,
2293 struct wil_ring
*ring
, bool check_stop
)
2295 spin_lock(&wil
->net_queue_lock
);
2296 __wil_update_net_queues(wil
, vif
, ring
, check_stop
);
2297 spin_unlock(&wil
->net_queue_lock
);
2300 void wil_update_net_queues_bh(struct wil6210_priv
*wil
, struct wil6210_vif
*vif
,
2301 struct wil_ring
*ring
, bool check_stop
)
2303 spin_lock_bh(&wil
->net_queue_lock
);
2304 __wil_update_net_queues(wil
, vif
, ring
, check_stop
);
2305 spin_unlock_bh(&wil
->net_queue_lock
);
2308 netdev_tx_t
wil_start_xmit(struct sk_buff
*skb
, struct net_device
*ndev
)
2310 struct wil6210_vif
*vif
= ndev_to_vif(ndev
);
2311 struct wil6210_priv
*wil
= vif_to_wil(vif
);
2312 const u8
*da
= wil_skb_get_da(skb
);
2313 bool bcast
= is_multicast_ether_addr(da
);
2314 struct wil_ring
*ring
;
2315 static bool pr_once_fw
;
2318 wil_dbg_txrx(wil
, "start_xmit\n");
2319 if (unlikely(!test_bit(wil_status_fwready
, wil
->status
))) {
2321 wil_err(wil
, "FW not ready\n");
2326 if (unlikely(!test_bit(wil_vif_fwconnected
, vif
->status
))) {
2327 wil_dbg_ratelimited(wil
,
2328 "VIF not connected, packet dropped\n");
2331 if (unlikely(vif
->wdev
.iftype
== NL80211_IFTYPE_MONITOR
)) {
2332 wil_err(wil
, "Xmit in monitor mode not supported\n");
2338 if (vif
->wdev
.iftype
== NL80211_IFTYPE_STATION
&& !vif
->pbss
) {
2339 /* in STA mode (ESS), all to same VRING (to AP) */
2340 ring
= wil_find_tx_ring_sta(wil
, vif
, skb
);
2342 if (vif
->pbss
|| wil_check_multicast_to_unicast(wil
, skb
))
2343 /* in pbss, no bcast VRING - duplicate skb in
2344 * all stations VRINGs
2346 ring
= wil_find_tx_bcast_2(wil
, vif
, skb
);
2347 else if (vif
->wdev
.iftype
== NL80211_IFTYPE_AP
)
2348 /* AP has a dedicated bcast VRING */
2349 ring
= wil_find_tx_bcast_1(wil
, vif
, skb
);
2351 /* unexpected combination, fallback to duplicating
2352 * the skb in all stations VRINGs
2354 ring
= wil_find_tx_bcast_2(wil
, vif
, skb
);
2356 /* unicast, find specific VRING by dest. address */
2357 ring
= wil_find_tx_ucast(wil
, vif
, skb
);
2359 if (unlikely(!ring
)) {
2360 wil_dbg_txrx(wil
, "No Tx RING found for %pM\n", da
);
2363 /* set up vring entry */
2364 rc
= wil_tx_ring(wil
, vif
, ring
, skb
);
2368 /* shall we stop net queues? */
2369 wil_update_net_queues_bh(wil
, vif
, ring
, true);
2370 /* statistics will be updated on the tx_complete */
2371 dev_kfree_skb_any(skb
);
2372 return NETDEV_TX_OK
;
2374 if (drop_if_ring_full
)
2376 return NETDEV_TX_BUSY
;
2378 break; /* goto drop; */
2381 ndev
->stats
.tx_dropped
++;
2382 dev_kfree_skb_any(skb
);
2384 return NET_XMIT_DROP
;
2387 void wil_tx_latency_calc(struct wil6210_priv
*wil
, struct sk_buff
*skb
,
2388 struct wil_sta_info
*sta
)
2393 if (!wil
->tx_latency
)
2396 if (ktime_to_ms(*(ktime_t
*)&skb
->cb
) == 0)
2399 skb_time_us
= ktime_us_delta(ktime_get(), *(ktime_t
*)&skb
->cb
);
2400 bin
= skb_time_us
/ wil
->tx_latency_res
;
2401 bin
= min_t(int, bin
, WIL_NUM_LATENCY_BINS
- 1);
2403 wil_dbg_txrx(wil
, "skb time %dus => bin %d\n", skb_time_us
, bin
);
2404 sta
->tx_latency_bins
[bin
]++;
2405 sta
->stats
.tx_latency_total_us
+= skb_time_us
;
2406 if (skb_time_us
< sta
->stats
.tx_latency_min_us
)
2407 sta
->stats
.tx_latency_min_us
= skb_time_us
;
2408 if (skb_time_us
> sta
->stats
.tx_latency_max_us
)
2409 sta
->stats
.tx_latency_max_us
= skb_time_us
;
2412 /* Clean up transmitted skb's from the Tx VRING
2414 * Return number of descriptors cleared
2416 * Safe to call from IRQ
2418 int wil_tx_complete(struct wil6210_vif
*vif
, int ringid
)
2420 struct wil6210_priv
*wil
= vif_to_wil(vif
);
2421 struct net_device
*ndev
= vif_to_ndev(vif
);
2422 struct device
*dev
= wil_to_dev(wil
);
2423 struct wil_ring
*vring
= &wil
->ring_tx
[ringid
];
2424 struct wil_ring_tx_data
*txdata
= &wil
->ring_tx_data
[ringid
];
2426 int cid
= wil
->ring2cid_tid
[ringid
][0];
2427 struct wil_net_stats
*stats
= NULL
;
2428 volatile struct vring_tx_desc
*_d
;
2429 int used_before_complete
;
2432 if (unlikely(!vring
->va
)) {
2433 wil_err(wil
, "Tx irq[%d]: vring not initialized\n", ringid
);
2437 if (unlikely(!txdata
->enabled
)) {
2438 wil_info(wil
, "Tx irq[%d]: vring disabled\n", ringid
);
2442 wil_dbg_txrx(wil
, "tx_complete: (%d)\n", ringid
);
2444 used_before_complete
= wil_ring_used_tx(vring
);
2446 if (cid
< wil
->max_assoc_sta
)
2447 stats
= &wil
->sta
[cid
].stats
;
2449 while (!wil_ring_is_empty(vring
)) {
2451 struct wil_ctx
*ctx
= &vring
->ctx
[vring
->swtail
];
2452 /* For the fragmented skb, HW will set DU bit only for the
2453 * last fragment. look for it.
2454 * In TSO the first DU will include hdr desc
2456 int lf
= (vring
->swtail
+ ctx
->nr_frags
) % vring
->size
;
2457 /* TODO: check we are not past head */
2459 _d
= &vring
->va
[lf
].tx
.legacy
;
2460 if (unlikely(!(_d
->dma
.status
& TX_DMA_STATUS_DU
)))
2463 new_swtail
= (lf
+ 1) % vring
->size
;
2464 while (vring
->swtail
!= new_swtail
) {
2465 struct vring_tx_desc dd
, *d
= &dd
;
2467 struct sk_buff
*skb
;
2469 ctx
= &vring
->ctx
[vring
->swtail
];
2471 _d
= &vring
->va
[vring
->swtail
].tx
.legacy
;
2475 dmalen
= le16_to_cpu(d
->dma
.length
);
2476 trace_wil6210_tx_done(ringid
, vring
->swtail
, dmalen
,
2479 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n",
2480 ringid
, vring
->swtail
, dmalen
,
2481 d
->dma
.status
, d
->dma
.error
);
2482 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE
, 32, 4,
2483 (const void *)d
, sizeof(*d
), false);
2485 wil
->txrx_ops
.tx_desc_unmap(dev
,
2486 (union wil_tx_desc
*)d
,
2490 if (likely(d
->dma
.error
== 0)) {
2491 ndev
->stats
.tx_packets
++;
2492 ndev
->stats
.tx_bytes
+= skb
->len
;
2494 stats
->tx_packets
++;
2495 stats
->tx_bytes
+= skb
->len
;
2497 wil_tx_latency_calc(wil
, skb
,
2501 ndev
->stats
.tx_errors
++;
2506 if (skb
->protocol
== cpu_to_be16(ETH_P_PAE
))
2507 wil_tx_complete_handle_eapol(vif
, skb
);
2509 wil_consume_skb(skb
, d
->dma
.error
== 0);
2511 memset(ctx
, 0, sizeof(*ctx
));
2512 /* Make sure the ctx is zeroed before updating the tail
2513 * to prevent a case where wil_tx_ring will see
2514 * this descriptor as used and handle it before ctx zero
2518 /* There is no need to touch HW descriptor:
2519 * - ststus bit TX_DMA_STATUS_DU is set by design,
2520 * so hardware will not try to process this desc.,
2521 * - rest of descriptor will be initialized on Tx.
2523 vring
->swtail
= wil_ring_next_tail(vring
);
2528 /* performance monitoring */
2529 used_new
= wil_ring_used_tx(vring
);
2530 if (wil_val_in_range(wil
->ring_idle_trsh
,
2531 used_new
, used_before_complete
)) {
2532 wil_dbg_txrx(wil
, "Ring[%2d] idle %d -> %d\n",
2533 ringid
, used_before_complete
, used_new
);
2534 txdata
->last_idle
= get_cycles();
2537 /* shall we wake net queues? */
2539 wil_update_net_queues(wil
, vif
, vring
, false);
2544 static inline int wil_tx_init(struct wil6210_priv
*wil
)
2549 static inline void wil_tx_fini(struct wil6210_priv
*wil
) {}
2551 static void wil_get_reorder_params(struct wil6210_priv
*wil
,
2552 struct sk_buff
*skb
, int *tid
, int *cid
,
2553 int *mid
, u16
*seq
, int *mcast
, int *retry
)
2555 struct vring_rx_desc
*d
= wil_skb_rxdesc(skb
);
2557 *tid
= wil_rxdesc_tid(d
);
2558 *cid
= wil_skb_get_cid(skb
);
2559 *mid
= wil_rxdesc_mid(d
);
2560 *seq
= wil_rxdesc_seq(d
);
2561 *mcast
= wil_rxdesc_mcast(d
);
2562 *retry
= wil_rxdesc_retry(d
);
2565 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv
*wil
)
2567 wil
->txrx_ops
.configure_interrupt_moderation
=
2568 wil_configure_interrupt_moderation
;
2570 wil
->txrx_ops
.tx_desc_map
= wil_tx_desc_map
;
2571 wil
->txrx_ops
.tx_desc_unmap
= wil_txdesc_unmap
;
2572 wil
->txrx_ops
.tx_ring_tso
= __wil_tx_vring_tso
;
2573 wil
->txrx_ops
.ring_init_tx
= wil_vring_init_tx
;
2574 wil
->txrx_ops
.ring_fini_tx
= wil_vring_free
;
2575 wil
->txrx_ops
.ring_init_bcast
= wil_vring_init_bcast
;
2576 wil
->txrx_ops
.tx_init
= wil_tx_init
;
2577 wil
->txrx_ops
.tx_fini
= wil_tx_fini
;
2578 wil
->txrx_ops
.tx_ring_modify
= wil_tx_vring_modify
;
2580 wil
->txrx_ops
.rx_init
= wil_rx_init
;
2581 wil
->txrx_ops
.wmi_addba_rx_resp
= wmi_addba_rx_resp
;
2582 wil
->txrx_ops
.get_reorder_params
= wil_get_reorder_params
;
2583 wil
->txrx_ops
.get_netif_rx_params
=
2584 wil_get_netif_rx_params
;
2585 wil
->txrx_ops
.rx_crypto_check
= wil_rx_crypto_check
;
2586 wil
->txrx_ops
.rx_error_check
= wil_rx_error_check
;
2587 wil
->txrx_ops
.is_rx_idle
= wil_is_rx_idle
;
2588 wil
->txrx_ops
.rx_fini
= wil_rx_fini
;