?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
blob483935b001c8ef48bb9e1fddfcd270e48612f5b5
1 /* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 * Copyright (c) 2016-2019 Broadcom Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/ip.h>
41 #include <net/tcp.h>
42 #include <net/udp.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <net/udp_tunnel.h>
46 #include <linux/workqueue.h>
47 #include <linux/prefetch.h>
48 #include <linux/cache.h>
49 #include <linux/log2.h>
50 #include <linux/aer.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <net/page_pool.h>
58
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_ulp.h"
62 #include "bnxt_sriov.h"
63 #include "bnxt_ethtool.h"
64 #include "bnxt_dcb.h"
65 #include "bnxt_xdp.h"
66 #include "bnxt_vfr.h"
67 #include "bnxt_tc.h"
68 #include "bnxt_devlink.h"
69 #include "bnxt_debugfs.h"
70
71 #define BNXT_TX_TIMEOUT (5 * HZ)
72
73 static const char version[] =
74 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
75
76 MODULE_LICENSE("GPL");
77 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
78 MODULE_VERSION(DRV_MODULE_VERSION);
79
80 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
81 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
82 #define BNXT_RX_COPY_THRESH 256
83
84 #define BNXT_TX_PUSH_THRESH 164
85
86 enum board_idx {
87 BCM57301,
88 BCM57302,
89 BCM57304,
90 BCM57417_NPAR,
91 BCM58700,
92 BCM57311,
93 BCM57312,
94 BCM57402,
95 BCM57404,
96 BCM57406,
97 BCM57402_NPAR,
98 BCM57407,
99 BCM57412,
100 BCM57414,
101 BCM57416,
102 BCM57417,
103 BCM57412_NPAR,
104 BCM57314,
105 BCM57417_SFP,
106 BCM57416_SFP,
107 BCM57404_NPAR,
108 BCM57406_NPAR,
109 BCM57407_SFP,
110 BCM57407_NPAR,
111 BCM57414_NPAR,
112 BCM57416_NPAR,
113 BCM57452,
114 BCM57454,
115 BCM5745x_NPAR,
116 BCM57508,
117 BCM57504,
118 BCM57502,
119 BCM57508_NPAR,
120 BCM57504_NPAR,
121 BCM57502_NPAR,
122 BCM58802,
123 BCM58804,
124 BCM58808,
125 NETXTREME_E_VF,
126 NETXTREME_C_VF,
127 NETXTREME_S_VF,
128 NETXTREME_E_P5_VF,
129 };
130
131 /* indexed by enum above */
132 static const struct {
133 char *name;
134 } board_info[] = {
135 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
136 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
137 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
138 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
139 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
140 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
141 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
142 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
143 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
144 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
145 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
146 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
147 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
148 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
149 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
150 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
151 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
152 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
153 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
154 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
155 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
156 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
157 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
158 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
159 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
160 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
161 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
162 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
163 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
164 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
165 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
166 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
167 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
168 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
169 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
170 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
171 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
172 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
173 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
174 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
175 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
176 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
177 };
178
179 static const struct pci_device_id bnxt_pci_tbl[] = {
180 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
181 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
182 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
183 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
184 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
185 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
186 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
187 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
188 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
189 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
190 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
191 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
192 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
193 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
194 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
195 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
196 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
197 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
198 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
199 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
200 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
201 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
202 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
203 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
204 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
205 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
206 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
207 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
208 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
209 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
210 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
211 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
212 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
213 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
214 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
215 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
216 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
217 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
218 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
219 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
220 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
221 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
222 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
223 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
224 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
225 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
226 #ifdef CONFIG_BNXT_SRIOV
227 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
228 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
229 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
230 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
231 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
232 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
233 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
234 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
235 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
236 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
237 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
238 #endif
239 { 0 }
240 };
241
242 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
243
244 static const u16 bnxt_vf_req_snif[] = {
245 HWRM_FUNC_CFG,
246 HWRM_FUNC_VF_CFG,
247 HWRM_PORT_PHY_QCFG,
248 HWRM_CFA_L2_FILTER_ALLOC,
249 };
250
251 static const u16 bnxt_async_events_arr[] = {
252 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
253 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
254 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
255 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
256 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
257 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
258 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
259 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
260 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
261 };
262
263 static struct workqueue_struct *bnxt_pf_wq;
264
265 static bool bnxt_vf_pciid(enum board_idx idx)
266 {
267 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
268 idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
269 }
270
271 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
272 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
273 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
274
275 #define BNXT_CP_DB_IRQ_DIS(db) \
276 writel(DB_CP_IRQ_DIS_FLAGS, db)
277
278 #define BNXT_DB_CQ(db, idx) \
279 writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
280
281 #define BNXT_DB_NQ_P5(db, idx) \
282 writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
283
284 #define BNXT_DB_CQ_ARM(db, idx) \
285 writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
286
287 #define BNXT_DB_NQ_ARM_P5(db, idx) \
288 writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
289
290 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
291 {
292 if (bp->flags & BNXT_FLAG_CHIP_P5)
293 BNXT_DB_NQ_P5(db, idx);
294 else
295 BNXT_DB_CQ(db, idx);
296 }
297
298 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
299 {
300 if (bp->flags & BNXT_FLAG_CHIP_P5)
301 BNXT_DB_NQ_ARM_P5(db, idx);
302 else
303 BNXT_DB_CQ_ARM(db, idx);
304 }
305
306 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
307 {
308 if (bp->flags & BNXT_FLAG_CHIP_P5)
309 writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
310 db->doorbell);
311 else
312 BNXT_DB_CQ(db, idx);
313 }
314
315 const u16 bnxt_lhint_arr[] = {
316 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
317 TX_BD_FLAGS_LHINT_512_TO_1023,
318 TX_BD_FLAGS_LHINT_1024_TO_2047,
319 TX_BD_FLAGS_LHINT_1024_TO_2047,
320 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
321 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
322 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
323 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
324 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
325 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
326 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
327 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
328 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
329 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
330 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
331 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
332 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
333 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
334 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
335 };
336
337 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
338 {
339 struct metadata_dst *md_dst = skb_metadata_dst(skb);
340
341 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
342 return 0;
343
344 return md_dst->u.port_info.port_id;
345 }
346
347 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
348 {
349 struct bnxt *bp = netdev_priv(dev);
350 struct tx_bd *txbd;
351 struct tx_bd_ext *txbd1;
352 struct netdev_queue *txq;
353 int i;
354 dma_addr_t mapping;
355 unsigned int length, pad = 0;
356 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
357 u16 prod, last_frag;
358 struct pci_dev *pdev = bp->pdev;
359 struct bnxt_tx_ring_info *txr;
360 struct bnxt_sw_tx_bd *tx_buf;
361
362 i = skb_get_queue_mapping(skb);
363 if (unlikely(i >= bp->tx_nr_rings)) {
364 dev_kfree_skb_any(skb);
365 return NETDEV_TX_OK;
366 }
367
368 txq = netdev_get_tx_queue(dev, i);
369 txr = &bp->tx_ring[bp->tx_ring_map[i]];
370 prod = txr->tx_prod;
371
372 free_size = bnxt_tx_avail(bp, txr);
373 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
374 netif_tx_stop_queue(txq);
375 return NETDEV_TX_BUSY;
376 }
377
378 length = skb->len;
379 len = skb_headlen(skb);
380 last_frag = skb_shinfo(skb)->nr_frags;
381
382 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
383
384 txbd->tx_bd_opaque = prod;
385
386 tx_buf = &txr->tx_buf_ring[prod];
387 tx_buf->skb = skb;
388 tx_buf->nr_frags = last_frag;
389
390 vlan_tag_flags = 0;
391 cfa_action = bnxt_xmit_get_cfa_action(skb);
392 if (skb_vlan_tag_present(skb)) {
393 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
394 skb_vlan_tag_get(skb);
395 /* Currently supports 8021Q, 8021AD vlan offloads
396 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
397 */
398 if (skb->vlan_proto == htons(ETH_P_8021Q))
399 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
400 }
401
402 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
403 struct tx_push_buffer *tx_push_buf = txr->tx_push;
404 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
405 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
406 void __iomem *db = txr->tx_db.doorbell;
407 void *pdata = tx_push_buf->data;
408 u64 *end;
409 int j, push_len;
410
411 /* Set COAL_NOW to be ready quickly for the next push */
412 tx_push->tx_bd_len_flags_type =
413 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
414 TX_BD_TYPE_LONG_TX_BD |
415 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
416 TX_BD_FLAGS_COAL_NOW |
417 TX_BD_FLAGS_PACKET_END |
418 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
419
420 if (skb->ip_summed == CHECKSUM_PARTIAL)
421 tx_push1->tx_bd_hsize_lflags =
422 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
423 else
424 tx_push1->tx_bd_hsize_lflags = 0;
425
426 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
427 tx_push1->tx_bd_cfa_action =
428 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
429
430 end = pdata + length;
431 end = PTR_ALIGN(end, 8) - 1;
432 *end = 0;
433
434 skb_copy_from_linear_data(skb, pdata, len);
435 pdata += len;
436 for (j = 0; j < last_frag; j++) {
437 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
438 void *fptr;
439
440 fptr = skb_frag_address_safe(frag);
441 if (!fptr)
442 goto normal_tx;
443
444 memcpy(pdata, fptr, skb_frag_size(frag));
445 pdata += skb_frag_size(frag);
446 }
447
448 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
449 txbd->tx_bd_haddr = txr->data_mapping;
450 prod = NEXT_TX(prod);
451 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
452 memcpy(txbd, tx_push1, sizeof(*txbd));
453 prod = NEXT_TX(prod);
454 tx_push->doorbell =
455 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
456 txr->tx_prod = prod;
457
458 tx_buf->is_push = 1;
459 netdev_tx_sent_queue(txq, skb->len);
460 wmb(); /* Sync is_push and byte queue before pushing data */
461
462 push_len = (length + sizeof(*tx_push) + 7) / 8;
463 if (push_len > 16) {
464 __iowrite64_copy(db, tx_push_buf, 16);
465 __iowrite32_copy(db + 4, tx_push_buf + 1,
466 (push_len - 16) << 1);
467 } else {
468 __iowrite64_copy(db, tx_push_buf, push_len);
469 }
470
471 goto tx_done;
472 }
473
474 normal_tx:
475 if (length < BNXT_MIN_PKT_SIZE) {
476 pad = BNXT_MIN_PKT_SIZE - length;
477 if (skb_pad(skb, pad)) {
478 /* SKB already freed. */
479 tx_buf->skb = NULL;
480 return NETDEV_TX_OK;
481 }
482 length = BNXT_MIN_PKT_SIZE;
483 }
484
485 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
486
487 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
488 dev_kfree_skb_any(skb);
489 tx_buf->skb = NULL;
490 return NETDEV_TX_OK;
491 }
492
493 dma_unmap_addr_set(tx_buf, mapping, mapping);
494 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
495 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
496
497 txbd->tx_bd_haddr = cpu_to_le64(mapping);
498
499 prod = NEXT_TX(prod);
500 txbd1 = (struct tx_bd_ext *)
501 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
502
503 txbd1->tx_bd_hsize_lflags = 0;
504 if (skb_is_gso(skb)) {
505 u32 hdr_len;
506
507 if (skb->encapsulation)
508 hdr_len = skb_inner_network_offset(skb) +
509 skb_inner_network_header_len(skb) +
510 inner_tcp_hdrlen(skb);
511 else
512 hdr_len = skb_transport_offset(skb) +
513 tcp_hdrlen(skb);
514
515 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
516 TX_BD_FLAGS_T_IPID |
517 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
518 length = skb_shinfo(skb)->gso_size;
519 txbd1->tx_bd_mss = cpu_to_le32(length);
520 length += hdr_len;
521 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
522 txbd1->tx_bd_hsize_lflags =
523 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
524 txbd1->tx_bd_mss = 0;
525 }
526
527 length >>= 9;
528 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
529 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
530 skb->len);
531 i = 0;
532 goto tx_dma_error;
533 }
534 flags |= bnxt_lhint_arr[length];
535 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
536
537 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
538 txbd1->tx_bd_cfa_action =
539 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
540 for (i = 0; i < last_frag; i++) {
541 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
542
543 prod = NEXT_TX(prod);
544 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
545
546 len = skb_frag_size(frag);
547 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
548 DMA_TO_DEVICE);
549
550 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
551 goto tx_dma_error;
552
553 tx_buf = &txr->tx_buf_ring[prod];
554 dma_unmap_addr_set(tx_buf, mapping, mapping);
555
556 txbd->tx_bd_haddr = cpu_to_le64(mapping);
557
558 flags = len << TX_BD_LEN_SHIFT;
559 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
560 }
561
562 flags &= ~TX_BD_LEN;
563 txbd->tx_bd_len_flags_type =
564 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
565 TX_BD_FLAGS_PACKET_END);
566
567 netdev_tx_sent_queue(txq, skb->len);
568
569 /* Sync BD data before updating doorbell */
570 wmb();
571
572 prod = NEXT_TX(prod);
573 txr->tx_prod = prod;
574
575 if (!netdev_xmit_more() || netif_xmit_stopped(txq))
576 bnxt_db_write(bp, &txr->tx_db, prod);
577
578 tx_done:
579
580 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
581 if (netdev_xmit_more() && !tx_buf->is_push)
582 bnxt_db_write(bp, &txr->tx_db, prod);
583
584 netif_tx_stop_queue(txq);
585
586 /* netif_tx_stop_queue() must be done before checking
587 * tx index in bnxt_tx_avail() below, because in
588 * bnxt_tx_int(), we update tx index before checking for
589 * netif_tx_queue_stopped().
590 */
591 smp_mb();
592 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
593 netif_tx_wake_queue(txq);
594 }
595 return NETDEV_TX_OK;
596
597 tx_dma_error:
598 last_frag = i;
599
600 /* start back at beginning and unmap skb */
601 prod = txr->tx_prod;
602 tx_buf = &txr->tx_buf_ring[prod];
603 tx_buf->skb = NULL;
604 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
605 skb_headlen(skb), PCI_DMA_TODEVICE);
606 prod = NEXT_TX(prod);
607
608 /* unmap remaining mapped pages */
609 for (i = 0; i < last_frag; i++) {
610 prod = NEXT_TX(prod);
611 tx_buf = &txr->tx_buf_ring[prod];
612 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
613 skb_frag_size(&skb_shinfo(skb)->frags[i]),
614 PCI_DMA_TODEVICE);
615 }
616
617 dev_kfree_skb_any(skb);
618 return NETDEV_TX_OK;
619 }
620
621 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
622 {
623 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
624 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
625 u16 cons = txr->tx_cons;
626 struct pci_dev *pdev = bp->pdev;
627 int i;
628 unsigned int tx_bytes = 0;
629
630 for (i = 0; i < nr_pkts; i++) {
631 struct bnxt_sw_tx_bd *tx_buf;
632 struct sk_buff *skb;
633 int j, last;
634
635 tx_buf = &txr->tx_buf_ring[cons];
636 cons = NEXT_TX(cons);
637 skb = tx_buf->skb;
638 tx_buf->skb = NULL;
639
640 if (tx_buf->is_push) {
641 tx_buf->is_push = 0;
642 goto next_tx_int;
643 }
644
645 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
646 skb_headlen(skb), PCI_DMA_TODEVICE);
647 last = tx_buf->nr_frags;
648
649 for (j = 0; j < last; j++) {
650 cons = NEXT_TX(cons);
651 tx_buf = &txr->tx_buf_ring[cons];
652 dma_unmap_page(
653 &pdev->dev,
654 dma_unmap_addr(tx_buf, mapping),
655 skb_frag_size(&skb_shinfo(skb)->frags[j]),
656 PCI_DMA_TODEVICE);
657 }
658
659 next_tx_int:
660 cons = NEXT_TX(cons);
661
662 tx_bytes += skb->len;
663 dev_kfree_skb_any(skb);
664 }
665
666 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
667 txr->tx_cons = cons;
668
669 /* Need to make the tx_cons update visible to bnxt_start_xmit()
670 * before checking for netif_tx_queue_stopped(). Without the
671 * memory barrier, there is a small possibility that bnxt_start_xmit()
672 * will miss it and cause the queue to be stopped forever.
673 */
674 smp_mb();
675
676 if (unlikely(netif_tx_queue_stopped(txq)) &&
677 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
678 __netif_tx_lock(txq, smp_processor_id());
679 if (netif_tx_queue_stopped(txq) &&
680 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
681 txr->dev_state != BNXT_DEV_STATE_CLOSING)
682 netif_tx_wake_queue(txq);
683 __netif_tx_unlock(txq);
684 }
685 }
686
687 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
688 struct bnxt_rx_ring_info *rxr,
689 gfp_t gfp)
690 {
691 struct device *dev = &bp->pdev->dev;
692 struct page *page;
693
694 page = page_pool_dev_alloc_pages(rxr->page_pool);
695 if (!page)
696 return NULL;
697
698 *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
699 DMA_ATTR_WEAK_ORDERING);
700 if (dma_mapping_error(dev, *mapping)) {
701 page_pool_recycle_direct(rxr->page_pool, page);
702 return NULL;
703 }
704 *mapping += bp->rx_dma_offset;
705 return page;
706 }
707
708 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
709 gfp_t gfp)
710 {
711 u8 *data;
712 struct pci_dev *pdev = bp->pdev;
713
714 data = kmalloc(bp->rx_buf_size, gfp);
715 if (!data)
716 return NULL;
717
718 *mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
719 bp->rx_buf_use_size, bp->rx_dir,
720 DMA_ATTR_WEAK_ORDERING);
721
722 if (dma_mapping_error(&pdev->dev, *mapping)) {
723 kfree(data);
724 data = NULL;
725 }
726 return data;
727 }
728
729 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
730 u16 prod, gfp_t gfp)
731 {
732 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
733 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
734 dma_addr_t mapping;
735
736 if (BNXT_RX_PAGE_MODE(bp)) {
737 struct page *page =
738 __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
739
740 if (!page)
741 return -ENOMEM;
742
743 rx_buf->data = page;
744 rx_buf->data_ptr = page_address(page) + bp->rx_offset;
745 } else {
746 u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
747
748 if (!data)
749 return -ENOMEM;
750
751 rx_buf->data = data;
752 rx_buf->data_ptr = data + bp->rx_offset;
753 }
754 rx_buf->mapping = mapping;
755
756 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
757 return 0;
758 }
759
760 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
761 {
762 u16 prod = rxr->rx_prod;
763 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
764 struct rx_bd *cons_bd, *prod_bd;
765
766 prod_rx_buf = &rxr->rx_buf_ring[prod];
767 cons_rx_buf = &rxr->rx_buf_ring[cons];
768
769 prod_rx_buf->data = data;
770 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
771
772 prod_rx_buf->mapping = cons_rx_buf->mapping;
773
774 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
775 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
776
777 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
778 }
779
780 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
781 {
782 u16 next, max = rxr->rx_agg_bmap_size;
783
784 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
785 if (next >= max)
786 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
787 return next;
788 }
789
790 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
791 struct bnxt_rx_ring_info *rxr,
792 u16 prod, gfp_t gfp)
793 {
794 struct rx_bd *rxbd =
795 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
796 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
797 struct pci_dev *pdev = bp->pdev;
798 struct page *page;
799 dma_addr_t mapping;
800 u16 sw_prod = rxr->rx_sw_agg_prod;
801 unsigned int offset = 0;
802
803 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
804 page = rxr->rx_page;
805 if (!page) {
806 page = alloc_page(gfp);
807 if (!page)
808 return -ENOMEM;
809 rxr->rx_page = page;
810 rxr->rx_page_offset = 0;
811 }
812 offset = rxr->rx_page_offset;
813 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
814 if (rxr->rx_page_offset == PAGE_SIZE)
815 rxr->rx_page = NULL;
816 else
817 get_page(page);
818 } else {
819 page = alloc_page(gfp);
820 if (!page)
821 return -ENOMEM;
822 }
823
824 mapping = dma_map_page_attrs(&pdev->dev, page, offset,
825 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
826 DMA_ATTR_WEAK_ORDERING);
827 if (dma_mapping_error(&pdev->dev, mapping)) {
828 __free_page(page);
829 return -EIO;
830 }
831
832 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
833 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
834
835 __set_bit(sw_prod, rxr->rx_agg_bmap);
836 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
837 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
838
839 rx_agg_buf->page = page;
840 rx_agg_buf->offset = offset;
841 rx_agg_buf->mapping = mapping;
842 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
843 rxbd->rx_bd_opaque = sw_prod;
844 return 0;
845 }
846
847 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
848 struct bnxt_cp_ring_info *cpr,
849 u16 cp_cons, u16 curr)
850 {
851 struct rx_agg_cmp *agg;
852
853 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
854 agg = (struct rx_agg_cmp *)
855 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
856 return agg;
857 }
858
859 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
860 struct bnxt_rx_ring_info *rxr,
861 u16 agg_id, u16 curr)
862 {
863 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
864
865 return &tpa_info->agg_arr[curr];
866 }
867
868 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
869 u16 start, u32 agg_bufs, bool tpa)
870 {
871 struct bnxt_napi *bnapi = cpr->bnapi;
872 struct bnxt *bp = bnapi->bp;
873 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
874 u16 prod = rxr->rx_agg_prod;
875 u16 sw_prod = rxr->rx_sw_agg_prod;
876 bool p5_tpa = false;
877 u32 i;
878
879 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
880 p5_tpa = true;
881
882 for (i = 0; i < agg_bufs; i++) {
883 u16 cons;
884 struct rx_agg_cmp *agg;
885 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
886 struct rx_bd *prod_bd;
887 struct page *page;
888
889 if (p5_tpa)
890 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
891 else
892 agg = bnxt_get_agg(bp, cpr, idx, start + i);
893 cons = agg->rx_agg_cmp_opaque;
894 __clear_bit(cons, rxr->rx_agg_bmap);
895
896 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
897 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
898
899 __set_bit(sw_prod, rxr->rx_agg_bmap);
900 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
901 cons_rx_buf = &rxr->rx_agg_ring[cons];
902
903 /* It is possible for sw_prod to be equal to cons, so
904 * set cons_rx_buf->page to NULL first.
905 */
906 page = cons_rx_buf->page;
907 cons_rx_buf->page = NULL;
908 prod_rx_buf->page = page;
909 prod_rx_buf->offset = cons_rx_buf->offset;
910
911 prod_rx_buf->mapping = cons_rx_buf->mapping;
912
913 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
914
915 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
916 prod_bd->rx_bd_opaque = sw_prod;
917
918 prod = NEXT_RX_AGG(prod);
919 sw_prod = NEXT_RX_AGG(sw_prod);
920 }
921 rxr->rx_agg_prod = prod;
922 rxr->rx_sw_agg_prod = sw_prod;
923 }
924
925 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
926 struct bnxt_rx_ring_info *rxr,
927 u16 cons, void *data, u8 *data_ptr,
928 dma_addr_t dma_addr,
929 unsigned int offset_and_len)
930 {
931 unsigned int payload = offset_and_len >> 16;
932 unsigned int len = offset_and_len & 0xffff;
933 skb_frag_t *frag;
934 struct page *page = data;
935 u16 prod = rxr->rx_prod;
936 struct sk_buff *skb;
937 int off, err;
938
939 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
940 if (unlikely(err)) {
941 bnxt_reuse_rx_data(rxr, cons, data);
942 return NULL;
943 }
944 dma_addr -= bp->rx_dma_offset;
945 dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
946 DMA_ATTR_WEAK_ORDERING);
947 page_pool_release_page(rxr->page_pool, page);
948
949 if (unlikely(!payload))
950 payload = eth_get_headlen(bp->dev, data_ptr, len);
951
952 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
953 if (!skb) {
954 __free_page(page);
955 return NULL;
956 }
957
958 off = (void *)data_ptr - page_address(page);
959 skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
960 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
961 payload + NET_IP_ALIGN);
962
963 frag = &skb_shinfo(skb)->frags[0];
964 skb_frag_size_sub(frag, payload);
965 skb_frag_off_add(frag, payload);
966 skb->data_len -= payload;
967 skb->tail += payload;
968
969 return skb;
970 }
971
972 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
973 struct bnxt_rx_ring_info *rxr, u16 cons,
974 void *data, u8 *data_ptr,
975 dma_addr_t dma_addr,
976 unsigned int offset_and_len)
977 {
978 u16 prod = rxr->rx_prod;
979 struct sk_buff *skb;
980 int err;
981
982 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
983 if (unlikely(err)) {
984 bnxt_reuse_rx_data(rxr, cons, data);
985 return NULL;
986 }
987
988 skb = build_skb(data, 0);
989 dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
990 bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
991 if (!skb) {
992 kfree(data);
993 return NULL;
994 }
995
996 skb_reserve(skb, bp->rx_offset);
997 skb_put(skb, offset_and_len & 0xffff);
998 return skb;
999 }
1000
1001 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1002 struct bnxt_cp_ring_info *cpr,
1003 struct sk_buff *skb, u16 idx,
1004 u32 agg_bufs, bool tpa)
1005 {
1006 struct bnxt_napi *bnapi = cpr->bnapi;
1007 struct pci_dev *pdev = bp->pdev;
1008 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1009 u16 prod = rxr->rx_agg_prod;
1010 bool p5_tpa = false;
1011 u32 i;
1012
1013 if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1014 p5_tpa = true;
1015
1016 for (i = 0; i < agg_bufs; i++) {
1017 u16 cons, frag_len;
1018 struct rx_agg_cmp *agg;
1019 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1020 struct page *page;
1021 dma_addr_t mapping;
1022
1023 if (p5_tpa)
1024 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1025 else
1026 agg = bnxt_get_agg(bp, cpr, idx, i);
1027 cons = agg->rx_agg_cmp_opaque;
1028 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1029 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1030
1031 cons_rx_buf = &rxr->rx_agg_ring[cons];
1032 skb_fill_page_desc(skb, i, cons_rx_buf->page,
1033 cons_rx_buf->offset, frag_len);
1034 __clear_bit(cons, rxr->rx_agg_bmap);
1035
1036 /* It is possible for bnxt_alloc_rx_page() to allocate
1037 * a sw_prod index that equals the cons index, so we
1038 * need to clear the cons entry now.
1039 */
1040 mapping = cons_rx_buf->mapping;
1041 page = cons_rx_buf->page;
1042 cons_rx_buf->page = NULL;
1043
1044 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1045 struct skb_shared_info *shinfo;
1046 unsigned int nr_frags;
1047
1048 shinfo = skb_shinfo(skb);
1049 nr_frags = --shinfo->nr_frags;
1050 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1051
1052 dev_kfree_skb(skb);
1053
1054 cons_rx_buf->page = page;
1055
1056 /* Update prod since possibly some pages have been
1057 * allocated already.
1058 */
1059 rxr->rx_agg_prod = prod;
1060 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1061 return NULL;
1062 }
1063
1064 dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1065 PCI_DMA_FROMDEVICE,
1066 DMA_ATTR_WEAK_ORDERING);
1067
1068 skb->data_len += frag_len;
1069 skb->len += frag_len;
1070 skb->truesize += PAGE_SIZE;
1071
1072 prod = NEXT_RX_AGG(prod);
1073 }
1074 rxr->rx_agg_prod = prod;
1075 return skb;
1076 }
1077
1078 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1079 u8 agg_bufs, u32 *raw_cons)
1080 {
1081 u16 last;
1082 struct rx_agg_cmp *agg;
1083
1084 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1085 last = RING_CMP(*raw_cons);
1086 agg = (struct rx_agg_cmp *)
1087 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1088 return RX_AGG_CMP_VALID(agg, *raw_cons);
1089 }
1090
1091 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1092 unsigned int len,
1093 dma_addr_t mapping)
1094 {
1095 struct bnxt *bp = bnapi->bp;
1096 struct pci_dev *pdev = bp->pdev;
1097 struct sk_buff *skb;
1098
1099 skb = napi_alloc_skb(&bnapi->napi, len);
1100 if (!skb)
1101 return NULL;
1102
1103 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1104 bp->rx_dir);
1105
1106 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1107 len + NET_IP_ALIGN);
1108
1109 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1110 bp->rx_dir);
1111
1112 skb_put(skb, len);
1113 return skb;
1114 }
1115
1116 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1117 u32 *raw_cons, void *cmp)
1118 {
1119 struct rx_cmp *rxcmp = cmp;
1120 u32 tmp_raw_cons = *raw_cons;
1121 u8 cmp_type, agg_bufs = 0;
1122
1123 cmp_type = RX_CMP_TYPE(rxcmp);
1124
1125 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1126 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1127 RX_CMP_AGG_BUFS) >>
1128 RX_CMP_AGG_BUFS_SHIFT;
1129 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1130 struct rx_tpa_end_cmp *tpa_end = cmp;
1131
1132 if (bp->flags & BNXT_FLAG_CHIP_P5)
1133 return 0;
1134
1135 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1136 }
1137
1138 if (agg_bufs) {
1139 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1140 return -EBUSY;
1141 }
1142 *raw_cons = tmp_raw_cons;
1143 return 0;
1144 }
1145
1146 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1147 {
1148 if (BNXT_PF(bp))
1149 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1150 else
1151 schedule_delayed_work(&bp->fw_reset_task, delay);
1152 }
1153
1154 static void bnxt_queue_sp_work(struct bnxt *bp)
1155 {
1156 if (BNXT_PF(bp))
1157 queue_work(bnxt_pf_wq, &bp->sp_task);
1158 else
1159 schedule_work(&bp->sp_task);
1160 }
1161
1162 static void bnxt_cancel_sp_work(struct bnxt *bp)
1163 {
1164 if (BNXT_PF(bp))
1165 flush_workqueue(bnxt_pf_wq);
1166 else
1167 cancel_work_sync(&bp->sp_task);
1168 }
1169
1170 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1171 {
1172 if (!rxr->bnapi->in_reset) {
1173 rxr->bnapi->in_reset = true;
1174 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1175 bnxt_queue_sp_work(bp);
1176 }
1177 rxr->rx_next_cons = 0xffff;
1178 }
1179
1180 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1181 {
1182 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1183 u16 idx = agg_id & MAX_TPA_P5_MASK;
1184
1185 if (test_bit(idx, map->agg_idx_bmap))
1186 idx = find_first_zero_bit(map->agg_idx_bmap,
1187 BNXT_AGG_IDX_BMAP_SIZE);
1188 __set_bit(idx, map->agg_idx_bmap);
1189 map->agg_id_tbl[agg_id] = idx;
1190 return idx;
1191 }
1192
1193 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1194 {
1195 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1196
1197 __clear_bit(idx, map->agg_idx_bmap);
1198 }
1199
1200 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1201 {
1202 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1203
1204 return map->agg_id_tbl[agg_id];
1205 }
1206
1207 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1208 struct rx_tpa_start_cmp *tpa_start,
1209 struct rx_tpa_start_cmp_ext *tpa_start1)
1210 {
1211 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1212 struct bnxt_tpa_info *tpa_info;
1213 u16 cons, prod, agg_id;
1214 struct rx_bd *prod_bd;
1215 dma_addr_t mapping;
1216
1217 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1218 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1219 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1220 } else {
1221 agg_id = TPA_START_AGG_ID(tpa_start);
1222 }
1223 cons = tpa_start->rx_tpa_start_cmp_opaque;
1224 prod = rxr->rx_prod;
1225 cons_rx_buf = &rxr->rx_buf_ring[cons];
1226 prod_rx_buf = &rxr->rx_buf_ring[prod];
1227 tpa_info = &rxr->rx_tpa[agg_id];
1228
1229 if (unlikely(cons != rxr->rx_next_cons ||
1230 TPA_START_ERROR(tpa_start))) {
1231 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1232 cons, rxr->rx_next_cons,
1233 TPA_START_ERROR_CODE(tpa_start1));
1234 bnxt_sched_reset(bp, rxr);
1235 return;
1236 }
1237 /* Store cfa_code in tpa_info to use in tpa_end
1238 * completion processing.
1239 */
1240 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1241 prod_rx_buf->data = tpa_info->data;
1242 prod_rx_buf->data_ptr = tpa_info->data_ptr;
1243
1244 mapping = tpa_info->mapping;
1245 prod_rx_buf->mapping = mapping;
1246
1247 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1248
1249 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1250
1251 tpa_info->data = cons_rx_buf->data;
1252 tpa_info->data_ptr = cons_rx_buf->data_ptr;
1253 cons_rx_buf->data = NULL;
1254 tpa_info->mapping = cons_rx_buf->mapping;
1255
1256 tpa_info->len =
1257 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1258 RX_TPA_START_CMP_LEN_SHIFT;
1259 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1260 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1261
1262 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1263 tpa_info->gso_type = SKB_GSO_TCPV4;
1264 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1265 if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1266 tpa_info->gso_type = SKB_GSO_TCPV6;
1267 tpa_info->rss_hash =
1268 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1269 } else {
1270 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1271 tpa_info->gso_type = 0;
1272 if (netif_msg_rx_err(bp))
1273 netdev_warn(bp->dev, "TPA packet without valid hash\n");
1274 }
1275 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1276 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1277 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1278 tpa_info->agg_count = 0;
1279
1280 rxr->rx_prod = NEXT_RX(prod);
1281 cons = NEXT_RX(cons);
1282 rxr->rx_next_cons = NEXT_RX(cons);
1283 cons_rx_buf = &rxr->rx_buf_ring[cons];
1284
1285 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1286 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1287 cons_rx_buf->data = NULL;
1288 }
1289
1290 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1291 {
1292 if (agg_bufs)
1293 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1294 }
1295
1296 #ifdef CONFIG_INET
1297 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1298 {
1299 struct udphdr *uh = NULL;
1300
1301 if (ip_proto == htons(ETH_P_IP)) {
1302 struct iphdr *iph = (struct iphdr *)skb->data;
1303
1304 if (iph->protocol == IPPROTO_UDP)
1305 uh = (struct udphdr *)(iph + 1);
1306 } else {
1307 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1308
1309 if (iph->nexthdr == IPPROTO_UDP)
1310 uh = (struct udphdr *)(iph + 1);
1311 }
1312 if (uh) {
1313 if (uh->check)
1314 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1315 else
1316 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1317 }
1318 }
1319 #endif
1320
1321 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1322 int payload_off, int tcp_ts,
1323 struct sk_buff *skb)
1324 {
1325 #ifdef CONFIG_INET
1326 struct tcphdr *th;
1327 int len, nw_off;
1328 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1329 u32 hdr_info = tpa_info->hdr_info;
1330 bool loopback = false;
1331
1332 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1333 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1334 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1335
1336 /* If the packet is an internal loopback packet, the offsets will
1337 * have an extra 4 bytes.
1338 */
1339 if (inner_mac_off == 4) {
1340 loopback = true;
1341 } else if (inner_mac_off > 4) {
1342 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1343 ETH_HLEN - 2));
1344
1345 /* We only support inner iPv4/ipv6. If we don't see the
1346 * correct protocol ID, it must be a loopback packet where
1347 * the offsets are off by 4.
1348 */
1349 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1350 loopback = true;
1351 }
1352 if (loopback) {
1353 /* internal loopback packet, subtract all offsets by 4 */
1354 inner_ip_off -= 4;
1355 inner_mac_off -= 4;
1356 outer_ip_off -= 4;
1357 }
1358
1359 nw_off = inner_ip_off - ETH_HLEN;
1360 skb_set_network_header(skb, nw_off);
1361 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1362 struct ipv6hdr *iph = ipv6_hdr(skb);
1363
1364 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1365 len = skb->len - skb_transport_offset(skb);
1366 th = tcp_hdr(skb);
1367 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1368 } else {
1369 struct iphdr *iph = ip_hdr(skb);
1370
1371 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1372 len = skb->len - skb_transport_offset(skb);
1373 th = tcp_hdr(skb);
1374 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1375 }
1376
1377 if (inner_mac_off) { /* tunnel */
1378 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1379 ETH_HLEN - 2));
1380
1381 bnxt_gro_tunnel(skb, proto);
1382 }
1383 #endif
1384 return skb;
1385 }
1386
1387 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1388 int payload_off, int tcp_ts,
1389 struct sk_buff *skb)
1390 {
1391 #ifdef CONFIG_INET
1392 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1393 u32 hdr_info = tpa_info->hdr_info;
1394 int iphdr_len, nw_off;
1395
1396 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1397 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1398 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1399
1400 nw_off = inner_ip_off - ETH_HLEN;
1401 skb_set_network_header(skb, nw_off);
1402 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1403 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1404 skb_set_transport_header(skb, nw_off + iphdr_len);
1405
1406 if (inner_mac_off) { /* tunnel */
1407 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1408 ETH_HLEN - 2));
1409
1410 bnxt_gro_tunnel(skb, proto);
1411 }
1412 #endif
1413 return skb;
1414 }
1415
1416 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1417 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1418
1419 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1420 int payload_off, int tcp_ts,
1421 struct sk_buff *skb)
1422 {
1423 #ifdef CONFIG_INET
1424 struct tcphdr *th;
1425 int len, nw_off, tcp_opt_len = 0;
1426
1427 if (tcp_ts)
1428 tcp_opt_len = 12;
1429
1430 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1431 struct iphdr *iph;
1432
1433 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1434 ETH_HLEN;
1435 skb_set_network_header(skb, nw_off);
1436 iph = ip_hdr(skb);
1437 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1438 len = skb->len - skb_transport_offset(skb);
1439 th = tcp_hdr(skb);
1440 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1441 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1442 struct ipv6hdr *iph;
1443
1444 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1445 ETH_HLEN;
1446 skb_set_network_header(skb, nw_off);
1447 iph = ipv6_hdr(skb);
1448 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1449 len = skb->len - skb_transport_offset(skb);
1450 th = tcp_hdr(skb);
1451 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1452 } else {
1453 dev_kfree_skb_any(skb);
1454 return NULL;
1455 }
1456
1457 if (nw_off) /* tunnel */
1458 bnxt_gro_tunnel(skb, skb->protocol);
1459 #endif
1460 return skb;
1461 }
1462
1463 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1464 struct bnxt_tpa_info *tpa_info,
1465 struct rx_tpa_end_cmp *tpa_end,
1466 struct rx_tpa_end_cmp_ext *tpa_end1,
1467 struct sk_buff *skb)
1468 {
1469 #ifdef CONFIG_INET
1470 int payload_off;
1471 u16 segs;
1472
1473 segs = TPA_END_TPA_SEGS(tpa_end);
1474 if (segs == 1)
1475 return skb;
1476
1477 NAPI_GRO_CB(skb)->count = segs;
1478 skb_shinfo(skb)->gso_size =
1479 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1480 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1481 if (bp->flags & BNXT_FLAG_CHIP_P5)
1482 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1483 else
1484 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1485 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1486 if (likely(skb))
1487 tcp_gro_complete(skb);
1488 #endif
1489 return skb;
1490 }
1491
1492 /* Given the cfa_code of a received packet determine which
1493 * netdev (vf-rep or PF) the packet is destined to.
1494 */
1495 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1496 {
1497 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1498
1499 /* if vf-rep dev is NULL, the must belongs to the PF */
1500 return dev ? dev : bp->dev;
1501 }
1502
1503 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1504 struct bnxt_cp_ring_info *cpr,
1505 u32 *raw_cons,
1506 struct rx_tpa_end_cmp *tpa_end,
1507 struct rx_tpa_end_cmp_ext *tpa_end1,
1508 u8 *event)
1509 {
1510 struct bnxt_napi *bnapi = cpr->bnapi;
1511 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1512 u8 *data_ptr, agg_bufs;
1513 unsigned int len;
1514 struct bnxt_tpa_info *tpa_info;
1515 dma_addr_t mapping;
1516 struct sk_buff *skb;
1517 u16 idx = 0, agg_id;
1518 void *data;
1519 bool gro;
1520
1521 if (unlikely(bnapi->in_reset)) {
1522 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1523
1524 if (rc < 0)
1525 return ERR_PTR(-EBUSY);
1526 return NULL;
1527 }
1528
1529 if (bp->flags & BNXT_FLAG_CHIP_P5) {
1530 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1531 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1532 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1533 tpa_info = &rxr->rx_tpa[agg_id];
1534 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1535 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1536 agg_bufs, tpa_info->agg_count);
1537 agg_bufs = tpa_info->agg_count;
1538 }
1539 tpa_info->agg_count = 0;
1540 *event |= BNXT_AGG_EVENT;
1541 bnxt_free_agg_idx(rxr, agg_id);
1542 idx = agg_id;
1543 gro = !!(bp->flags & BNXT_FLAG_GRO);
1544 } else {
1545 agg_id = TPA_END_AGG_ID(tpa_end);
1546 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1547 tpa_info = &rxr->rx_tpa[agg_id];
1548 idx = RING_CMP(*raw_cons);
1549 if (agg_bufs) {
1550 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1551 return ERR_PTR(-EBUSY);
1552
1553 *event |= BNXT_AGG_EVENT;
1554 idx = NEXT_CMP(idx);
1555 }
1556 gro = !!TPA_END_GRO(tpa_end);
1557 }
1558 data = tpa_info->data;
1559 data_ptr = tpa_info->data_ptr;
1560 prefetch(data_ptr);
1561 len = tpa_info->len;
1562 mapping = tpa_info->mapping;
1563
1564 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1565 bnxt_abort_tpa(cpr, idx, agg_bufs);
1566 if (agg_bufs > MAX_SKB_FRAGS)
1567 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1568 agg_bufs, (int)MAX_SKB_FRAGS);
1569 return NULL;
1570 }
1571
1572 if (len <= bp->rx_copy_thresh) {
1573 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1574 if (!skb) {
1575 bnxt_abort_tpa(cpr, idx, agg_bufs);
1576 return NULL;
1577 }
1578 } else {
1579 u8 *new_data;
1580 dma_addr_t new_mapping;
1581
1582 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1583 if (!new_data) {
1584 bnxt_abort_tpa(cpr, idx, agg_bufs);
1585 return NULL;
1586 }
1587
1588 tpa_info->data = new_data;
1589 tpa_info->data_ptr = new_data + bp->rx_offset;
1590 tpa_info->mapping = new_mapping;
1591
1592 skb = build_skb(data, 0);
1593 dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1594 bp->rx_buf_use_size, bp->rx_dir,
1595 DMA_ATTR_WEAK_ORDERING);
1596
1597 if (!skb) {
1598 kfree(data);
1599 bnxt_abort_tpa(cpr, idx, agg_bufs);
1600 return NULL;
1601 }
1602 skb_reserve(skb, bp->rx_offset);
1603 skb_put(skb, len);
1604 }
1605
1606 if (agg_bufs) {
1607 skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1608 if (!skb) {
1609 /* Page reuse already handled by bnxt_rx_pages(). */
1610 return NULL;
1611 }
1612 }
1613
1614 skb->protocol =
1615 eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1616
1617 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1618 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1619
1620 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1621 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1622 u16 vlan_proto = tpa_info->metadata >>
1623 RX_CMP_FLAGS2_METADATA_TPID_SFT;
1624 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1625
1626 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1627 }
1628
1629 skb_checksum_none_assert(skb);
1630 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1631 skb->ip_summed = CHECKSUM_UNNECESSARY;
1632 skb->csum_level =
1633 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1634 }
1635
1636 if (gro)
1637 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1638
1639 return skb;
1640 }
1641
1642 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1643 struct rx_agg_cmp *rx_agg)
1644 {
1645 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1646 struct bnxt_tpa_info *tpa_info;
1647
1648 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1649 tpa_info = &rxr->rx_tpa[agg_id];
1650 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1651 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1652 }
1653
1654 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1655 struct sk_buff *skb)
1656 {
1657 if (skb->dev != bp->dev) {
1658 /* this packet belongs to a vf-rep */
1659 bnxt_vf_rep_rx(bp, skb);
1660 return;
1661 }
1662 skb_record_rx_queue(skb, bnapi->index);
1663 napi_gro_receive(&bnapi->napi, skb);
1664 }
1665
1666 /* returns the following:
1667 * 1 - 1 packet successfully received
1668 * 0 - successful TPA_START, packet not completed yet
1669 * -EBUSY - completion ring does not have all the agg buffers yet
1670 * -ENOMEM - packet aborted due to out of memory
1671 * -EIO - packet aborted due to hw error indicated in BD
1672 */
1673 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1674 u32 *raw_cons, u8 *event)
1675 {
1676 struct bnxt_napi *bnapi = cpr->bnapi;
1677 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1678 struct net_device *dev = bp->dev;
1679 struct rx_cmp *rxcmp;
1680 struct rx_cmp_ext *rxcmp1;
1681 u32 tmp_raw_cons = *raw_cons;
1682 u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1683 struct bnxt_sw_rx_bd *rx_buf;
1684 unsigned int len;
1685 u8 *data_ptr, agg_bufs, cmp_type;
1686 dma_addr_t dma_addr;
1687 struct sk_buff *skb;
1688 void *data;
1689 int rc = 0;
1690 u32 misc;
1691
1692 rxcmp = (struct rx_cmp *)
1693 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1694
1695 cmp_type = RX_CMP_TYPE(rxcmp);
1696
1697 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1698 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1699 goto next_rx_no_prod_no_len;
1700 }
1701
1702 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1703 cp_cons = RING_CMP(tmp_raw_cons);
1704 rxcmp1 = (struct rx_cmp_ext *)
1705 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1706
1707 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1708 return -EBUSY;
1709
1710 prod = rxr->rx_prod;
1711
1712 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1713 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1714 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1715
1716 *event |= BNXT_RX_EVENT;
1717 goto next_rx_no_prod_no_len;
1718
1719 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1720 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1721 (struct rx_tpa_end_cmp *)rxcmp,
1722 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1723
1724 if (IS_ERR(skb))
1725 return -EBUSY;
1726
1727 rc = -ENOMEM;
1728 if (likely(skb)) {
1729 bnxt_deliver_skb(bp, bnapi, skb);
1730 rc = 1;
1731 }
1732 *event |= BNXT_RX_EVENT;
1733 goto next_rx_no_prod_no_len;
1734 }
1735
1736 cons = rxcmp->rx_cmp_opaque;
1737 if (unlikely(cons != rxr->rx_next_cons)) {
1738 int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1739
1740 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1741 cons, rxr->rx_next_cons);
1742 bnxt_sched_reset(bp, rxr);
1743 return rc1;
1744 }
1745 rx_buf = &rxr->rx_buf_ring[cons];
1746 data = rx_buf->data;
1747 data_ptr = rx_buf->data_ptr;
1748 prefetch(data_ptr);
1749
1750 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1751 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1752
1753 if (agg_bufs) {
1754 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1755 return -EBUSY;
1756
1757 cp_cons = NEXT_CMP(cp_cons);
1758 *event |= BNXT_AGG_EVENT;
1759 }
1760 *event |= BNXT_RX_EVENT;
1761
1762 rx_buf->data = NULL;
1763 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1764 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1765
1766 bnxt_reuse_rx_data(rxr, cons, data);
1767 if (agg_bufs)
1768 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1769 false);
1770
1771 rc = -EIO;
1772 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1773 bnapi->cp_ring.rx_buf_errors++;
1774 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
1775 netdev_warn(bp->dev, "RX buffer error %x\n",
1776 rx_err);
1777 bnxt_sched_reset(bp, rxr);
1778 }
1779 }
1780 goto next_rx_no_len;
1781 }
1782
1783 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1784 dma_addr = rx_buf->mapping;
1785
1786 if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1787 rc = 1;
1788 goto next_rx;
1789 }
1790
1791 if (len <= bp->rx_copy_thresh) {
1792 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1793 bnxt_reuse_rx_data(rxr, cons, data);
1794 if (!skb) {
1795 if (agg_bufs)
1796 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1797 agg_bufs, false);
1798 rc = -ENOMEM;
1799 goto next_rx;
1800 }
1801 } else {
1802 u32 payload;
1803
1804 if (rx_buf->data_ptr == data_ptr)
1805 payload = misc & RX_CMP_PAYLOAD_OFFSET;
1806 else
1807 payload = 0;
1808 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1809 payload | len);
1810 if (!skb) {
1811 rc = -ENOMEM;
1812 goto next_rx;
1813 }
1814 }
1815
1816 if (agg_bufs) {
1817 skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1818 if (!skb) {
1819 rc = -ENOMEM;
1820 goto next_rx;
1821 }
1822 }
1823
1824 if (RX_CMP_HASH_VALID(rxcmp)) {
1825 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1826 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1827
1828 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1829 if (hash_type != 1 && hash_type != 3)
1830 type = PKT_HASH_TYPE_L3;
1831 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1832 }
1833
1834 cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1835 skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1836
1837 if ((rxcmp1->rx_cmp_flags2 &
1838 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1839 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1840 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1841 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1842 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1843
1844 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1845 }
1846
1847 skb_checksum_none_assert(skb);
1848 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1849 if (dev->features & NETIF_F_RXCSUM) {
1850 skb->ip_summed = CHECKSUM_UNNECESSARY;
1851 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1852 }
1853 } else {
1854 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1855 if (dev->features & NETIF_F_RXCSUM)
1856 bnapi->cp_ring.rx_l4_csum_errors++;
1857 }
1858 }
1859
1860 bnxt_deliver_skb(bp, bnapi, skb);
1861 rc = 1;
1862
1863 next_rx:
1864 cpr->rx_packets += 1;
1865 cpr->rx_bytes += len;
1866
1867 next_rx_no_len:
1868 rxr->rx_prod = NEXT_RX(prod);
1869 rxr->rx_next_cons = NEXT_RX(cons);
1870
1871 next_rx_no_prod_no_len:
1872 *raw_cons = tmp_raw_cons;
1873
1874 return rc;
1875 }
1876
1877 /* In netpoll mode, if we are using a combined completion ring, we need to
1878 * discard the rx packets and recycle the buffers.
1879 */
1880 static int bnxt_force_rx_discard(struct bnxt *bp,
1881 struct bnxt_cp_ring_info *cpr,
1882 u32 *raw_cons, u8 *event)
1883 {
1884 u32 tmp_raw_cons = *raw_cons;
1885 struct rx_cmp_ext *rxcmp1;
1886 struct rx_cmp *rxcmp;
1887 u16 cp_cons;
1888 u8 cmp_type;
1889
1890 cp_cons = RING_CMP(tmp_raw_cons);
1891 rxcmp = (struct rx_cmp *)
1892 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1893
1894 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1895 cp_cons = RING_CMP(tmp_raw_cons);
1896 rxcmp1 = (struct rx_cmp_ext *)
1897 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1898
1899 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1900 return -EBUSY;
1901
1902 cmp_type = RX_CMP_TYPE(rxcmp);
1903 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1904 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1905 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1906 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1907 struct rx_tpa_end_cmp_ext *tpa_end1;
1908
1909 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1910 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1911 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1912 }
1913 return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1914 }
1915
1916 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1917 {
1918 struct bnxt_fw_health *fw_health = bp->fw_health;
1919 u32 reg = fw_health->regs[reg_idx];
1920 u32 reg_type, reg_off, val = 0;
1921
1922 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1923 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1924 switch (reg_type) {
1925 case BNXT_FW_HEALTH_REG_TYPE_CFG:
1926 pci_read_config_dword(bp->pdev, reg_off, &val);
1927 break;
1928 case BNXT_FW_HEALTH_REG_TYPE_GRC:
1929 reg_off = fw_health->mapped_regs[reg_idx];
1930 /* fall through */
1931 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1932 val = readl(bp->bar0 + reg_off);
1933 break;
1934 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1935 val = readl(bp->bar1 + reg_off);
1936 break;
1937 }
1938 if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1939 val &= fw_health->fw_reset_inprog_reg_mask;
1940 return val;
1941 }
1942
1943 #define BNXT_GET_EVENT_PORT(data) \
1944 ((data) & \
1945 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1946
1947 static int bnxt_async_event_process(struct bnxt *bp,
1948 struct hwrm_async_event_cmpl *cmpl)
1949 {
1950 u16 event_id = le16_to_cpu(cmpl->event_id);
1951
1952 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1953 switch (event_id) {
1954 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1955 u32 data1 = le32_to_cpu(cmpl->event_data1);
1956 struct bnxt_link_info *link_info = &bp->link_info;
1957
1958 if (BNXT_VF(bp))
1959 goto async_event_process_exit;
1960
1961 /* print unsupported speed warning in forced speed mode only */
1962 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1963 (data1 & 0x20000)) {
1964 u16 fw_speed = link_info->force_link_speed;
1965 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1966
1967 if (speed != SPEED_UNKNOWN)
1968 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1969 speed);
1970 }
1971 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1972 }
1973 /* fall through */
1974 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1975 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1976 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
1977 /* fall through */
1978 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1979 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1980 break;
1981 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1982 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1983 break;
1984 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1985 u32 data1 = le32_to_cpu(cmpl->event_data1);
1986 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1987
1988 if (BNXT_VF(bp))
1989 break;
1990
1991 if (bp->pf.port_id != port_id)
1992 break;
1993
1994 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1995 break;
1996 }
1997 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1998 if (BNXT_PF(bp))
1999 goto async_event_process_exit;
2000 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2001 break;
2002 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2003 u32 data1 = le32_to_cpu(cmpl->event_data1);
2004
2005 if (!bp->fw_health)
2006 goto async_event_process_exit;
2007
2008 bp->fw_reset_timestamp = jiffies;
2009 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2010 if (!bp->fw_reset_min_dsecs)
2011 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2012 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2013 if (!bp->fw_reset_max_dsecs)
2014 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2015 if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2016 netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2017 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2018 } else {
2019 netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2020 bp->fw_reset_max_dsecs * 100);
2021 }
2022 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2023 break;
2024 }
2025 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2026 struct bnxt_fw_health *fw_health = bp->fw_health;
2027 u32 data1 = le32_to_cpu(cmpl->event_data1);
2028
2029 if (!fw_health)
2030 goto async_event_process_exit;
2031
2032 fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2033 fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2034 if (!fw_health->enabled)
2035 break;
2036
2037 if (netif_msg_drv(bp))
2038 netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2039 fw_health->enabled, fw_health->master,
2040 bnxt_fw_health_readl(bp,
2041 BNXT_FW_RESET_CNT_REG),
2042 bnxt_fw_health_readl(bp,
2043 BNXT_FW_HEALTH_REG));
2044 fw_health->tmr_multiplier =
2045 DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2046 bp->current_interval * 10);
2047 fw_health->tmr_counter = fw_health->tmr_multiplier;
2048 fw_health->last_fw_heartbeat =
2049 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2050 fw_health->last_fw_reset_cnt =
2051 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2052 goto async_event_process_exit;
2053 }
2054 default:
2055 goto async_event_process_exit;
2056 }
2057 bnxt_queue_sp_work(bp);
2058 async_event_process_exit:
2059 bnxt_ulp_async_events(bp, cmpl);
2060 return 0;
2061 }
2062
2063 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2064 {
2065 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2066 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2067 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2068 (struct hwrm_fwd_req_cmpl *)txcmp;
2069
2070 switch (cmpl_type) {
2071 case CMPL_BASE_TYPE_HWRM_DONE:
2072 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2073 if (seq_id == bp->hwrm_intr_seq_id)
2074 bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2075 else
2076 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2077 break;
2078
2079 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2080 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2081
2082 if ((vf_id < bp->pf.first_vf_id) ||
2083 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2084 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2085 vf_id);
2086 return -EINVAL;
2087 }
2088
2089 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2090 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2091 bnxt_queue_sp_work(bp);
2092 break;
2093
2094 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2095 bnxt_async_event_process(bp,
2096 (struct hwrm_async_event_cmpl *)txcmp);
2097
2098 default:
2099 break;
2100 }
2101
2102 return 0;
2103 }
2104
2105 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2106 {
2107 struct bnxt_napi *bnapi = dev_instance;
2108 struct bnxt *bp = bnapi->bp;
2109 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2110 u32 cons = RING_CMP(cpr->cp_raw_cons);
2111
2112 cpr->event_ctr++;
2113 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2114 napi_schedule(&bnapi->napi);
2115 return IRQ_HANDLED;
2116 }
2117
2118 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2119 {
2120 u32 raw_cons = cpr->cp_raw_cons;
2121 u16 cons = RING_CMP(raw_cons);
2122 struct tx_cmp *txcmp;
2123
2124 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2125
2126 return TX_CMP_VALID(txcmp, raw_cons);
2127 }
2128
2129 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2130 {
2131 struct bnxt_napi *bnapi = dev_instance;
2132 struct bnxt *bp = bnapi->bp;
2133 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2134 u32 cons = RING_CMP(cpr->cp_raw_cons);
2135 u32 int_status;
2136
2137 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2138
2139 if (!bnxt_has_work(bp, cpr)) {
2140 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2141 /* return if erroneous interrupt */
2142 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2143 return IRQ_NONE;
2144 }
2145
2146 /* disable ring IRQ */
2147 BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2148
2149 /* Return here if interrupt is shared and is disabled. */
2150 if (unlikely(atomic_read(&bp->intr_sem) != 0))
2151 return IRQ_HANDLED;
2152
2153 napi_schedule(&bnapi->napi);
2154 return IRQ_HANDLED;
2155 }
2156
2157 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2158 int budget)
2159 {
2160 struct bnxt_napi *bnapi = cpr->bnapi;
2161 u32 raw_cons = cpr->cp_raw_cons;
2162 u32 cons;
2163 int tx_pkts = 0;
2164 int rx_pkts = 0;
2165 u8 event = 0;
2166 struct tx_cmp *txcmp;
2167
2168 cpr->has_more_work = 0;
2169 while (1) {
2170 int rc;
2171
2172 cons = RING_CMP(raw_cons);
2173 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2174
2175 if (!TX_CMP_VALID(txcmp, raw_cons))
2176 break;
2177
2178 /* The valid test of the entry must be done first before
2179 * reading any further.
2180 */
2181 dma_rmb();
2182 cpr->had_work_done = 1;
2183 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2184 tx_pkts++;
2185 /* return full budget so NAPI will complete. */
2186 if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2187 rx_pkts = budget;
2188 raw_cons = NEXT_RAW_CMP(raw_cons);
2189 if (budget)
2190 cpr->has_more_work = 1;
2191 break;
2192 }
2193 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2194 if (likely(budget))
2195 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2196 else
2197 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2198 &event);
2199 if (likely(rc >= 0))
2200 rx_pkts += rc;
2201 /* Increment rx_pkts when rc is -ENOMEM to count towards
2202 * the NAPI budget. Otherwise, we may potentially loop
2203 * here forever if we consistently cannot allocate
2204 * buffers.
2205 */
2206 else if (rc == -ENOMEM && budget)
2207 rx_pkts++;
2208 else if (rc == -EBUSY) /* partial completion */
2209 break;
2210 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
2211 CMPL_BASE_TYPE_HWRM_DONE) ||
2212 (TX_CMP_TYPE(txcmp) ==
2213 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2214 (TX_CMP_TYPE(txcmp) ==
2215 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2216 bnxt_hwrm_handler(bp, txcmp);
2217 }
2218 raw_cons = NEXT_RAW_CMP(raw_cons);
2219
2220 if (rx_pkts && rx_pkts == budget) {
2221 cpr->has_more_work = 1;
2222 break;
2223 }
2224 }
2225
2226 if (event & BNXT_REDIRECT_EVENT)
2227 xdp_do_flush_map();
2228
2229 if (event & BNXT_TX_EVENT) {
2230 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2231 u16 prod = txr->tx_prod;
2232
2233 /* Sync BD data before updating doorbell */
2234 wmb();
2235
2236 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2237 }
2238
2239 cpr->cp_raw_cons = raw_cons;
2240 bnapi->tx_pkts += tx_pkts;
2241 bnapi->events |= event;
2242 return rx_pkts;
2243 }
2244
2245 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2246 {
2247 if (bnapi->tx_pkts) {
2248 bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2249 bnapi->tx_pkts = 0;
2250 }
2251
2252 if (bnapi->events & BNXT_RX_EVENT) {
2253 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2254
2255 if (bnapi->events & BNXT_AGG_EVENT)
2256 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2257 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2258 }
2259 bnapi->events = 0;
2260 }
2261
2262 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2263 int budget)
2264 {
2265 struct bnxt_napi *bnapi = cpr->bnapi;
2266 int rx_pkts;
2267
2268 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2269
2270 /* ACK completion ring before freeing tx ring and producing new
2271 * buffers in rx/agg rings to prevent overflowing the completion
2272 * ring.
2273 */
2274 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2275
2276 __bnxt_poll_work_done(bp, bnapi);
2277 return rx_pkts;
2278 }
2279
2280 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2281 {
2282 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2283 struct bnxt *bp = bnapi->bp;
2284 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2285 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2286 struct tx_cmp *txcmp;
2287 struct rx_cmp_ext *rxcmp1;
2288 u32 cp_cons, tmp_raw_cons;
2289 u32 raw_cons = cpr->cp_raw_cons;
2290 u32 rx_pkts = 0;
2291 u8 event = 0;
2292
2293 while (1) {
2294 int rc;
2295
2296 cp_cons = RING_CMP(raw_cons);
2297 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2298
2299 if (!TX_CMP_VALID(txcmp, raw_cons))
2300 break;
2301
2302 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2303 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2304 cp_cons = RING_CMP(tmp_raw_cons);
2305 rxcmp1 = (struct rx_cmp_ext *)
2306 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2307
2308 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2309 break;
2310
2311 /* force an error to recycle the buffer */
2312 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2313 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2314
2315 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2316 if (likely(rc == -EIO) && budget)
2317 rx_pkts++;
2318 else if (rc == -EBUSY) /* partial completion */
2319 break;
2320 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
2321 CMPL_BASE_TYPE_HWRM_DONE)) {
2322 bnxt_hwrm_handler(bp, txcmp);
2323 } else {
2324 netdev_err(bp->dev,
2325 "Invalid completion received on special ring\n");
2326 }
2327 raw_cons = NEXT_RAW_CMP(raw_cons);
2328
2329 if (rx_pkts == budget)
2330 break;
2331 }
2332
2333 cpr->cp_raw_cons = raw_cons;
2334 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2335 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2336
2337 if (event & BNXT_AGG_EVENT)
2338 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2339
2340 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2341 napi_complete_done(napi, rx_pkts);
2342 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2343 }
2344 return rx_pkts;
2345 }
2346
2347 static int bnxt_poll(struct napi_struct *napi, int budget)
2348 {
2349 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2350 struct bnxt *bp = bnapi->bp;
2351 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2352 int work_done = 0;
2353
2354 while (1) {
2355 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2356
2357 if (work_done >= budget) {
2358 if (!budget)
2359 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2360 break;
2361 }
2362
2363 if (!bnxt_has_work(bp, cpr)) {
2364 if (napi_complete_done(napi, work_done))
2365 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2366 break;
2367 }
2368 }
2369 if (bp->flags & BNXT_FLAG_DIM) {
2370 struct dim_sample dim_sample = {};
2371
2372 dim_update_sample(cpr->event_ctr,
2373 cpr->rx_packets,
2374 cpr->rx_bytes,
2375 &dim_sample);
2376 net_dim(&cpr->dim, dim_sample);
2377 }
2378 return work_done;
2379 }
2380
2381 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2382 {
2383 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2384 int i, work_done = 0;
2385
2386 for (i = 0; i < 2; i++) {
2387 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2388
2389 if (cpr2) {
2390 work_done += __bnxt_poll_work(bp, cpr2,
2391 budget - work_done);
2392 cpr->has_more_work |= cpr2->has_more_work;
2393 }
2394 }
2395 return work_done;
2396 }
2397
2398 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2399 u64 dbr_type, bool all)
2400 {
2401 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2402 int i;
2403
2404 for (i = 0; i < 2; i++) {
2405 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2406 struct bnxt_db_info *db;
2407
2408 if (cpr2 && (all || cpr2->had_work_done)) {
2409 db = &cpr2->cp_db;
2410 writeq(db->db_key64 | dbr_type |
2411 RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2412 cpr2->had_work_done = 0;
2413 }
2414 }
2415 __bnxt_poll_work_done(bp, bnapi);
2416 }
2417
2418 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2419 {
2420 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2421 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2422 u32 raw_cons = cpr->cp_raw_cons;
2423 struct bnxt *bp = bnapi->bp;
2424 struct nqe_cn *nqcmp;
2425 int work_done = 0;
2426 u32 cons;
2427
2428 if (cpr->has_more_work) {
2429 cpr->has_more_work = 0;
2430 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2431 if (cpr->has_more_work) {
2432 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, false);
2433 return work_done;
2434 }
2435 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL, true);
2436 if (napi_complete_done(napi, work_done))
2437 BNXT_DB_NQ_ARM_P5(&cpr->cp_db, cpr->cp_raw_cons);
2438 return work_done;
2439 }
2440 while (1) {
2441 cons = RING_CMP(raw_cons);
2442 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2443
2444 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2445 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2446 false);
2447 cpr->cp_raw_cons = raw_cons;
2448 if (napi_complete_done(napi, work_done))
2449 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2450 cpr->cp_raw_cons);
2451 return work_done;
2452 }
2453
2454 /* The valid test of the entry must be done first before
2455 * reading any further.
2456 */
2457 dma_rmb();
2458
2459 if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2460 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2461 struct bnxt_cp_ring_info *cpr2;
2462
2463 cpr2 = cpr->cp_ring_arr[idx];
2464 work_done += __bnxt_poll_work(bp, cpr2,
2465 budget - work_done);
2466 cpr->has_more_work = cpr2->has_more_work;
2467 } else {
2468 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2469 }
2470 raw_cons = NEXT_RAW_CMP(raw_cons);
2471 if (cpr->has_more_work)
2472 break;
2473 }
2474 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, true);
2475 cpr->cp_raw_cons = raw_cons;
2476 return work_done;
2477 }
2478
2479 static void bnxt_free_tx_skbs(struct bnxt *bp)
2480 {
2481 int i, max_idx;
2482 struct pci_dev *pdev = bp->pdev;
2483
2484 if (!bp->tx_ring)
2485 return;
2486
2487 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2488 for (i = 0; i < bp->tx_nr_rings; i++) {
2489 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2490 int j;
2491
2492 for (j = 0; j < max_idx;) {
2493 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2494 struct sk_buff *skb;
2495 int k, last;
2496
2497 if (i < bp->tx_nr_rings_xdp &&
2498 tx_buf->action == XDP_REDIRECT) {
2499 dma_unmap_single(&pdev->dev,
2500 dma_unmap_addr(tx_buf, mapping),
2501 dma_unmap_len(tx_buf, len),
2502 PCI_DMA_TODEVICE);
2503 xdp_return_frame(tx_buf->xdpf);
2504 tx_buf->action = 0;
2505 tx_buf->xdpf = NULL;
2506 j++;
2507 continue;
2508 }
2509
2510 skb = tx_buf->skb;
2511 if (!skb) {
2512 j++;
2513 continue;
2514 }
2515
2516 tx_buf->skb = NULL;
2517
2518 if (tx_buf->is_push) {
2519 dev_kfree_skb(skb);
2520 j += 2;
2521 continue;
2522 }
2523
2524 dma_unmap_single(&pdev->dev,
2525 dma_unmap_addr(tx_buf, mapping),
2526 skb_headlen(skb),
2527 PCI_DMA_TODEVICE);
2528
2529 last = tx_buf->nr_frags;
2530 j += 2;
2531 for (k = 0; k < last; k++, j++) {
2532 int ring_idx = j & bp->tx_ring_mask;
2533 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2534
2535 tx_buf = &txr->tx_buf_ring[ring_idx];
2536 dma_unmap_page(
2537 &pdev->dev,
2538 dma_unmap_addr(tx_buf, mapping),
2539 skb_frag_size(frag), PCI_DMA_TODEVICE);
2540 }
2541 dev_kfree_skb(skb);
2542 }
2543 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2544 }
2545 }
2546
2547 static void bnxt_free_rx_skbs(struct bnxt *bp)
2548 {
2549 int i, max_idx, max_agg_idx;
2550 struct pci_dev *pdev = bp->pdev;
2551
2552 if (!bp->rx_ring)
2553 return;
2554
2555 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2556 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2557 for (i = 0; i < bp->rx_nr_rings; i++) {
2558 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2559 struct bnxt_tpa_idx_map *map;
2560 int j;
2561
2562 if (rxr->rx_tpa) {
2563 for (j = 0; j < bp->max_tpa; j++) {
2564 struct bnxt_tpa_info *tpa_info =
2565 &rxr->rx_tpa[j];
2566 u8 *data = tpa_info->data;
2567
2568 if (!data)
2569 continue;
2570
2571 dma_unmap_single_attrs(&pdev->dev,
2572 tpa_info->mapping,
2573 bp->rx_buf_use_size,
2574 bp->rx_dir,
2575 DMA_ATTR_WEAK_ORDERING);
2576
2577 tpa_info->data = NULL;
2578
2579 kfree(data);
2580 }
2581 }
2582
2583 for (j = 0; j < max_idx; j++) {
2584 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2585 dma_addr_t mapping = rx_buf->mapping;
2586 void *data = rx_buf->data;
2587
2588 if (!data)
2589 continue;
2590
2591 rx_buf->data = NULL;
2592
2593 if (BNXT_RX_PAGE_MODE(bp)) {
2594 mapping -= bp->rx_dma_offset;
2595 dma_unmap_page_attrs(&pdev->dev, mapping,
2596 PAGE_SIZE, bp->rx_dir,
2597 DMA_ATTR_WEAK_ORDERING);
2598 page_pool_recycle_direct(rxr->page_pool, data);
2599 } else {
2600 dma_unmap_single_attrs(&pdev->dev, mapping,
2601 bp->rx_buf_use_size,
2602 bp->rx_dir,
2603 DMA_ATTR_WEAK_ORDERING);
2604 kfree(data);
2605 }
2606 }
2607
2608 for (j = 0; j < max_agg_idx; j++) {
2609 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2610 &rxr->rx_agg_ring[j];
2611 struct page *page = rx_agg_buf->page;
2612
2613 if (!page)
2614 continue;
2615
2616 dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2617 BNXT_RX_PAGE_SIZE,
2618 PCI_DMA_FROMDEVICE,
2619 DMA_ATTR_WEAK_ORDERING);
2620
2621 rx_agg_buf->page = NULL;
2622 __clear_bit(j, rxr->rx_agg_bmap);
2623
2624 __free_page(page);
2625 }
2626 if (rxr->rx_page) {
2627 __free_page(rxr->rx_page);
2628 rxr->rx_page = NULL;
2629 }
2630 map = rxr->rx_tpa_idx_map;
2631 if (map)
2632 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2633 }
2634 }
2635
2636 static void bnxt_free_skbs(struct bnxt *bp)
2637 {
2638 bnxt_free_tx_skbs(bp);
2639 bnxt_free_rx_skbs(bp);
2640 }
2641
2642 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2643 {
2644 struct pci_dev *pdev = bp->pdev;
2645 int i;
2646
2647 for (i = 0; i < rmem->nr_pages; i++) {
2648 if (!rmem->pg_arr[i])
2649 continue;
2650
2651 dma_free_coherent(&pdev->dev, rmem->page_size,
2652 rmem->pg_arr[i], rmem->dma_arr[i]);
2653
2654 rmem->pg_arr[i] = NULL;
2655 }
2656 if (rmem->pg_tbl) {
2657 size_t pg_tbl_size = rmem->nr_pages * 8;
2658
2659 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2660 pg_tbl_size = rmem->page_size;
2661 dma_free_coherent(&pdev->dev, pg_tbl_size,
2662 rmem->pg_tbl, rmem->pg_tbl_map);
2663 rmem->pg_tbl = NULL;
2664 }
2665 if (rmem->vmem_size && *rmem->vmem) {
2666 vfree(*rmem->vmem);
2667 *rmem->vmem = NULL;
2668 }
2669 }
2670
2671 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2672 {
2673 struct pci_dev *pdev = bp->pdev;
2674 u64 valid_bit = 0;
2675 int i;
2676
2677 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2678 valid_bit = PTU_PTE_VALID;
2679 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2680 size_t pg_tbl_size = rmem->nr_pages * 8;
2681
2682 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2683 pg_tbl_size = rmem->page_size;
2684 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2685 &rmem->pg_tbl_map,
2686 GFP_KERNEL);
2687 if (!rmem->pg_tbl)
2688 return -ENOMEM;
2689 }
2690
2691 for (i = 0; i < rmem->nr_pages; i++) {
2692 u64 extra_bits = valid_bit;
2693
2694 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2695 rmem->page_size,
2696 &rmem->dma_arr[i],
2697 GFP_KERNEL);
2698 if (!rmem->pg_arr[i])
2699 return -ENOMEM;
2700
2701 if (rmem->init_val)
2702 memset(rmem->pg_arr[i], rmem->init_val,
2703 rmem->page_size);
2704 if (rmem->nr_pages > 1 || rmem->depth > 0) {
2705 if (i == rmem->nr_pages - 2 &&
2706 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2707 extra_bits |= PTU_PTE_NEXT_TO_LAST;
2708 else if (i == rmem->nr_pages - 1 &&
2709 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2710 extra_bits |= PTU_PTE_LAST;
2711 rmem->pg_tbl[i] =
2712 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2713 }
2714 }
2715
2716 if (rmem->vmem_size) {
2717 *rmem->vmem = vzalloc(rmem->vmem_size);
2718 if (!(*rmem->vmem))
2719 return -ENOMEM;
2720 }
2721 return 0;
2722 }
2723
2724 static void bnxt_free_tpa_info(struct bnxt *bp)
2725 {
2726 int i;
2727
2728 for (i = 0; i < bp->rx_nr_rings; i++) {
2729 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2730
2731 kfree(rxr->rx_tpa_idx_map);
2732 rxr->rx_tpa_idx_map = NULL;
2733 if (rxr->rx_tpa) {
2734 kfree(rxr->rx_tpa[0].agg_arr);
2735 rxr->rx_tpa[0].agg_arr = NULL;
2736 }
2737 kfree(rxr->rx_tpa);
2738 rxr->rx_tpa = NULL;
2739 }
2740 }
2741
2742 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2743 {
2744 int i, j, total_aggs = 0;
2745
2746 bp->max_tpa = MAX_TPA;
2747 if (bp->flags & BNXT_FLAG_CHIP_P5) {
2748 if (!bp->max_tpa_v2)
2749 return 0;
2750 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2751 total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2752 }
2753
2754 for (i = 0; i < bp->rx_nr_rings; i++) {
2755 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2756 struct rx_agg_cmp *agg;
2757
2758 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2759 GFP_KERNEL);
2760 if (!rxr->rx_tpa)
2761 return -ENOMEM;
2762
2763 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2764 continue;
2765 agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2766 rxr->rx_tpa[0].agg_arr = agg;
2767 if (!agg)
2768 return -ENOMEM;
2769 for (j = 1; j < bp->max_tpa; j++)
2770 rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2771 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2772 GFP_KERNEL);
2773 if (!rxr->rx_tpa_idx_map)
2774 return -ENOMEM;
2775 }
2776 return 0;
2777 }
2778
2779 static void bnxt_free_rx_rings(struct bnxt *bp)
2780 {
2781 int i;
2782
2783 if (!bp->rx_ring)
2784 return;
2785
2786 bnxt_free_tpa_info(bp);
2787 for (i = 0; i < bp->rx_nr_rings; i++) {
2788 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2789 struct bnxt_ring_struct *ring;
2790
2791 if (rxr->xdp_prog)
2792 bpf_prog_put(rxr->xdp_prog);
2793
2794 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2795 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2796
2797 page_pool_destroy(rxr->page_pool);
2798 rxr->page_pool = NULL;
2799
2800 kfree(rxr->rx_agg_bmap);
2801 rxr->rx_agg_bmap = NULL;
2802
2803 ring = &rxr->rx_ring_struct;
2804 bnxt_free_ring(bp, &ring->ring_mem);
2805
2806 ring = &rxr->rx_agg_ring_struct;
2807 bnxt_free_ring(bp, &ring->ring_mem);
2808 }
2809 }
2810
2811 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2812 struct bnxt_rx_ring_info *rxr)
2813 {
2814 struct page_pool_params pp = { 0 };
2815
2816 pp.pool_size = bp->rx_ring_size;
2817 pp.nid = dev_to_node(&bp->pdev->dev);
2818 pp.dev = &bp->pdev->dev;
2819 pp.dma_dir = DMA_BIDIRECTIONAL;
2820
2821 rxr->page_pool = page_pool_create(&pp);
2822 if (IS_ERR(rxr->page_pool)) {
2823 int err = PTR_ERR(rxr->page_pool);
2824
2825 rxr->page_pool = NULL;
2826 return err;
2827 }
2828 return 0;
2829 }
2830
2831 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2832 {
2833 int i, rc = 0, agg_rings = 0;
2834
2835 if (!bp->rx_ring)
2836 return -ENOMEM;
2837
2838 if (bp->flags & BNXT_FLAG_AGG_RINGS)
2839 agg_rings = 1;
2840
2841 for (i = 0; i < bp->rx_nr_rings; i++) {
2842 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2843 struct bnxt_ring_struct *ring;
2844
2845 ring = &rxr->rx_ring_struct;
2846
2847 rc = bnxt_alloc_rx_page_pool(bp, rxr);
2848 if (rc)
2849 return rc;
2850
2851 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2852 if (rc < 0)
2853 return rc;
2854
2855 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2856 MEM_TYPE_PAGE_POOL,
2857 rxr->page_pool);
2858 if (rc) {
2859 xdp_rxq_info_unreg(&rxr->xdp_rxq);
2860 return rc;
2861 }
2862
2863 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2864 if (rc)
2865 return rc;
2866
2867 ring->grp_idx = i;
2868 if (agg_rings) {
2869 u16 mem_size;
2870
2871 ring = &rxr->rx_agg_ring_struct;
2872 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2873 if (rc)
2874 return rc;
2875
2876 ring->grp_idx = i;
2877 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2878 mem_size = rxr->rx_agg_bmap_size / 8;
2879 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2880 if (!rxr->rx_agg_bmap)
2881 return -ENOMEM;
2882 }
2883 }
2884 if (bp->flags & BNXT_FLAG_TPA)
2885 rc = bnxt_alloc_tpa_info(bp);
2886 return rc;
2887 }
2888
2889 static void bnxt_free_tx_rings(struct bnxt *bp)
2890 {
2891 int i;
2892 struct pci_dev *pdev = bp->pdev;
2893
2894 if (!bp->tx_ring)
2895 return;
2896
2897 for (i = 0; i < bp->tx_nr_rings; i++) {
2898 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2899 struct bnxt_ring_struct *ring;
2900
2901 if (txr->tx_push) {
2902 dma_free_coherent(&pdev->dev, bp->tx_push_size,
2903 txr->tx_push, txr->tx_push_mapping);
2904 txr->tx_push = NULL;
2905 }
2906
2907 ring = &txr->tx_ring_struct;
2908
2909 bnxt_free_ring(bp, &ring->ring_mem);
2910 }
2911 }
2912
2913 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2914 {
2915 int i, j, rc;
2916 struct pci_dev *pdev = bp->pdev;
2917
2918 bp->tx_push_size = 0;
2919 if (bp->tx_push_thresh) {
2920 int push_size;
2921
2922 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2923 bp->tx_push_thresh);
2924
2925 if (push_size > 256) {
2926 push_size = 0;
2927 bp->tx_push_thresh = 0;
2928 }
2929
2930 bp->tx_push_size = push_size;
2931 }
2932
2933 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2934 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2935 struct bnxt_ring_struct *ring;
2936 u8 qidx;
2937
2938 ring = &txr->tx_ring_struct;
2939
2940 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2941 if (rc)
2942 return rc;
2943
2944 ring->grp_idx = txr->bnapi->index;
2945 if (bp->tx_push_size) {
2946 dma_addr_t mapping;
2947
2948 /* One pre-allocated DMA buffer to backup
2949 * TX push operation
2950 */
2951 txr->tx_push = dma_alloc_coherent(&pdev->dev,
2952 bp->tx_push_size,
2953 &txr->tx_push_mapping,
2954 GFP_KERNEL);
2955
2956 if (!txr->tx_push)
2957 return -ENOMEM;
2958
2959 mapping = txr->tx_push_mapping +
2960 sizeof(struct tx_push_bd);
2961 txr->data_mapping = cpu_to_le64(mapping);
2962 }
2963 qidx = bp->tc_to_qidx[j];
2964 ring->queue_id = bp->q_info[qidx].queue_id;
2965 if (i < bp->tx_nr_rings_xdp)
2966 continue;
2967 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2968 j++;
2969 }
2970 return 0;
2971 }
2972
2973 static void bnxt_free_cp_rings(struct bnxt *bp)
2974 {
2975 int i;
2976
2977 if (!bp->bnapi)
2978 return;
2979
2980 for (i = 0; i < bp->cp_nr_rings; i++) {
2981 struct bnxt_napi *bnapi = bp->bnapi[i];
2982 struct bnxt_cp_ring_info *cpr;
2983 struct bnxt_ring_struct *ring;
2984 int j;
2985
2986 if (!bnapi)
2987 continue;
2988
2989 cpr = &bnapi->cp_ring;
2990 ring = &cpr->cp_ring_struct;
2991
2992 bnxt_free_ring(bp, &ring->ring_mem);
2993
2994 for (j = 0; j < 2; j++) {
2995 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2996
2997 if (cpr2) {
2998 ring = &cpr2->cp_ring_struct;
2999 bnxt_free_ring(bp, &ring->ring_mem);
3000 kfree(cpr2);
3001 cpr->cp_ring_arr[j] = NULL;
3002 }
3003 }
3004 }
3005 }
3006
3007 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3008 {
3009 struct bnxt_ring_mem_info *rmem;
3010 struct bnxt_ring_struct *ring;
3011 struct bnxt_cp_ring_info *cpr;
3012 int rc;
3013
3014 cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3015 if (!cpr)
3016 return NULL;
3017
3018 ring = &cpr->cp_ring_struct;
3019 rmem = &ring->ring_mem;
3020 rmem->nr_pages = bp->cp_nr_pages;
3021 rmem->page_size = HW_CMPD_RING_SIZE;
3022 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3023 rmem->dma_arr = cpr->cp_desc_mapping;
3024 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3025 rc = bnxt_alloc_ring(bp, rmem);
3026 if (rc) {
3027 bnxt_free_ring(bp, rmem);
3028 kfree(cpr);
3029 cpr = NULL;
3030 }
3031 return cpr;
3032 }
3033
3034 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3035 {
3036 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3037 int i, rc, ulp_base_vec, ulp_msix;
3038
3039 ulp_msix = bnxt_get_ulp_msix_num(bp);
3040 ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3041 for (i = 0; i < bp->cp_nr_rings; i++) {
3042 struct bnxt_napi *bnapi = bp->bnapi[i];
3043 struct bnxt_cp_ring_info *cpr;
3044 struct bnxt_ring_struct *ring;
3045
3046 if (!bnapi)
3047 continue;
3048
3049 cpr = &bnapi->cp_ring;
3050 cpr->bnapi = bnapi;
3051 ring = &cpr->cp_ring_struct;
3052
3053 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3054 if (rc)
3055 return rc;
3056
3057 if (ulp_msix && i >= ulp_base_vec)
3058 ring->map_idx = i + ulp_msix;
3059 else
3060 ring->map_idx = i;
3061
3062 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3063 continue;
3064
3065 if (i < bp->rx_nr_rings) {
3066 struct bnxt_cp_ring_info *cpr2 =
3067 bnxt_alloc_cp_sub_ring(bp);
3068
3069 cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3070 if (!cpr2)
3071 return -ENOMEM;
3072 cpr2->bnapi = bnapi;
3073 }
3074 if ((sh && i < bp->tx_nr_rings) ||
3075 (!sh && i >= bp->rx_nr_rings)) {
3076 struct bnxt_cp_ring_info *cpr2 =
3077 bnxt_alloc_cp_sub_ring(bp);
3078
3079 cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3080 if (!cpr2)
3081 return -ENOMEM;
3082 cpr2->bnapi = bnapi;
3083 }
3084 }
3085 return 0;
3086 }
3087
3088 static void bnxt_init_ring_struct(struct bnxt *bp)
3089 {
3090 int i;
3091
3092 for (i = 0; i < bp->cp_nr_rings; i++) {
3093 struct bnxt_napi *bnapi = bp->bnapi[i];
3094 struct bnxt_ring_mem_info *rmem;
3095 struct bnxt_cp_ring_info *cpr;
3096 struct bnxt_rx_ring_info *rxr;
3097 struct bnxt_tx_ring_info *txr;
3098 struct bnxt_ring_struct *ring;
3099
3100 if (!bnapi)
3101 continue;
3102
3103 cpr = &bnapi->cp_ring;
3104 ring = &cpr->cp_ring_struct;
3105 rmem = &ring->ring_mem;
3106 rmem->nr_pages = bp->cp_nr_pages;
3107 rmem->page_size = HW_CMPD_RING_SIZE;
3108 rmem->pg_arr = (void **)cpr->cp_desc_ring;
3109 rmem->dma_arr = cpr->cp_desc_mapping;
3110 rmem->vmem_size = 0;
3111
3112 rxr = bnapi->rx_ring;
3113 if (!rxr)
3114 goto skip_rx;
3115
3116 ring = &rxr->rx_ring_struct;
3117 rmem = &ring->ring_mem;
3118 rmem->nr_pages = bp->rx_nr_pages;
3119 rmem->page_size = HW_RXBD_RING_SIZE;
3120 rmem->pg_arr = (void **)rxr->rx_desc_ring;
3121 rmem->dma_arr = rxr->rx_desc_mapping;
3122 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3123 rmem->vmem = (void **)&rxr->rx_buf_ring;
3124
3125 ring = &rxr->rx_agg_ring_struct;
3126 rmem = &ring->ring_mem;
3127 rmem->nr_pages = bp->rx_agg_nr_pages;
3128 rmem->page_size = HW_RXBD_RING_SIZE;
3129 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3130 rmem->dma_arr = rxr->rx_agg_desc_mapping;
3131 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3132 rmem->vmem = (void **)&rxr->rx_agg_ring;
3133
3134 skip_rx:
3135 txr = bnapi->tx_ring;
3136 if (!txr)
3137 continue;
3138
3139 ring = &txr->tx_ring_struct;
3140 rmem = &ring->ring_mem;
3141 rmem->nr_pages = bp->tx_nr_pages;
3142 rmem->page_size = HW_RXBD_RING_SIZE;
3143 rmem->pg_arr = (void **)txr->tx_desc_ring;
3144 rmem->dma_arr = txr->tx_desc_mapping;
3145 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3146 rmem->vmem = (void **)&txr->tx_buf_ring;
3147 }
3148 }
3149
3150 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3151 {
3152 int i;
3153 u32 prod;
3154 struct rx_bd **rx_buf_ring;
3155
3156 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3157 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3158 int j;
3159 struct rx_bd *rxbd;
3160
3161 rxbd = rx_buf_ring[i];
3162 if (!rxbd)
3163 continue;
3164
3165 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3166 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3167 rxbd->rx_bd_opaque = prod;
3168 }
3169 }
3170 }
3171
3172 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3173 {
3174 struct net_device *dev = bp->dev;
3175 struct bnxt_rx_ring_info *rxr;
3176 struct bnxt_ring_struct *ring;
3177 u32 prod, type;
3178 int i;
3179
3180 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3181 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3182
3183 if (NET_IP_ALIGN == 2)
3184 type |= RX_BD_FLAGS_SOP;
3185
3186 rxr = &bp->rx_ring[ring_nr];
3187 ring = &rxr->rx_ring_struct;
3188 bnxt_init_rxbd_pages(ring, type);
3189
3190 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3191 bpf_prog_add(bp->xdp_prog, 1);
3192 rxr->xdp_prog = bp->xdp_prog;
3193 }
3194 prod = rxr->rx_prod;
3195 for (i = 0; i < bp->rx_ring_size; i++) {
3196 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3197 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3198 ring_nr, i, bp->rx_ring_size);
3199 break;
3200 }
3201 prod = NEXT_RX(prod);
3202 }
3203 rxr->rx_prod = prod;
3204 ring->fw_ring_id = INVALID_HW_RING_ID;
3205
3206 ring = &rxr->rx_agg_ring_struct;
3207 ring->fw_ring_id = INVALID_HW_RING_ID;
3208
3209 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3210 return 0;
3211
3212 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3213 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3214
3215 bnxt_init_rxbd_pages(ring, type);
3216
3217 prod = rxr->rx_agg_prod;
3218 for (i = 0; i < bp->rx_agg_ring_size; i++) {
3219 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3220 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3221 ring_nr, i, bp->rx_ring_size);
3222 break;
3223 }
3224 prod = NEXT_RX_AGG(prod);
3225 }
3226 rxr->rx_agg_prod = prod;
3227
3228 if (bp->flags & BNXT_FLAG_TPA) {
3229 if (rxr->rx_tpa) {
3230 u8 *data;
3231 dma_addr_t mapping;
3232
3233 for (i = 0; i < bp->max_tpa; i++) {
3234 data = __bnxt_alloc_rx_data(bp, &mapping,
3235 GFP_KERNEL);
3236 if (!data)
3237 return -ENOMEM;
3238
3239 rxr->rx_tpa[i].data = data;
3240 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3241 rxr->rx_tpa[i].mapping = mapping;
3242 }
3243 } else {
3244 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3245 return -ENOMEM;
3246 }
3247 }
3248
3249 return 0;
3250 }
3251
3252 static void bnxt_init_cp_rings(struct bnxt *bp)
3253 {
3254 int i, j;
3255
3256 for (i = 0; i < bp->cp_nr_rings; i++) {
3257 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3258 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3259
3260 ring->fw_ring_id = INVALID_HW_RING_ID;
3261 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3262 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3263 for (j = 0; j < 2; j++) {
3264 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3265
3266 if (!cpr2)
3267 continue;
3268
3269 ring = &cpr2->cp_ring_struct;
3270 ring->fw_ring_id = INVALID_HW_RING_ID;
3271 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3272 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3273 }
3274 }
3275 }
3276
3277 static int bnxt_init_rx_rings(struct bnxt *bp)
3278 {
3279 int i, rc = 0;
3280
3281 if (BNXT_RX_PAGE_MODE(bp)) {
3282 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3283 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3284 } else {
3285 bp->rx_offset = BNXT_RX_OFFSET;
3286 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3287 }
3288
3289 for (i = 0; i < bp->rx_nr_rings; i++) {
3290 rc = bnxt_init_one_rx_ring(bp, i);
3291 if (rc)
3292 break;
3293 }
3294
3295 return rc;
3296 }
3297
3298 static int bnxt_init_tx_rings(struct bnxt *bp)
3299 {
3300 u16 i;
3301
3302 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3303 MAX_SKB_FRAGS + 1);
3304
3305 for (i = 0; i < bp->tx_nr_rings; i++) {
3306 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3307 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3308
3309 ring->fw_ring_id = INVALID_HW_RING_ID;
3310 }
3311
3312 return 0;
3313 }
3314
3315 static void bnxt_free_ring_grps(struct bnxt *bp)
3316 {
3317 kfree(bp->grp_info);
3318 bp->grp_info = NULL;
3319 }
3320
3321 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3322 {
3323 int i;
3324
3325 if (irq_re_init) {
3326 bp->grp_info = kcalloc(bp->cp_nr_rings,
3327 sizeof(struct bnxt_ring_grp_info),
3328 GFP_KERNEL);
3329 if (!bp->grp_info)
3330 return -ENOMEM;
3331 }
3332 for (i = 0; i < bp->cp_nr_rings; i++) {
3333 if (irq_re_init)
3334 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3335 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3336 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3337 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3338 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3339 }
3340 return 0;
3341 }
3342
3343 static void bnxt_free_vnics(struct bnxt *bp)
3344 {
3345 kfree(bp->vnic_info);
3346 bp->vnic_info = NULL;
3347 bp->nr_vnics = 0;
3348 }
3349
3350 static int bnxt_alloc_vnics(struct bnxt *bp)
3351 {
3352 int num_vnics = 1;
3353
3354 #ifdef CONFIG_RFS_ACCEL
3355 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3356 num_vnics += bp->rx_nr_rings;
3357 #endif
3358
3359 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3360 num_vnics++;
3361
3362 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3363 GFP_KERNEL);
3364 if (!bp->vnic_info)
3365 return -ENOMEM;
3366
3367 bp->nr_vnics = num_vnics;
3368 return 0;
3369 }
3370
3371 static void bnxt_init_vnics(struct bnxt *bp)
3372 {
3373 int i;
3374
3375 for (i = 0; i < bp->nr_vnics; i++) {
3376 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3377 int j;
3378
3379 vnic->fw_vnic_id = INVALID_HW_RING_ID;
3380 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3381 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3382
3383 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3384
3385 if (bp->vnic_info[i].rss_hash_key) {
3386 if (i == 0)
3387 prandom_bytes(vnic->rss_hash_key,
3388 HW_HASH_KEY_SIZE);
3389 else
3390 memcpy(vnic->rss_hash_key,
3391 bp->vnic_info[0].rss_hash_key,
3392 HW_HASH_KEY_SIZE);
3393 }
3394 }
3395 }
3396
3397 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3398 {
3399 int pages;
3400
3401 pages = ring_size / desc_per_pg;
3402
3403 if (!pages)
3404 return 1;
3405
3406 pages++;
3407
3408 while (pages & (pages - 1))
3409 pages++;
3410
3411 return pages;
3412 }
3413
3414 void bnxt_set_tpa_flags(struct bnxt *bp)
3415 {
3416 bp->flags &= ~BNXT_FLAG_TPA;
3417 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3418 return;
3419 if (bp->dev->features & NETIF_F_LRO)
3420 bp->flags |= BNXT_FLAG_LRO;
3421 else if (bp->dev->features & NETIF_F_GRO_HW)
3422 bp->flags |= BNXT_FLAG_GRO;
3423 }
3424
3425 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3426 * be set on entry.
3427 */
3428 void bnxt_set_ring_params(struct bnxt *bp)
3429 {
3430 u32 ring_size, rx_size, rx_space;
3431 u32 agg_factor = 0, agg_ring_size = 0;
3432
3433 /* 8 for CRC and VLAN */
3434 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3435
3436 rx_space = rx_size + NET_SKB_PAD +
3437 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3438
3439 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3440 ring_size = bp->rx_ring_size;
3441 bp->rx_agg_ring_size = 0;
3442 bp->rx_agg_nr_pages = 0;
3443
3444 if (bp->flags & BNXT_FLAG_TPA)
3445 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3446
3447 bp->flags &= ~BNXT_FLAG_JUMBO;
3448 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3449 u32 jumbo_factor;
3450
3451 bp->flags |= BNXT_FLAG_JUMBO;
3452 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3453 if (jumbo_factor > agg_factor)
3454 agg_factor = jumbo_factor;
3455 }
3456 agg_ring_size = ring_size * agg_factor;
3457
3458 if (agg_ring_size) {
3459 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3460 RX_DESC_CNT);
3461 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3462 u32 tmp = agg_ring_size;
3463
3464 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3465 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3466 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3467 tmp, agg_ring_size);
3468 }
3469 bp->rx_agg_ring_size = agg_ring_size;
3470 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3471 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3472 rx_space = rx_size + NET_SKB_PAD +
3473 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3474 }
3475
3476 bp->rx_buf_use_size = rx_size;
3477 bp->rx_buf_size = rx_space;
3478
3479 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3480 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3481
3482 ring_size = bp->tx_ring_size;
3483 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3484 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3485
3486 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
3487 bp->cp_ring_size = ring_size;
3488
3489 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3490 if (bp->cp_nr_pages > MAX_CP_PAGES) {
3491 bp->cp_nr_pages = MAX_CP_PAGES;
3492 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3493 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3494 ring_size, bp->cp_ring_size);
3495 }
3496 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3497 bp->cp_ring_mask = bp->cp_bit - 1;
3498 }
3499
3500 /* Changing allocation mode of RX rings.
3501 * TODO: Update when extending xdp_rxq_info to support allocation modes.
3502 */
3503 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3504 {
3505 if (page_mode) {
3506 if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3507 return -EOPNOTSUPP;
3508 bp->dev->max_mtu =
3509 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3510 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3511 bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3512 bp->rx_dir = DMA_BIDIRECTIONAL;
3513 bp->rx_skb_func = bnxt_rx_page_skb;
3514 /* Disable LRO or GRO_HW */
3515 netdev_update_features(bp->dev);
3516 } else {
3517 bp->dev->max_mtu = bp->max_mtu;
3518 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3519 bp->rx_dir = DMA_FROM_DEVICE;
3520 bp->rx_skb_func = bnxt_rx_skb;
3521 }
3522 return 0;
3523 }
3524
3525 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3526 {
3527 int i;
3528 struct bnxt_vnic_info *vnic;
3529 struct pci_dev *pdev = bp->pdev;
3530
3531 if (!bp->vnic_info)
3532 return;
3533
3534 for (i = 0; i < bp->nr_vnics; i++) {
3535 vnic = &bp->vnic_info[i];
3536
3537 kfree(vnic->fw_grp_ids);
3538 vnic->fw_grp_ids = NULL;
3539
3540 kfree(vnic->uc_list);
3541 vnic->uc_list = NULL;
3542
3543 if (vnic->mc_list) {
3544 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3545 vnic->mc_list, vnic->mc_list_mapping);
3546 vnic->mc_list = NULL;
3547 }
3548
3549 if (vnic->rss_table) {
3550 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3551 vnic->rss_table,
3552 vnic->rss_table_dma_addr);
3553 vnic->rss_table = NULL;
3554 }
3555
3556 vnic->rss_hash_key = NULL;
3557 vnic->flags = 0;
3558 }
3559 }
3560
3561 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3562 {
3563 int i, rc = 0, size;
3564 struct bnxt_vnic_info *vnic;
3565 struct pci_dev *pdev = bp->pdev;
3566 int max_rings;
3567
3568 for (i = 0; i < bp->nr_vnics; i++) {
3569 vnic = &bp->vnic_info[i];
3570
3571 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3572 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3573
3574 if (mem_size > 0) {
3575 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3576 if (!vnic->uc_list) {
3577 rc = -ENOMEM;
3578 goto out;
3579 }
3580 }
3581 }
3582
3583 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3584 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3585 vnic->mc_list =
3586 dma_alloc_coherent(&pdev->dev,
3587 vnic->mc_list_size,
3588 &vnic->mc_list_mapping,
3589 GFP_KERNEL);
3590 if (!vnic->mc_list) {
3591 rc = -ENOMEM;
3592 goto out;
3593 }
3594 }
3595
3596 if (bp->flags & BNXT_FLAG_CHIP_P5)
3597 goto vnic_skip_grps;
3598
3599 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3600 max_rings = bp->rx_nr_rings;
3601 else
3602 max_rings = 1;
3603
3604 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3605 if (!vnic->fw_grp_ids) {
3606 rc = -ENOMEM;
3607 goto out;
3608 }
3609 vnic_skip_grps:
3610 if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3611 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3612 continue;
3613
3614 /* Allocate rss table and hash key */
3615 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3616 &vnic->rss_table_dma_addr,
3617 GFP_KERNEL);
3618 if (!vnic->rss_table) {
3619 rc = -ENOMEM;
3620 goto out;
3621 }
3622
3623 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3624
3625 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3626 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3627 }
3628 return 0;
3629
3630 out:
3631 return rc;
3632 }
3633
3634 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3635 {
3636 struct pci_dev *pdev = bp->pdev;
3637
3638 if (bp->hwrm_cmd_resp_addr) {
3639 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3640 bp->hwrm_cmd_resp_dma_addr);
3641 bp->hwrm_cmd_resp_addr = NULL;
3642 }
3643
3644 if (bp->hwrm_cmd_kong_resp_addr) {
3645 dma_free_coherent(&pdev->dev, PAGE_SIZE,
3646 bp->hwrm_cmd_kong_resp_addr,
3647 bp->hwrm_cmd_kong_resp_dma_addr);
3648 bp->hwrm_cmd_kong_resp_addr = NULL;
3649 }
3650 }
3651
3652 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3653 {
3654 struct pci_dev *pdev = bp->pdev;
3655
3656 if (bp->hwrm_cmd_kong_resp_addr)
3657 return 0;
3658
3659 bp->hwrm_cmd_kong_resp_addr =
3660 dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3661 &bp->hwrm_cmd_kong_resp_dma_addr,
3662 GFP_KERNEL);
3663 if (!bp->hwrm_cmd_kong_resp_addr)
3664 return -ENOMEM;
3665
3666 return 0;
3667 }
3668
3669 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3670 {
3671 struct pci_dev *pdev = bp->pdev;
3672
3673 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3674 &bp->hwrm_cmd_resp_dma_addr,
3675 GFP_KERNEL);
3676 if (!bp->hwrm_cmd_resp_addr)
3677 return -ENOMEM;
3678
3679 return 0;
3680 }
3681
3682 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3683 {
3684 if (bp->hwrm_short_cmd_req_addr) {
3685 struct pci_dev *pdev = bp->pdev;
3686
3687 dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3688 bp->hwrm_short_cmd_req_addr,
3689 bp->hwrm_short_cmd_req_dma_addr);
3690 bp->hwrm_short_cmd_req_addr = NULL;
3691 }
3692 }
3693
3694 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3695 {
3696 struct pci_dev *pdev = bp->pdev;
3697
3698 if (bp->hwrm_short_cmd_req_addr)
3699 return 0;
3700
3701 bp->hwrm_short_cmd_req_addr =
3702 dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3703 &bp->hwrm_short_cmd_req_dma_addr,
3704 GFP_KERNEL);
3705 if (!bp->hwrm_short_cmd_req_addr)
3706 return -ENOMEM;
3707
3708 return 0;
3709 }
3710
3711 static void bnxt_free_port_stats(struct bnxt *bp)
3712 {
3713 struct pci_dev *pdev = bp->pdev;
3714
3715 bp->flags &= ~BNXT_FLAG_PORT_STATS;
3716 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3717
3718 if (bp->hw_rx_port_stats) {
3719 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
3720 bp->hw_rx_port_stats,
3721 bp->hw_rx_port_stats_map);
3722 bp->hw_rx_port_stats = NULL;
3723 }
3724
3725 if (bp->hw_tx_port_stats_ext) {
3726 dma_free_coherent(&pdev->dev, sizeof(struct tx_port_stats_ext),
3727 bp->hw_tx_port_stats_ext,
3728 bp->hw_tx_port_stats_ext_map);
3729 bp->hw_tx_port_stats_ext = NULL;
3730 }
3731
3732 if (bp->hw_rx_port_stats_ext) {
3733 dma_free_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3734 bp->hw_rx_port_stats_ext,
3735 bp->hw_rx_port_stats_ext_map);
3736 bp->hw_rx_port_stats_ext = NULL;
3737 }
3738
3739 if (bp->hw_pcie_stats) {
3740 dma_free_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3741 bp->hw_pcie_stats, bp->hw_pcie_stats_map);
3742 bp->hw_pcie_stats = NULL;
3743 }
3744 }
3745
3746 static void bnxt_free_ring_stats(struct bnxt *bp)
3747 {
3748 struct pci_dev *pdev = bp->pdev;
3749 int size, i;
3750
3751 if (!bp->bnapi)
3752 return;
3753
3754 size = bp->hw_ring_stats_size;
3755
3756 for (i = 0; i < bp->cp_nr_rings; i++) {
3757 struct bnxt_napi *bnapi = bp->bnapi[i];
3758 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3759
3760 if (cpr->hw_stats) {
3761 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
3762 cpr->hw_stats_map);
3763 cpr->hw_stats = NULL;
3764 }
3765 }
3766 }
3767
3768 static int bnxt_alloc_stats(struct bnxt *bp)
3769 {
3770 u32 size, i;
3771 struct pci_dev *pdev = bp->pdev;
3772
3773 size = bp->hw_ring_stats_size;
3774
3775 for (i = 0; i < bp->cp_nr_rings; i++) {
3776 struct bnxt_napi *bnapi = bp->bnapi[i];
3777 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3778
3779 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
3780 &cpr->hw_stats_map,
3781 GFP_KERNEL);
3782 if (!cpr->hw_stats)
3783 return -ENOMEM;
3784
3785 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3786 }
3787
3788 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3789 return 0;
3790
3791 if (bp->hw_rx_port_stats)
3792 goto alloc_ext_stats;
3793
3794 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
3795 sizeof(struct tx_port_stats) + 1024;
3796
3797 bp->hw_rx_port_stats =
3798 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
3799 &bp->hw_rx_port_stats_map,
3800 GFP_KERNEL);
3801 if (!bp->hw_rx_port_stats)
3802 return -ENOMEM;
3803
3804 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) + 512;
3805 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
3806 sizeof(struct rx_port_stats) + 512;
3807 bp->flags |= BNXT_FLAG_PORT_STATS;
3808
3809 alloc_ext_stats:
3810 /* Display extended statistics only if FW supports it */
3811 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3812 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3813 return 0;
3814
3815 if (bp->hw_rx_port_stats_ext)
3816 goto alloc_tx_ext_stats;
3817
3818 bp->hw_rx_port_stats_ext =
3819 dma_alloc_coherent(&pdev->dev, sizeof(struct rx_port_stats_ext),
3820 &bp->hw_rx_port_stats_ext_map, GFP_KERNEL);
3821 if (!bp->hw_rx_port_stats_ext)
3822 return 0;
3823
3824 alloc_tx_ext_stats:
3825 if (bp->hw_tx_port_stats_ext)
3826 goto alloc_pcie_stats;
3827
3828 if (bp->hwrm_spec_code >= 0x10902 ||
3829 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3830 bp->hw_tx_port_stats_ext =
3831 dma_alloc_coherent(&pdev->dev,
3832 sizeof(struct tx_port_stats_ext),
3833 &bp->hw_tx_port_stats_ext_map,
3834 GFP_KERNEL);
3835 }
3836 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3837
3838 alloc_pcie_stats:
3839 if (bp->hw_pcie_stats ||
3840 !(bp->fw_cap & BNXT_FW_CAP_PCIE_STATS_SUPPORTED))
3841 return 0;
3842
3843 bp->hw_pcie_stats =
3844 dma_alloc_coherent(&pdev->dev, sizeof(struct pcie_ctx_hw_stats),
3845 &bp->hw_pcie_stats_map, GFP_KERNEL);
3846 if (!bp->hw_pcie_stats)
3847 return 0;
3848
3849 bp->flags |= BNXT_FLAG_PCIE_STATS;
3850 return 0;
3851 }
3852
3853 static void bnxt_clear_ring_indices(struct bnxt *bp)
3854 {
3855 int i;
3856
3857 if (!bp->bnapi)
3858 return;
3859
3860 for (i = 0; i < bp->cp_nr_rings; i++) {
3861 struct bnxt_napi *bnapi = bp->bnapi[i];
3862 struct bnxt_cp_ring_info *cpr;
3863 struct bnxt_rx_ring_info *rxr;
3864 struct bnxt_tx_ring_info *txr;
3865
3866 if (!bnapi)
3867 continue;
3868
3869 cpr = &bnapi->cp_ring;
3870 cpr->cp_raw_cons = 0;
3871
3872 txr = bnapi->tx_ring;
3873 if (txr) {
3874 txr->tx_prod = 0;
3875 txr->tx_cons = 0;
3876 }
3877
3878 rxr = bnapi->rx_ring;
3879 if (rxr) {
3880 rxr->rx_prod = 0;
3881 rxr->rx_agg_prod = 0;
3882 rxr->rx_sw_agg_prod = 0;
3883 rxr->rx_next_cons = 0;
3884 }
3885 }
3886 }
3887
3888 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3889 {
3890 #ifdef CONFIG_RFS_ACCEL
3891 int i;
3892
3893 /* Under rtnl_lock and all our NAPIs have been disabled. It's
3894 * safe to delete the hash table.
3895 */
3896 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
3897 struct hlist_head *head;
3898 struct hlist_node *tmp;
3899 struct bnxt_ntuple_filter *fltr;
3900
3901 head = &bp->ntp_fltr_hash_tbl[i];
3902 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
3903 hlist_del(&fltr->hash);
3904 kfree(fltr);
3905 }
3906 }
3907 if (irq_reinit) {
3908 kfree(bp->ntp_fltr_bmap);
3909 bp->ntp_fltr_bmap = NULL;
3910 }
3911 bp->ntp_fltr_count = 0;
3912 #endif
3913 }
3914
3915 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
3916 {
3917 #ifdef CONFIG_RFS_ACCEL
3918 int i, rc = 0;
3919
3920 if (!(bp->flags & BNXT_FLAG_RFS))
3921 return 0;
3922
3923 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
3924 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
3925
3926 bp->ntp_fltr_count = 0;
3927 bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
3928 sizeof(long),
3929 GFP_KERNEL);
3930
3931 if (!bp->ntp_fltr_bmap)
3932 rc = -ENOMEM;
3933
3934 return rc;
3935 #else
3936 return 0;
3937 #endif
3938 }
3939
3940 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
3941 {
3942 bnxt_free_vnic_attributes(bp);
3943 bnxt_free_tx_rings(bp);
3944 bnxt_free_rx_rings(bp);
3945 bnxt_free_cp_rings(bp);
3946 bnxt_free_ntp_fltrs(bp, irq_re_init);
3947 if (irq_re_init) {
3948 bnxt_free_ring_stats(bp);
3949 bnxt_free_ring_grps(bp);
3950 bnxt_free_vnics(bp);
3951 kfree(bp->tx_ring_map);
3952 bp->tx_ring_map = NULL;
3953 kfree(bp->tx_ring);
3954 bp->tx_ring = NULL;
3955 kfree(bp->rx_ring);
3956 bp->rx_ring = NULL;
3957 kfree(bp->bnapi);
3958 bp->bnapi = NULL;
3959 } else {
3960 bnxt_clear_ring_indices(bp);
3961 }
3962 }
3963
3964 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
3965 {
3966 int i, j, rc, size, arr_size;
3967 void *bnapi;
3968
3969 if (irq_re_init) {
3970 /* Allocate bnapi mem pointer array and mem block for
3971 * all queues
3972 */
3973 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
3974 bp->cp_nr_rings);
3975 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
3976 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
3977 if (!bnapi)
3978 return -ENOMEM;
3979
3980 bp->bnapi = bnapi;
3981 bnapi += arr_size;
3982 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
3983 bp->bnapi[i] = bnapi;
3984 bp->bnapi[i]->index = i;
3985 bp->bnapi[i]->bp = bp;
3986 if (bp->flags & BNXT_FLAG_CHIP_P5) {
3987 struct bnxt_cp_ring_info *cpr =
3988 &bp->bnapi[i]->cp_ring;
3989
3990 cpr->cp_ring_struct.ring_mem.flags =
3991 BNXT_RMEM_RING_PTE_FLAG;
3992 }
3993 }
3994
3995 bp->rx_ring = kcalloc(bp->rx_nr_rings,
3996 sizeof(struct bnxt_rx_ring_info),
3997 GFP_KERNEL);
3998 if (!bp->rx_ring)
3999 return -ENOMEM;
4000
4001 for (i = 0; i < bp->rx_nr_rings; i++) {
4002 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4003
4004 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4005 rxr->rx_ring_struct.ring_mem.flags =
4006 BNXT_RMEM_RING_PTE_FLAG;
4007 rxr->rx_agg_ring_struct.ring_mem.flags =
4008 BNXT_RMEM_RING_PTE_FLAG;
4009 }
4010 rxr->bnapi = bp->bnapi[i];
4011 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4012 }
4013
4014 bp->tx_ring = kcalloc(bp->tx_nr_rings,
4015 sizeof(struct bnxt_tx_ring_info),
4016 GFP_KERNEL);
4017 if (!bp->tx_ring)
4018 return -ENOMEM;
4019
4020 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4021 GFP_KERNEL);
4022
4023 if (!bp->tx_ring_map)
4024 return -ENOMEM;
4025
4026 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4027 j = 0;
4028 else
4029 j = bp->rx_nr_rings;
4030
4031 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4032 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4033
4034 if (bp->flags & BNXT_FLAG_CHIP_P5)
4035 txr->tx_ring_struct.ring_mem.flags =
4036 BNXT_RMEM_RING_PTE_FLAG;
4037 txr->bnapi = bp->bnapi[j];
4038 bp->bnapi[j]->tx_ring = txr;
4039 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4040 if (i >= bp->tx_nr_rings_xdp) {
4041 txr->txq_index = i - bp->tx_nr_rings_xdp;
4042 bp->bnapi[j]->tx_int = bnxt_tx_int;
4043 } else {
4044 bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4045 bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4046 }
4047 }
4048
4049 rc = bnxt_alloc_stats(bp);
4050 if (rc)
4051 goto alloc_mem_err;
4052
4053 rc = bnxt_alloc_ntp_fltrs(bp);
4054 if (rc)
4055 goto alloc_mem_err;
4056
4057 rc = bnxt_alloc_vnics(bp);
4058 if (rc)
4059 goto alloc_mem_err;
4060 }
4061
4062 bnxt_init_ring_struct(bp);
4063
4064 rc = bnxt_alloc_rx_rings(bp);
4065 if (rc)
4066 goto alloc_mem_err;
4067
4068 rc = bnxt_alloc_tx_rings(bp);
4069 if (rc)
4070 goto alloc_mem_err;
4071
4072 rc = bnxt_alloc_cp_rings(bp);
4073 if (rc)
4074 goto alloc_mem_err;
4075
4076 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4077 BNXT_VNIC_UCAST_FLAG;
4078 rc = bnxt_alloc_vnic_attributes(bp);
4079 if (rc)
4080 goto alloc_mem_err;
4081 return 0;
4082
4083 alloc_mem_err:
4084 bnxt_free_mem(bp, true);
4085 return rc;
4086 }
4087
4088 static void bnxt_disable_int(struct bnxt *bp)
4089 {
4090 int i;
4091
4092 if (!bp->bnapi)
4093 return;
4094
4095 for (i = 0; i < bp->cp_nr_rings; i++) {
4096 struct bnxt_napi *bnapi = bp->bnapi[i];
4097 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4098 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4099
4100 if (ring->fw_ring_id != INVALID_HW_RING_ID)
4101 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4102 }
4103 }
4104
4105 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4106 {
4107 struct bnxt_napi *bnapi = bp->bnapi[n];
4108 struct bnxt_cp_ring_info *cpr;
4109
4110 cpr = &bnapi->cp_ring;
4111 return cpr->cp_ring_struct.map_idx;
4112 }
4113
4114 static void bnxt_disable_int_sync(struct bnxt *bp)
4115 {
4116 int i;
4117
4118 atomic_inc(&bp->intr_sem);
4119
4120 bnxt_disable_int(bp);
4121 for (i = 0; i < bp->cp_nr_rings; i++) {
4122 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4123
4124 synchronize_irq(bp->irq_tbl[map_idx].vector);
4125 }
4126 }
4127
4128 static void bnxt_enable_int(struct bnxt *bp)
4129 {
4130 int i;
4131
4132 atomic_set(&bp->intr_sem, 0);
4133 for (i = 0; i < bp->cp_nr_rings; i++) {
4134 struct bnxt_napi *bnapi = bp->bnapi[i];
4135 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4136
4137 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4138 }
4139 }
4140
4141 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4142 u16 cmpl_ring, u16 target_id)
4143 {
4144 struct input *req = request;
4145
4146 req->req_type = cpu_to_le16(req_type);
4147 req->cmpl_ring = cpu_to_le16(cmpl_ring);
4148 req->target_id = cpu_to_le16(target_id);
4149 if (bnxt_kong_hwrm_message(bp, req))
4150 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4151 else
4152 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4153 }
4154
4155 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4156 {
4157 switch (hwrm_err) {
4158 case HWRM_ERR_CODE_SUCCESS:
4159 return 0;
4160 case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4161 return -EACCES;
4162 case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4163 return -ENOSPC;
4164 case HWRM_ERR_CODE_INVALID_PARAMS:
4165 case HWRM_ERR_CODE_INVALID_FLAGS:
4166 case HWRM_ERR_CODE_INVALID_ENABLES:
4167 case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4168 case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4169 return -EINVAL;
4170 case HWRM_ERR_CODE_NO_BUFFER:
4171 return -ENOMEM;
4172 case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4173 return -EAGAIN;
4174 case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4175 return -EOPNOTSUPP;
4176 default:
4177 return -EIO;
4178 }
4179 }
4180
4181 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4182 int timeout, bool silent)
4183 {
4184 int i, intr_process, rc, tmo_count;
4185 struct input *req = msg;
4186 u32 *data = msg;
4187 __le32 *resp_len;
4188 u8 *valid;
4189 u16 cp_ring_id, len = 0;
4190 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4191 u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4192 struct hwrm_short_input short_input = {0};
4193 u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4194 u8 *resp_addr = (u8 *)bp->hwrm_cmd_resp_addr;
4195 u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4196 u16 dst = BNXT_HWRM_CHNL_CHIMP;
4197
4198 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4199 return -EBUSY;
4200
4201 if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4202 if (msg_len > bp->hwrm_max_ext_req_len ||
4203 !bp->hwrm_short_cmd_req_addr)
4204 return -EINVAL;
4205 }
4206
4207 if (bnxt_hwrm_kong_chnl(bp, req)) {
4208 dst = BNXT_HWRM_CHNL_KONG;
4209 bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4210 doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4211 resp = bp->hwrm_cmd_kong_resp_addr;
4212 resp_addr = (u8 *)bp->hwrm_cmd_kong_resp_addr;
4213 }
4214
4215 memset(resp, 0, PAGE_SIZE);
4216 cp_ring_id = le16_to_cpu(req->cmpl_ring);
4217 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4218
4219 req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4220 /* currently supports only one outstanding message */
4221 if (intr_process)
4222 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4223
4224 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4225 msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4226 void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4227 u16 max_msg_len;
4228
4229 /* Set boundary for maximum extended request length for short
4230 * cmd format. If passed up from device use the max supported
4231 * internal req length.
4232 */
4233 max_msg_len = bp->hwrm_max_ext_req_len;
4234
4235 memcpy(short_cmd_req, req, msg_len);
4236 if (msg_len < max_msg_len)
4237 memset(short_cmd_req + msg_len, 0,
4238 max_msg_len - msg_len);
4239
4240 short_input.req_type = req->req_type;
4241 short_input.signature =
4242 cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4243 short_input.size = cpu_to_le16(msg_len);
4244 short_input.req_addr =
4245 cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4246
4247 data = (u32 *)&short_input;
4248 msg_len = sizeof(short_input);
4249
4250 /* Sync memory write before updating doorbell */
4251 wmb();
4252
4253 max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4254 }
4255
4256 /* Write request msg to hwrm channel */
4257 __iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4258
4259 for (i = msg_len; i < max_req_len; i += 4)
4260 writel(0, bp->bar0 + bar_offset + i);
4261
4262 /* Ring channel doorbell */
4263 writel(1, bp->bar0 + doorbell_offset);
4264
4265 if (!pci_is_enabled(bp->pdev))
4266 return 0;
4267
4268 if (!timeout)
4269 timeout = DFLT_HWRM_CMD_TIMEOUT;
4270 /* convert timeout to usec */
4271 timeout *= 1000;
4272
4273 i = 0;
4274 /* Short timeout for the first few iterations:
4275 * number of loops = number of loops for short timeout +
4276 * number of loops for standard timeout.
4277 */
4278 tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4279 timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4280 tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4281 resp_len = (__le32 *)(resp_addr + HWRM_RESP_LEN_OFFSET);
4282
4283 if (intr_process) {
4284 u16 seq_id = bp->hwrm_intr_seq_id;
4285
4286 /* Wait until hwrm response cmpl interrupt is processed */
4287 while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4288 i++ < tmo_count) {
4289 /* Abort the wait for completion if the FW health
4290 * check has failed.
4291 */
4292 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4293 return -EBUSY;
4294 /* on first few passes, just barely sleep */
4295 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4296 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4297 HWRM_SHORT_MAX_TIMEOUT);
4298 else
4299 usleep_range(HWRM_MIN_TIMEOUT,
4300 HWRM_MAX_TIMEOUT);
4301 }
4302
4303 if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4304 if (!silent)
4305 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4306 le16_to_cpu(req->req_type));
4307 return -EBUSY;
4308 }
4309 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4310 HWRM_RESP_LEN_SFT;
4311 valid = resp_addr + len - 1;
4312 } else {
4313 int j;
4314
4315 /* Check if response len is updated */
4316 for (i = 0; i < tmo_count; i++) {
4317 /* Abort the wait for completion if the FW health
4318 * check has failed.
4319 */
4320 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4321 return -EBUSY;
4322 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
4323 HWRM_RESP_LEN_SFT;
4324 if (len)
4325 break;
4326 /* on first few passes, just barely sleep */
4327 if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4328 usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4329 HWRM_SHORT_MAX_TIMEOUT);
4330 else
4331 usleep_range(HWRM_MIN_TIMEOUT,
4332 HWRM_MAX_TIMEOUT);
4333 }
4334
4335 if (i >= tmo_count) {
4336 if (!silent)
4337 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4338 HWRM_TOTAL_TIMEOUT(i),
4339 le16_to_cpu(req->req_type),
4340 le16_to_cpu(req->seq_id), len);
4341 return -EBUSY;
4342 }
4343
4344 /* Last byte of resp contains valid bit */
4345 valid = resp_addr + len - 1;
4346 for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4347 /* make sure we read from updated DMA memory */
4348 dma_rmb();
4349 if (*valid)
4350 break;
4351 usleep_range(1, 5);
4352 }
4353
4354 if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4355 if (!silent)
4356 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4357 HWRM_TOTAL_TIMEOUT(i),
4358 le16_to_cpu(req->req_type),
4359 le16_to_cpu(req->seq_id), len,
4360 *valid);
4361 return -EBUSY;
4362 }
4363 }
4364
4365 /* Zero valid bit for compatibility. Valid bit in an older spec
4366 * may become a new field in a newer spec. We must make sure that
4367 * a new field not implemented by old spec will read zero.
4368 */
4369 *valid = 0;
4370 rc = le16_to_cpu(resp->error_code);
4371 if (rc && !silent)
4372 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4373 le16_to_cpu(resp->req_type),
4374 le16_to_cpu(resp->seq_id), rc);
4375 return bnxt_hwrm_to_stderr(rc);
4376 }
4377
4378 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4379 {
4380 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4381 }
4382
4383 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4384 int timeout)
4385 {
4386 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4387 }
4388
4389 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4390 {
4391 int rc;
4392
4393 mutex_lock(&bp->hwrm_cmd_lock);
4394 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4395 mutex_unlock(&bp->hwrm_cmd_lock);
4396 return rc;
4397 }
4398
4399 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4400 int timeout)
4401 {
4402 int rc;
4403
4404 mutex_lock(&bp->hwrm_cmd_lock);
4405 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4406 mutex_unlock(&bp->hwrm_cmd_lock);
4407 return rc;
4408 }
4409
4410 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4411 bool async_only)
4412 {
4413 struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4414 struct hwrm_func_drv_rgtr_input req = {0};
4415 DECLARE_BITMAP(async_events_bmap, 256);
4416 u32 *events = (u32 *)async_events_bmap;
4417 u32 flags;
4418 int rc, i;
4419
4420 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4421
4422 req.enables =
4423 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4424 FUNC_DRV_RGTR_REQ_ENABLES_VER |
4425 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4426
4427 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4428 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4429 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4430 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4431 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4432 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4433 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4434 req.flags = cpu_to_le32(flags);
4435 req.ver_maj_8b = DRV_VER_MAJ;
4436 req.ver_min_8b = DRV_VER_MIN;
4437 req.ver_upd_8b = DRV_VER_UPD;
4438 req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4439 req.ver_min = cpu_to_le16(DRV_VER_MIN);
4440 req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4441
4442 if (BNXT_PF(bp)) {
4443 u32 data[8];
4444 int i;
4445
4446 memset(data, 0, sizeof(data));
4447 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4448 u16 cmd = bnxt_vf_req_snif[i];
4449 unsigned int bit, idx;
4450
4451 idx = cmd / 32;
4452 bit = cmd % 32;
4453 data[idx] |= 1 << bit;
4454 }
4455
4456 for (i = 0; i < 8; i++)
4457 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4458
4459 req.enables |=
4460 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4461 }
4462
4463 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4464 req.flags |= cpu_to_le32(
4465 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4466
4467 memset(async_events_bmap, 0, sizeof(async_events_bmap));
4468 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4469 u16 event_id = bnxt_async_events_arr[i];
4470
4471 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4472 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4473 continue;
4474 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
4475 }
4476 if (bmap && bmap_size) {
4477 for (i = 0; i < bmap_size; i++) {
4478 if (test_bit(i, bmap))
4479 __set_bit(i, async_events_bmap);
4480 }
4481 }
4482 for (i = 0; i < 8; i++)
4483 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4484
4485 if (async_only)
4486 req.enables =
4487 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4488
4489 mutex_lock(&bp->hwrm_cmd_lock);
4490 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4491 if (!rc) {
4492 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4493 if (resp->flags &
4494 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4495 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4496 }
4497 mutex_unlock(&bp->hwrm_cmd_lock);
4498 return rc;
4499 }
4500
4501 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4502 {
4503 struct hwrm_func_drv_unrgtr_input req = {0};
4504
4505 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4506 return 0;
4507
4508 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4509 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4510 }
4511
4512 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4513 {
4514 u32 rc = 0;
4515 struct hwrm_tunnel_dst_port_free_input req = {0};
4516
4517 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4518 req.tunnel_type = tunnel_type;
4519
4520 switch (tunnel_type) {
4521 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4522 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
4523 break;
4524 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4525 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
4526 break;
4527 default:
4528 break;
4529 }
4530
4531 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4532 if (rc)
4533 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4534 rc);
4535 return rc;
4536 }
4537
4538 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4539 u8 tunnel_type)
4540 {
4541 u32 rc = 0;
4542 struct hwrm_tunnel_dst_port_alloc_input req = {0};
4543 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4544
4545 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4546
4547 req.tunnel_type = tunnel_type;
4548 req.tunnel_dst_port_val = port;
4549
4550 mutex_lock(&bp->hwrm_cmd_lock);
4551 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4552 if (rc) {
4553 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4554 rc);
4555 goto err_out;
4556 }
4557
4558 switch (tunnel_type) {
4559 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4560 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
4561 break;
4562 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4563 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
4564 break;
4565 default:
4566 break;
4567 }
4568
4569 err_out:
4570 mutex_unlock(&bp->hwrm_cmd_lock);
4571 return rc;
4572 }
4573
4574 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4575 {
4576 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4577 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4578
4579 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4580 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4581
4582 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4583 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4584 req.mask = cpu_to_le32(vnic->rx_mask);
4585 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4586 }
4587
4588 #ifdef CONFIG_RFS_ACCEL
4589 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4590 struct bnxt_ntuple_filter *fltr)
4591 {
4592 struct hwrm_cfa_ntuple_filter_free_input req = {0};
4593
4594 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4595 req.ntuple_filter_id = fltr->filter_id;
4596 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4597 }
4598
4599 #define BNXT_NTP_FLTR_FLAGS \
4600 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
4601 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
4602 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
4603 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
4604 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
4605 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
4606 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
4607 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
4608 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
4609 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
4610 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
4611 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
4612 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
4613 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4614
4615 #define BNXT_NTP_TUNNEL_FLTR_FLAG \
4616 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4617
4618 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4619 struct bnxt_ntuple_filter *fltr)
4620 {
4621 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4622 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4623 struct flow_keys *keys = &fltr->fkeys;
4624 struct bnxt_vnic_info *vnic;
4625 u32 flags = 0;
4626 int rc = 0;
4627
4628 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4629 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4630
4631 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4632 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4633 req.dst_id = cpu_to_le16(fltr->rxq);
4634 } else {
4635 vnic = &bp->vnic_info[fltr->rxq + 1];
4636 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4637 }
4638 req.flags = cpu_to_le32(flags);
4639 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4640
4641 req.ethertype = htons(ETH_P_IP);
4642 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4643 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4644 req.ip_protocol = keys->basic.ip_proto;
4645
4646 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4647 int i;
4648
4649 req.ethertype = htons(ETH_P_IPV6);
4650 req.ip_addr_type =
4651 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4652 *(struct in6_addr *)&req.src_ipaddr[0] =
4653 keys->addrs.v6addrs.src;
4654 *(struct in6_addr *)&req.dst_ipaddr[0] =
4655 keys->addrs.v6addrs.dst;
4656 for (i = 0; i < 4; i++) {
4657 req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4658 req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4659 }
4660 } else {
4661 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4662 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4663 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4664 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4665 }
4666 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4667 req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4668 req.tunnel_type =
4669 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4670 }
4671
4672 req.src_port = keys->ports.src;
4673 req.src_port_mask = cpu_to_be16(0xffff);
4674 req.dst_port = keys->ports.dst;
4675 req.dst_port_mask = cpu_to_be16(0xffff);
4676
4677 mutex_lock(&bp->hwrm_cmd_lock);
4678 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4679 if (!rc) {
4680 resp = bnxt_get_hwrm_resp_addr(bp, &req);
4681 fltr->filter_id = resp->ntuple_filter_id;
4682 }
4683 mutex_unlock(&bp->hwrm_cmd_lock);
4684 return rc;
4685 }
4686 #endif
4687
4688 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4689 u8 *mac_addr)
4690 {
4691 u32 rc = 0;
4692 struct hwrm_cfa_l2_filter_alloc_input req = {0};
4693 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4694
4695 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4696 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4697 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4698 req.flags |=
4699 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4700 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4701 req.enables =
4702 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4703 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4704 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4705 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4706 req.l2_addr_mask[0] = 0xff;
4707 req.l2_addr_mask[1] = 0xff;
4708 req.l2_addr_mask[2] = 0xff;
4709 req.l2_addr_mask[3] = 0xff;
4710 req.l2_addr_mask[4] = 0xff;
4711 req.l2_addr_mask[5] = 0xff;
4712
4713 mutex_lock(&bp->hwrm_cmd_lock);
4714 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4715 if (!rc)
4716 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4717 resp->l2_filter_id;
4718 mutex_unlock(&bp->hwrm_cmd_lock);
4719 return rc;
4720 }
4721
4722 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4723 {
4724 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4725 int rc = 0;
4726
4727 /* Any associated ntuple filters will also be cleared by firmware. */
4728 mutex_lock(&bp->hwrm_cmd_lock);
4729 for (i = 0; i < num_of_vnics; i++) {
4730 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4731
4732 for (j = 0; j < vnic->uc_filter_count; j++) {
4733 struct hwrm_cfa_l2_filter_free_input req = {0};
4734
4735 bnxt_hwrm_cmd_hdr_init(bp, &req,
4736 HWRM_CFA_L2_FILTER_FREE, -1, -1);
4737
4738 req.l2_filter_id = vnic->fw_l2_filter_id[j];
4739
4740 rc = _hwrm_send_message(bp, &req, sizeof(req),
4741 HWRM_CMD_TIMEOUT);
4742 }
4743 vnic->uc_filter_count = 0;
4744 }
4745 mutex_unlock(&bp->hwrm_cmd_lock);
4746
4747 return rc;
4748 }
4749
4750 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4751 {
4752 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4753 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4754 struct hwrm_vnic_tpa_cfg_input req = {0};
4755
4756 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4757 return 0;
4758
4759 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4760
4761 if (tpa_flags) {
4762 u16 mss = bp->dev->mtu - 40;
4763 u32 nsegs, n, segs = 0, flags;
4764
4765 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4766 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4767 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4768 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4769 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4770 if (tpa_flags & BNXT_FLAG_GRO)
4771 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4772
4773 req.flags = cpu_to_le32(flags);
4774
4775 req.enables =
4776 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4777 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4778 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4779
4780 /* Number of segs are log2 units, and first packet is not
4781 * included as part of this units.
4782 */
4783 if (mss <= BNXT_RX_PAGE_SIZE) {
4784 n = BNXT_RX_PAGE_SIZE / mss;
4785 nsegs = (MAX_SKB_FRAGS - 1) * n;
4786 } else {
4787 n = mss / BNXT_RX_PAGE_SIZE;
4788 if (mss & (BNXT_RX_PAGE_SIZE - 1))
4789 n++;
4790 nsegs = (MAX_SKB_FRAGS - n) / n;
4791 }
4792
4793 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4794 segs = MAX_TPA_SEGS_P5;
4795 max_aggs = bp->max_tpa;
4796 } else {
4797 segs = ilog2(nsegs);
4798 }
4799 req.max_agg_segs = cpu_to_le16(segs);
4800 req.max_aggs = cpu_to_le16(max_aggs);
4801
4802 req.min_agg_len = cpu_to_le32(512);
4803 }
4804 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4805
4806 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4807 }
4808
4809 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4810 {
4811 struct bnxt_ring_grp_info *grp_info;
4812
4813 grp_info = &bp->grp_info[ring->grp_idx];
4814 return grp_info->cp_fw_ring_id;
4815 }
4816
4817 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4818 {
4819 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4820 struct bnxt_napi *bnapi = rxr->bnapi;
4821 struct bnxt_cp_ring_info *cpr;
4822
4823 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4824 return cpr->cp_ring_struct.fw_ring_id;
4825 } else {
4826 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4827 }
4828 }
4829
4830 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4831 {
4832 if (bp->flags & BNXT_FLAG_CHIP_P5) {
4833 struct bnxt_napi *bnapi = txr->bnapi;
4834 struct bnxt_cp_ring_info *cpr;
4835
4836 cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4837 return cpr->cp_ring_struct.fw_ring_id;
4838 } else {
4839 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4840 }
4841 }
4842
4843 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
4844 {
4845 u32 i, j, max_rings;
4846 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4847 struct hwrm_vnic_rss_cfg_input req = {0};
4848
4849 if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
4850 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
4851 return 0;
4852
4853 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4854 if (set_rss) {
4855 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4856 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4857 if (vnic->flags & BNXT_VNIC_RSS_FLAG) {
4858 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4859 max_rings = bp->rx_nr_rings - 1;
4860 else
4861 max_rings = bp->rx_nr_rings;
4862 } else {
4863 max_rings = 1;
4864 }
4865
4866 /* Fill the RSS indirection table with ring group ids */
4867 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
4868 if (j == max_rings)
4869 j = 0;
4870 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
4871 }
4872
4873 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4874 req.hash_key_tbl_addr =
4875 cpu_to_le64(vnic->rss_hash_key_dma_addr);
4876 }
4877 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
4878 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4879 }
4880
4881 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
4882 {
4883 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4884 u32 i, j, k, nr_ctxs, max_rings = bp->rx_nr_rings;
4885 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
4886 struct hwrm_vnic_rss_cfg_input req = {0};
4887
4888 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
4889 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4890 if (!set_rss) {
4891 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4892 return 0;
4893 }
4894 req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
4895 req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
4896 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
4897 req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
4898 nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
4899 for (i = 0, k = 0; i < nr_ctxs; i++) {
4900 __le16 *ring_tbl = vnic->rss_table;
4901 int rc;
4902
4903 req.ring_table_pair_index = i;
4904 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
4905 for (j = 0; j < 64; j++) {
4906 u16 ring_id;
4907
4908 ring_id = rxr->rx_ring_struct.fw_ring_id;
4909 *ring_tbl++ = cpu_to_le16(ring_id);
4910 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
4911 *ring_tbl++ = cpu_to_le16(ring_id);
4912 rxr++;
4913 k++;
4914 if (k == max_rings) {
4915 k = 0;
4916 rxr = &bp->rx_ring[0];
4917 }
4918 }
4919 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4920 if (rc)
4921 return rc;
4922 }
4923 return 0;
4924 }
4925
4926 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
4927 {
4928 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4929 struct hwrm_vnic_plcmodes_cfg_input req = {0};
4930
4931 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
4932 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
4933 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
4934 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
4935 req.enables =
4936 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
4937 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
4938 /* thresholds not implemented in firmware yet */
4939 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
4940 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
4941 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4942 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4943 }
4944
4945 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
4946 u16 ctx_idx)
4947 {
4948 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
4949
4950 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
4951 req.rss_cos_lb_ctx_id =
4952 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
4953
4954 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4955 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
4956 }
4957
4958 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
4959 {
4960 int i, j;
4961
4962 for (i = 0; i < bp->nr_vnics; i++) {
4963 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4964
4965 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
4966 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
4967 bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
4968 }
4969 }
4970 bp->rsscos_nr_ctxs = 0;
4971 }
4972
4973 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
4974 {
4975 int rc;
4976 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
4977 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
4978 bp->hwrm_cmd_resp_addr;
4979
4980 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
4981 -1);
4982
4983 mutex_lock(&bp->hwrm_cmd_lock);
4984 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4985 if (!rc)
4986 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
4987 le16_to_cpu(resp->rss_cos_lb_ctx_id);
4988 mutex_unlock(&bp->hwrm_cmd_lock);
4989
4990 return rc;
4991 }
4992
4993 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
4994 {
4995 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
4996 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
4997 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
4998 }
4999
5000 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5001 {
5002 unsigned int ring = 0, grp_idx;
5003 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5004 struct hwrm_vnic_cfg_input req = {0};
5005 u16 def_vlan = 0;
5006
5007 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5008
5009 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5010 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5011
5012 req.default_rx_ring_id =
5013 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5014 req.default_cmpl_ring_id =
5015 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5016 req.enables =
5017 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5018 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5019 goto vnic_mru;
5020 }
5021 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5022 /* Only RSS support for now TBD: COS & LB */
5023 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5024 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5025 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5026 VNIC_CFG_REQ_ENABLES_MRU);
5027 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5028 req.rss_rule =
5029 cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5030 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5031 VNIC_CFG_REQ_ENABLES_MRU);
5032 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5033 } else {
5034 req.rss_rule = cpu_to_le16(0xffff);
5035 }
5036
5037 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5038 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5039 req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5040 req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5041 } else {
5042 req.cos_rule = cpu_to_le16(0xffff);
5043 }
5044
5045 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5046 ring = 0;
5047 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5048 ring = vnic_id - 1;
5049 else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5050 ring = bp->rx_nr_rings - 1;
5051
5052 grp_idx = bp->rx_ring[ring].bnapi->index;
5053 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5054 req.lb_rule = cpu_to_le16(0xffff);
5055 vnic_mru:
5056 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
5057 VLAN_HLEN);
5058
5059 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5060 #ifdef CONFIG_BNXT_SRIOV
5061 if (BNXT_VF(bp))
5062 def_vlan = bp->vf.vlan;
5063 #endif
5064 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5065 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5066 if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5067 req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5068
5069 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5070 }
5071
5072 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5073 {
5074 u32 rc = 0;
5075
5076 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5077 struct hwrm_vnic_free_input req = {0};
5078
5079 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5080 req.vnic_id =
5081 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5082
5083 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5084 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5085 }
5086 return rc;
5087 }
5088
5089 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5090 {
5091 u16 i;
5092
5093 for (i = 0; i < bp->nr_vnics; i++)
5094 bnxt_hwrm_vnic_free_one(bp, i);
5095 }
5096
5097 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5098 unsigned int start_rx_ring_idx,
5099 unsigned int nr_rings)
5100 {
5101 int rc = 0;
5102 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5103 struct hwrm_vnic_alloc_input req = {0};
5104 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5105 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5106
5107 if (bp->flags & BNXT_FLAG_CHIP_P5)
5108 goto vnic_no_ring_grps;
5109
5110 /* map ring groups to this vnic */
5111 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5112 grp_idx = bp->rx_ring[i].bnapi->index;
5113 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5114 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5115 j, nr_rings);
5116 break;
5117 }
5118 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5119 }
5120
5121 vnic_no_ring_grps:
5122 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5123 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5124 if (vnic_id == 0)
5125 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5126
5127 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5128
5129 mutex_lock(&bp->hwrm_cmd_lock);
5130 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5131 if (!rc)
5132 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5133 mutex_unlock(&bp->hwrm_cmd_lock);
5134 return rc;
5135 }
5136
5137 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5138 {
5139 struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5140 struct hwrm_vnic_qcaps_input req = {0};
5141 int rc;
5142
5143 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5144 bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5145 if (bp->hwrm_spec_code < 0x10600)
5146 return 0;
5147
5148 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5149 mutex_lock(&bp->hwrm_cmd_lock);
5150 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5151 if (!rc) {
5152 u32 flags = le32_to_cpu(resp->flags);
5153
5154 if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5155 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5156 bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5157 if (flags &
5158 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5159 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5160 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5161 if (bp->max_tpa_v2)
5162 bp->hw_ring_stats_size =
5163 sizeof(struct ctx_hw_stats_ext);
5164 }
5165 mutex_unlock(&bp->hwrm_cmd_lock);
5166 return rc;
5167 }
5168
5169 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5170 {
5171 u16 i;
5172 u32 rc = 0;
5173
5174 if (bp->flags & BNXT_FLAG_CHIP_P5)
5175 return 0;
5176
5177 mutex_lock(&bp->hwrm_cmd_lock);
5178 for (i = 0; i < bp->rx_nr_rings; i++) {
5179 struct hwrm_ring_grp_alloc_input req = {0};
5180 struct hwrm_ring_grp_alloc_output *resp =
5181 bp->hwrm_cmd_resp_addr;
5182 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5183
5184 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5185
5186 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5187 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5188 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5189 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5190
5191 rc = _hwrm_send_message(bp, &req, sizeof(req),
5192 HWRM_CMD_TIMEOUT);
5193 if (rc)
5194 break;
5195
5196 bp->grp_info[grp_idx].fw_grp_id =
5197 le32_to_cpu(resp->ring_group_id);
5198 }
5199 mutex_unlock(&bp->hwrm_cmd_lock);
5200 return rc;
5201 }
5202
5203 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5204 {
5205 u16 i;
5206 u32 rc = 0;
5207 struct hwrm_ring_grp_free_input req = {0};
5208
5209 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5210 return 0;
5211
5212 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5213
5214 mutex_lock(&bp->hwrm_cmd_lock);
5215 for (i = 0; i < bp->cp_nr_rings; i++) {
5216 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5217 continue;
5218 req.ring_group_id =
5219 cpu_to_le32(bp->grp_info[i].fw_grp_id);
5220
5221 rc = _hwrm_send_message(bp, &req, sizeof(req),
5222 HWRM_CMD_TIMEOUT);
5223 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5224 }
5225 mutex_unlock(&bp->hwrm_cmd_lock);
5226 return rc;
5227 }
5228
5229 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5230 struct bnxt_ring_struct *ring,
5231 u32 ring_type, u32 map_index)
5232 {
5233 int rc = 0, err = 0;
5234 struct hwrm_ring_alloc_input req = {0};
5235 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5236 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5237 struct bnxt_ring_grp_info *grp_info;
5238 u16 ring_id;
5239
5240 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5241
5242 req.enables = 0;
5243 if (rmem->nr_pages > 1) {
5244 req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5245 /* Page size is in log2 units */
5246 req.page_size = BNXT_PAGE_SHIFT;
5247 req.page_tbl_depth = 1;
5248 } else {
5249 req.page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
5250 }
5251 req.fbo = 0;
5252 /* Association of ring index with doorbell index and MSIX number */
5253 req.logical_id = cpu_to_le16(map_index);
5254
5255 switch (ring_type) {
5256 case HWRM_RING_ALLOC_TX: {
5257 struct bnxt_tx_ring_info *txr;
5258
5259 txr = container_of(ring, struct bnxt_tx_ring_info,
5260 tx_ring_struct);
5261 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5262 /* Association of transmit ring with completion ring */
5263 grp_info = &bp->grp_info[ring->grp_idx];
5264 req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5265 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5266 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5267 req.queue_id = cpu_to_le16(ring->queue_id);
5268 break;
5269 }
5270 case HWRM_RING_ALLOC_RX:
5271 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5272 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5273 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5274 u16 flags = 0;
5275
5276 /* Association of rx ring with stats context */
5277 grp_info = &bp->grp_info[ring->grp_idx];
5278 req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5279 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5280 req.enables |= cpu_to_le32(
5281 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5282 if (NET_IP_ALIGN == 2)
5283 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5284 req.flags = cpu_to_le16(flags);
5285 }
5286 break;
5287 case HWRM_RING_ALLOC_AGG:
5288 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5289 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5290 /* Association of agg ring with rx ring */
5291 grp_info = &bp->grp_info[ring->grp_idx];
5292 req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5293 req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5294 req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5295 req.enables |= cpu_to_le32(
5296 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5297 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5298 } else {
5299 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5300 }
5301 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5302 break;
5303 case HWRM_RING_ALLOC_CMPL:
5304 req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5305 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5306 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5307 /* Association of cp ring with nq */
5308 grp_info = &bp->grp_info[map_index];
5309 req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5310 req.cq_handle = cpu_to_le64(ring->handle);
5311 req.enables |= cpu_to_le32(
5312 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5313 } else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5314 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5315 }
5316 break;
5317 case HWRM_RING_ALLOC_NQ:
5318 req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5319 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5320 if (bp->flags & BNXT_FLAG_USING_MSIX)
5321 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5322 break;
5323 default:
5324 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5325 ring_type);
5326 return -1;
5327 }
5328
5329 mutex_lock(&bp->hwrm_cmd_lock);
5330 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5331 err = le16_to_cpu(resp->error_code);
5332 ring_id = le16_to_cpu(resp->ring_id);
5333 mutex_unlock(&bp->hwrm_cmd_lock);
5334
5335 if (rc || err) {
5336 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5337 ring_type, rc, err);
5338 return -EIO;
5339 }
5340 ring->fw_ring_id = ring_id;
5341 return rc;
5342 }
5343
5344 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5345 {
5346 int rc;
5347
5348 if (BNXT_PF(bp)) {
5349 struct hwrm_func_cfg_input req = {0};
5350
5351 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5352 req.fid = cpu_to_le16(0xffff);
5353 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5354 req.async_event_cr = cpu_to_le16(idx);
5355 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5356 } else {
5357 struct hwrm_func_vf_cfg_input req = {0};
5358
5359 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5360 req.enables =
5361 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5362 req.async_event_cr = cpu_to_le16(idx);
5363 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5364 }
5365 return rc;
5366 }
5367
5368 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5369 u32 map_idx, u32 xid)
5370 {
5371 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5372 if (BNXT_PF(bp))
5373 db->doorbell = bp->bar1 + 0x10000;
5374 else
5375 db->doorbell = bp->bar1 + 0x4000;
5376 switch (ring_type) {
5377 case HWRM_RING_ALLOC_TX:
5378 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5379 break;
5380 case HWRM_RING_ALLOC_RX:
5381 case HWRM_RING_ALLOC_AGG:
5382 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5383 break;
5384 case HWRM_RING_ALLOC_CMPL:
5385 db->db_key64 = DBR_PATH_L2;
5386 break;
5387 case HWRM_RING_ALLOC_NQ:
5388 db->db_key64 = DBR_PATH_L2;
5389 break;
5390 }
5391 db->db_key64 |= (u64)xid << DBR_XID_SFT;
5392 } else {
5393 db->doorbell = bp->bar1 + map_idx * 0x80;
5394 switch (ring_type) {
5395 case HWRM_RING_ALLOC_TX:
5396 db->db_key32 = DB_KEY_TX;
5397 break;
5398 case HWRM_RING_ALLOC_RX:
5399 case HWRM_RING_ALLOC_AGG:
5400 db->db_key32 = DB_KEY_RX;
5401 break;
5402 case HWRM_RING_ALLOC_CMPL:
5403 db->db_key32 = DB_KEY_CP;
5404 break;
5405 }
5406 }
5407 }
5408
5409 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5410 {
5411 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5412 int i, rc = 0;
5413 u32 type;
5414
5415 if (bp->flags & BNXT_FLAG_CHIP_P5)
5416 type = HWRM_RING_ALLOC_NQ;
5417 else
5418 type = HWRM_RING_ALLOC_CMPL;
5419 for (i = 0; i < bp->cp_nr_rings; i++) {
5420 struct bnxt_napi *bnapi = bp->bnapi[i];
5421 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5422 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5423 u32 map_idx = ring->map_idx;
5424 unsigned int vector;
5425
5426 vector = bp->irq_tbl[map_idx].vector;
5427 disable_irq_nosync(vector);
5428 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5429 if (rc) {
5430 enable_irq(vector);
5431 goto err_out;
5432 }
5433 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5434 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5435 enable_irq(vector);
5436 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5437
5438 if (!i) {
5439 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5440 if (rc)
5441 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5442 }
5443 }
5444
5445 type = HWRM_RING_ALLOC_TX;
5446 for (i = 0; i < bp->tx_nr_rings; i++) {
5447 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5448 struct bnxt_ring_struct *ring;
5449 u32 map_idx;
5450
5451 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5452 struct bnxt_napi *bnapi = txr->bnapi;
5453 struct bnxt_cp_ring_info *cpr, *cpr2;
5454 u32 type2 = HWRM_RING_ALLOC_CMPL;
5455
5456 cpr = &bnapi->cp_ring;
5457 cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5458 ring = &cpr2->cp_ring_struct;
5459 ring->handle = BNXT_TX_HDL;
5460 map_idx = bnapi->index;
5461 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5462 if (rc)
5463 goto err_out;
5464 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5465 ring->fw_ring_id);
5466 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5467 }
5468 ring = &txr->tx_ring_struct;
5469 map_idx = i;
5470 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5471 if (rc)
5472 goto err_out;
5473 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5474 }
5475
5476 type = HWRM_RING_ALLOC_RX;
5477 for (i = 0; i < bp->rx_nr_rings; i++) {
5478 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5479 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5480 struct bnxt_napi *bnapi = rxr->bnapi;
5481 u32 map_idx = bnapi->index;
5482
5483 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5484 if (rc)
5485 goto err_out;
5486 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5487 /* If we have agg rings, post agg buffers first. */
5488 if (!agg_rings)
5489 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5490 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5491 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5492 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5493 u32 type2 = HWRM_RING_ALLOC_CMPL;
5494 struct bnxt_cp_ring_info *cpr2;
5495
5496 cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5497 ring = &cpr2->cp_ring_struct;
5498 ring->handle = BNXT_RX_HDL;
5499 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5500 if (rc)
5501 goto err_out;
5502 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5503 ring->fw_ring_id);
5504 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5505 }
5506 }
5507
5508 if (agg_rings) {
5509 type = HWRM_RING_ALLOC_AGG;
5510 for (i = 0; i < bp->rx_nr_rings; i++) {
5511 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5512 struct bnxt_ring_struct *ring =
5513 &rxr->rx_agg_ring_struct;
5514 u32 grp_idx = ring->grp_idx;
5515 u32 map_idx = grp_idx + bp->rx_nr_rings;
5516
5517 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5518 if (rc)
5519 goto err_out;
5520
5521 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5522 ring->fw_ring_id);
5523 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5524 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5525 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5526 }
5527 }
5528 err_out:
5529 return rc;
5530 }
5531
5532 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5533 struct bnxt_ring_struct *ring,
5534 u32 ring_type, int cmpl_ring_id)
5535 {
5536 int rc;
5537 struct hwrm_ring_free_input req = {0};
5538 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5539 u16 error_code;
5540
5541 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
5542 return 0;
5543
5544 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5545 req.ring_type = ring_type;
5546 req.ring_id = cpu_to_le16(ring->fw_ring_id);
5547
5548 mutex_lock(&bp->hwrm_cmd_lock);
5549 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5550 error_code = le16_to_cpu(resp->error_code);
5551 mutex_unlock(&bp->hwrm_cmd_lock);
5552
5553 if (rc || error_code) {
5554 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5555 ring_type, rc, error_code);
5556 return -EIO;
5557 }
5558 return 0;
5559 }
5560
5561 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5562 {
5563 u32 type;
5564 int i;
5565
5566 if (!bp->bnapi)
5567 return;
5568
5569 for (i = 0; i < bp->tx_nr_rings; i++) {
5570 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5571 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5572
5573 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5574 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5575
5576 hwrm_ring_free_send_msg(bp, ring,
5577 RING_FREE_REQ_RING_TYPE_TX,
5578 close_path ? cmpl_ring_id :
5579 INVALID_HW_RING_ID);
5580 ring->fw_ring_id = INVALID_HW_RING_ID;
5581 }
5582 }
5583
5584 for (i = 0; i < bp->rx_nr_rings; i++) {
5585 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5586 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5587 u32 grp_idx = rxr->bnapi->index;
5588
5589 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5590 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5591
5592 hwrm_ring_free_send_msg(bp, ring,
5593 RING_FREE_REQ_RING_TYPE_RX,
5594 close_path ? cmpl_ring_id :
5595 INVALID_HW_RING_ID);
5596 ring->fw_ring_id = INVALID_HW_RING_ID;
5597 bp->grp_info[grp_idx].rx_fw_ring_id =
5598 INVALID_HW_RING_ID;
5599 }
5600 }
5601
5602 if (bp->flags & BNXT_FLAG_CHIP_P5)
5603 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5604 else
5605 type = RING_FREE_REQ_RING_TYPE_RX;
5606 for (i = 0; i < bp->rx_nr_rings; i++) {
5607 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5608 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5609 u32 grp_idx = rxr->bnapi->index;
5610
5611 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5612 u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5613
5614 hwrm_ring_free_send_msg(bp, ring, type,
5615 close_path ? cmpl_ring_id :
5616 INVALID_HW_RING_ID);
5617 ring->fw_ring_id = INVALID_HW_RING_ID;
5618 bp->grp_info[grp_idx].agg_fw_ring_id =
5619 INVALID_HW_RING_ID;
5620 }
5621 }
5622
5623 /* The completion rings are about to be freed. After that the
5624 * IRQ doorbell will not work anymore. So we need to disable
5625 * IRQ here.
5626 */
5627 bnxt_disable_int_sync(bp);
5628
5629 if (bp->flags & BNXT_FLAG_CHIP_P5)
5630 type = RING_FREE_REQ_RING_TYPE_NQ;
5631 else
5632 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5633 for (i = 0; i < bp->cp_nr_rings; i++) {
5634 struct bnxt_napi *bnapi = bp->bnapi[i];
5635 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5636 struct bnxt_ring_struct *ring;
5637 int j;
5638
5639 for (j = 0; j < 2; j++) {
5640 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5641
5642 if (cpr2) {
5643 ring = &cpr2->cp_ring_struct;
5644 if (ring->fw_ring_id == INVALID_HW_RING_ID)
5645 continue;
5646 hwrm_ring_free_send_msg(bp, ring,
5647 RING_FREE_REQ_RING_TYPE_L2_CMPL,
5648 INVALID_HW_RING_ID);
5649 ring->fw_ring_id = INVALID_HW_RING_ID;
5650 }
5651 }
5652 ring = &cpr->cp_ring_struct;
5653 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5654 hwrm_ring_free_send_msg(bp, ring, type,
5655 INVALID_HW_RING_ID);
5656 ring->fw_ring_id = INVALID_HW_RING_ID;
5657 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5658 }
5659 }
5660 }
5661
5662 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5663 bool shared);
5664
5665 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5666 {
5667 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5668 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5669 struct hwrm_func_qcfg_input req = {0};
5670 int rc;
5671
5672 if (bp->hwrm_spec_code < 0x10601)
5673 return 0;
5674
5675 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5676 req.fid = cpu_to_le16(0xffff);
5677 mutex_lock(&bp->hwrm_cmd_lock);
5678 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5679 if (rc) {
5680 mutex_unlock(&bp->hwrm_cmd_lock);
5681 return rc;
5682 }
5683
5684 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5685 if (BNXT_NEW_RM(bp)) {
5686 u16 cp, stats;
5687
5688 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5689 hw_resc->resv_hw_ring_grps =
5690 le32_to_cpu(resp->alloc_hw_ring_grps);
5691 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5692 cp = le16_to_cpu(resp->alloc_cmpl_rings);
5693 stats = le16_to_cpu(resp->alloc_stat_ctx);
5694 hw_resc->resv_irqs = cp;
5695 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5696 int rx = hw_resc->resv_rx_rings;
5697 int tx = hw_resc->resv_tx_rings;
5698
5699 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5700 rx >>= 1;
5701 if (cp < (rx + tx)) {
5702 bnxt_trim_rings(bp, &rx, &tx, cp, false);
5703 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5704 rx <<= 1;
5705 hw_resc->resv_rx_rings = rx;
5706 hw_resc->resv_tx_rings = tx;
5707 }
5708 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5709 hw_resc->resv_hw_ring_grps = rx;
5710 }
5711 hw_resc->resv_cp_rings = cp;
5712 hw_resc->resv_stat_ctxs = stats;
5713 }
5714 mutex_unlock(&bp->hwrm_cmd_lock);
5715 return 0;
5716 }
5717
5718 /* Caller must hold bp->hwrm_cmd_lock */
5719 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5720 {
5721 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5722 struct hwrm_func_qcfg_input req = {0};
5723 int rc;
5724
5725 if (bp->hwrm_spec_code < 0x10601)
5726 return 0;
5727
5728 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5729 req.fid = cpu_to_le16(fid);
5730 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5731 if (!rc)
5732 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5733
5734 return rc;
5735 }
5736
5737 static bool bnxt_rfs_supported(struct bnxt *bp);
5738
5739 static void
5740 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5741 int tx_rings, int rx_rings, int ring_grps,
5742 int cp_rings, int stats, int vnics)
5743 {
5744 u32 enables = 0;
5745
5746 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5747 req->fid = cpu_to_le16(0xffff);
5748 enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5749 req->num_tx_rings = cpu_to_le16(tx_rings);
5750 if (BNXT_NEW_RM(bp)) {
5751 enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5752 enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5753 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5754 enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5755 enables |= tx_rings + ring_grps ?
5756 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5757 enables |= rx_rings ?
5758 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5759 } else {
5760 enables |= cp_rings ?
5761 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5762 enables |= ring_grps ?
5763 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5764 FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5765 }
5766 enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5767
5768 req->num_rx_rings = cpu_to_le16(rx_rings);
5769 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5770 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5771 req->num_msix = cpu_to_le16(cp_rings);
5772 req->num_rsscos_ctxs =
5773 cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5774 } else {
5775 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5776 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5777 req->num_rsscos_ctxs = cpu_to_le16(1);
5778 if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5779 bnxt_rfs_supported(bp))
5780 req->num_rsscos_ctxs =
5781 cpu_to_le16(ring_grps + 1);
5782 }
5783 req->num_stat_ctxs = cpu_to_le16(stats);
5784 req->num_vnics = cpu_to_le16(vnics);
5785 }
5786 req->enables = cpu_to_le32(enables);
5787 }
5788
5789 static void
5790 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5791 struct hwrm_func_vf_cfg_input *req, int tx_rings,
5792 int rx_rings, int ring_grps, int cp_rings,
5793 int stats, int vnics)
5794 {
5795 u32 enables = 0;
5796
5797 bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5798 enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5799 enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5800 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5801 enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5802 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5803 enables |= tx_rings + ring_grps ?
5804 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5805 } else {
5806 enables |= cp_rings ?
5807 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5808 enables |= ring_grps ?
5809 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5810 }
5811 enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5812 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5813
5814 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
5815 req->num_tx_rings = cpu_to_le16(tx_rings);
5816 req->num_rx_rings = cpu_to_le16(rx_rings);
5817 if (bp->flags & BNXT_FLAG_CHIP_P5) {
5818 req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5819 req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5820 } else {
5821 req->num_cmpl_rings = cpu_to_le16(cp_rings);
5822 req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5823 req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
5824 }
5825 req->num_stat_ctxs = cpu_to_le16(stats);
5826 req->num_vnics = cpu_to_le16(vnics);
5827
5828 req->enables = cpu_to_le32(enables);
5829 }
5830
5831 static int
5832 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5833 int ring_grps, int cp_rings, int stats, int vnics)
5834 {
5835 struct hwrm_func_cfg_input req = {0};
5836 int rc;
5837
5838 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5839 cp_rings, stats, vnics);
5840 if (!req.enables)
5841 return 0;
5842
5843 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5844 if (rc)
5845 return rc;
5846
5847 if (bp->hwrm_spec_code < 0x10601)
5848 bp->hw_resc.resv_tx_rings = tx_rings;
5849
5850 rc = bnxt_hwrm_get_rings(bp);
5851 return rc;
5852 }
5853
5854 static int
5855 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
5856 int ring_grps, int cp_rings, int stats, int vnics)
5857 {
5858 struct hwrm_func_vf_cfg_input req = {0};
5859 int rc;
5860
5861 if (!BNXT_NEW_RM(bp)) {
5862 bp->hw_resc.resv_tx_rings = tx_rings;
5863 return 0;
5864 }
5865
5866 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
5867 cp_rings, stats, vnics);
5868 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5869 if (rc)
5870 return rc;
5871
5872 rc = bnxt_hwrm_get_rings(bp);
5873 return rc;
5874 }
5875
5876 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
5877 int cp, int stat, int vnic)
5878 {
5879 if (BNXT_PF(bp))
5880 return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
5881 vnic);
5882 else
5883 return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
5884 vnic);
5885 }
5886
5887 int bnxt_nq_rings_in_use(struct bnxt *bp)
5888 {
5889 int cp = bp->cp_nr_rings;
5890 int ulp_msix, ulp_base;
5891
5892 ulp_msix = bnxt_get_ulp_msix_num(bp);
5893 if (ulp_msix) {
5894 ulp_base = bnxt_get_ulp_msix_base(bp);
5895 cp += ulp_msix;
5896 if ((ulp_base + ulp_msix) > cp)
5897 cp = ulp_base + ulp_msix;
5898 }
5899 return cp;
5900 }
5901
5902 static int bnxt_cp_rings_in_use(struct bnxt *bp)
5903 {
5904 int cp;
5905
5906 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
5907 return bnxt_nq_rings_in_use(bp);
5908
5909 cp = bp->tx_nr_rings + bp->rx_nr_rings;
5910 return cp;
5911 }
5912
5913 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
5914 {
5915 int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
5916 int cp = bp->cp_nr_rings;
5917
5918 if (!ulp_stat)
5919 return cp;
5920
5921 if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
5922 return bnxt_get_ulp_msix_base(bp) + ulp_stat;
5923
5924 return cp + ulp_stat;
5925 }
5926
5927 static bool bnxt_need_reserve_rings(struct bnxt *bp)
5928 {
5929 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5930 int cp = bnxt_cp_rings_in_use(bp);
5931 int nq = bnxt_nq_rings_in_use(bp);
5932 int rx = bp->rx_nr_rings, stat;
5933 int vnic = 1, grp = rx;
5934
5935 if (bp->hwrm_spec_code < 0x10601)
5936 return false;
5937
5938 if (hw_resc->resv_tx_rings != bp->tx_nr_rings)
5939 return true;
5940
5941 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5942 vnic = rx + 1;
5943 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5944 rx <<= 1;
5945 stat = bnxt_get_func_stat_ctxs(bp);
5946 if (BNXT_NEW_RM(bp) &&
5947 (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
5948 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
5949 (hw_resc->resv_hw_ring_grps != grp &&
5950 !(bp->flags & BNXT_FLAG_CHIP_P5))))
5951 return true;
5952 if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
5953 hw_resc->resv_irqs != nq)
5954 return true;
5955 return false;
5956 }
5957
5958 static int __bnxt_reserve_rings(struct bnxt *bp)
5959 {
5960 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5961 int cp = bnxt_nq_rings_in_use(bp);
5962 int tx = bp->tx_nr_rings;
5963 int rx = bp->rx_nr_rings;
5964 int grp, rx_rings, rc;
5965 int vnic = 1, stat;
5966 bool sh = false;
5967
5968 if (!bnxt_need_reserve_rings(bp))
5969 return 0;
5970
5971 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5972 sh = true;
5973 if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
5974 vnic = rx + 1;
5975 if (bp->flags & BNXT_FLAG_AGG_RINGS)
5976 rx <<= 1;
5977 grp = bp->rx_nr_rings;
5978 stat = bnxt_get_func_stat_ctxs(bp);
5979
5980 rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
5981 if (rc)
5982 return rc;
5983
5984 tx = hw_resc->resv_tx_rings;
5985 if (BNXT_NEW_RM(bp)) {
5986 rx = hw_resc->resv_rx_rings;
5987 cp = hw_resc->resv_irqs;
5988 grp = hw_resc->resv_hw_ring_grps;
5989 vnic = hw_resc->resv_vnics;
5990 stat = hw_resc->resv_stat_ctxs;
5991 }
5992
5993 rx_rings = rx;
5994 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
5995 if (rx >= 2) {
5996 rx_rings = rx >> 1;
5997 } else {
5998 if (netif_running(bp->dev))
5999 return -ENOMEM;
6000
6001 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6002 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6003 bp->dev->hw_features &= ~NETIF_F_LRO;
6004 bp->dev->features &= ~NETIF_F_LRO;
6005 bnxt_set_ring_params(bp);
6006 }
6007 }
6008 rx_rings = min_t(int, rx_rings, grp);
6009 cp = min_t(int, cp, bp->cp_nr_rings);
6010 if (stat > bnxt_get_ulp_stat_ctxs(bp))
6011 stat -= bnxt_get_ulp_stat_ctxs(bp);
6012 cp = min_t(int, cp, stat);
6013 rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6014 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6015 rx = rx_rings << 1;
6016 cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6017 bp->tx_nr_rings = tx;
6018 bp->rx_nr_rings = rx_rings;
6019 bp->cp_nr_rings = cp;
6020
6021 if (!tx || !rx || !cp || !grp || !vnic || !stat)
6022 return -ENOMEM;
6023
6024 return rc;
6025 }
6026
6027 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6028 int ring_grps, int cp_rings, int stats,
6029 int vnics)
6030 {
6031 struct hwrm_func_vf_cfg_input req = {0};
6032 u32 flags;
6033 int rc;
6034
6035 if (!BNXT_NEW_RM(bp))
6036 return 0;
6037
6038 __bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6039 cp_rings, stats, vnics);
6040 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6041 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6042 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6043 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6044 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6045 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6046 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6047 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6048
6049 req.flags = cpu_to_le32(flags);
6050 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6051 return rc;
6052 }
6053
6054 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6055 int ring_grps, int cp_rings, int stats,
6056 int vnics)
6057 {
6058 struct hwrm_func_cfg_input req = {0};
6059 u32 flags;
6060 int rc;
6061
6062 __bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6063 cp_rings, stats, vnics);
6064 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6065 if (BNXT_NEW_RM(bp)) {
6066 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6067 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6068 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6069 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6070 if (bp->flags & BNXT_FLAG_CHIP_P5)
6071 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6072 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6073 else
6074 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6075 }
6076
6077 req.flags = cpu_to_le32(flags);
6078 rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6079 return rc;
6080 }
6081
6082 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6083 int ring_grps, int cp_rings, int stats,
6084 int vnics)
6085 {
6086 if (bp->hwrm_spec_code < 0x10801)
6087 return 0;
6088
6089 if (BNXT_PF(bp))
6090 return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6091 ring_grps, cp_rings, stats,
6092 vnics);
6093
6094 return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6095 cp_rings, stats, vnics);
6096 }
6097
6098 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6099 {
6100 struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6101 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6102 struct hwrm_ring_aggint_qcaps_input req = {0};
6103 int rc;
6104
6105 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6106 coal_cap->num_cmpl_dma_aggr_max = 63;
6107 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6108 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6109 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6110 coal_cap->int_lat_tmr_min_max = 65535;
6111 coal_cap->int_lat_tmr_max_max = 65535;
6112 coal_cap->num_cmpl_aggr_int_max = 65535;
6113 coal_cap->timer_units = 80;
6114
6115 if (bp->hwrm_spec_code < 0x10902)
6116 return;
6117
6118 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6119 mutex_lock(&bp->hwrm_cmd_lock);
6120 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6121 if (!rc) {
6122 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6123 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6124 coal_cap->num_cmpl_dma_aggr_max =
6125 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6126 coal_cap->num_cmpl_dma_aggr_during_int_max =
6127 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6128 coal_cap->cmpl_aggr_dma_tmr_max =
6129 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6130 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6131 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6132 coal_cap->int_lat_tmr_min_max =
6133 le16_to_cpu(resp->int_lat_tmr_min_max);
6134 coal_cap->int_lat_tmr_max_max =
6135 le16_to_cpu(resp->int_lat_tmr_max_max);
6136 coal_cap->num_cmpl_aggr_int_max =
6137 le16_to_cpu(resp->num_cmpl_aggr_int_max);
6138 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6139 }
6140 mutex_unlock(&bp->hwrm_cmd_lock);
6141 }
6142
6143 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6144 {
6145 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6146
6147 return usec * 1000 / coal_cap->timer_units;
6148 }
6149
6150 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6151 struct bnxt_coal *hw_coal,
6152 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6153 {
6154 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6155 u32 cmpl_params = coal_cap->cmpl_params;
6156 u16 val, tmr, max, flags = 0;
6157
6158 max = hw_coal->bufs_per_record * 128;
6159 if (hw_coal->budget)
6160 max = hw_coal->bufs_per_record * hw_coal->budget;
6161 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6162
6163 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6164 req->num_cmpl_aggr_int = cpu_to_le16(val);
6165
6166 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6167 req->num_cmpl_dma_aggr = cpu_to_le16(val);
6168
6169 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6170 coal_cap->num_cmpl_dma_aggr_during_int_max);
6171 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6172
6173 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6174 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6175 req->int_lat_tmr_max = cpu_to_le16(tmr);
6176
6177 /* min timer set to 1/2 of interrupt timer */
6178 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6179 val = tmr / 2;
6180 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6181 req->int_lat_tmr_min = cpu_to_le16(val);
6182 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6183 }
6184
6185 /* buf timer set to 1/4 of interrupt timer */
6186 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6187 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6188
6189 if (cmpl_params &
6190 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6191 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6192 val = clamp_t(u16, tmr, 1,
6193 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6194 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6195 req->enables |=
6196 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6197 }
6198
6199 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6200 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6201 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6202 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6203 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6204 req->flags = cpu_to_le16(flags);
6205 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6206 }
6207
6208 /* Caller holds bp->hwrm_cmd_lock */
6209 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6210 struct bnxt_coal *hw_coal)
6211 {
6212 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6213 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6214 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6215 u32 nq_params = coal_cap->nq_params;
6216 u16 tmr;
6217
6218 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6219 return 0;
6220
6221 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6222 -1, -1);
6223 req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6224 req.flags =
6225 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6226
6227 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6228 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6229 req.int_lat_tmr_min = cpu_to_le16(tmr);
6230 req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6231 return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6232 }
6233
6234 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6235 {
6236 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6237 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6238 struct bnxt_coal coal;
6239
6240 /* Tick values in micro seconds.
6241 * 1 coal_buf x bufs_per_record = 1 completion record.
6242 */
6243 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6244
6245 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6246 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6247
6248 if (!bnapi->rx_ring)
6249 return -ENODEV;
6250
6251 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6252 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6253
6254 bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6255
6256 req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6257
6258 return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6259 HWRM_CMD_TIMEOUT);
6260 }
6261
6262 int bnxt_hwrm_set_coal(struct bnxt *bp)
6263 {
6264 int i, rc = 0;
6265 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6266 req_tx = {0}, *req;
6267
6268 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6269 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6270 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6271 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6272
6273 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6274 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6275
6276 mutex_lock(&bp->hwrm_cmd_lock);
6277 for (i = 0; i < bp->cp_nr_rings; i++) {
6278 struct bnxt_napi *bnapi = bp->bnapi[i];
6279 struct bnxt_coal *hw_coal;
6280 u16 ring_id;
6281
6282 req = &req_rx;
6283 if (!bnapi->rx_ring) {
6284 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6285 req = &req_tx;
6286 } else {
6287 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6288 }
6289 req->ring_id = cpu_to_le16(ring_id);
6290
6291 rc = _hwrm_send_message(bp, req, sizeof(*req),
6292 HWRM_CMD_TIMEOUT);
6293 if (rc)
6294 break;
6295
6296 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6297 continue;
6298
6299 if (bnapi->rx_ring && bnapi->tx_ring) {
6300 req = &req_tx;
6301 ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6302 req->ring_id = cpu_to_le16(ring_id);
6303 rc = _hwrm_send_message(bp, req, sizeof(*req),
6304 HWRM_CMD_TIMEOUT);
6305 if (rc)
6306 break;
6307 }
6308 if (bnapi->rx_ring)
6309 hw_coal = &bp->rx_coal;
6310 else
6311 hw_coal = &bp->tx_coal;
6312 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6313 }
6314 mutex_unlock(&bp->hwrm_cmd_lock);
6315 return rc;
6316 }
6317
6318 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6319 {
6320 int rc = 0, i;
6321 struct hwrm_stat_ctx_free_input req = {0};
6322
6323 if (!bp->bnapi)
6324 return 0;
6325
6326 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6327 return 0;
6328
6329 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6330
6331 mutex_lock(&bp->hwrm_cmd_lock);
6332 for (i = 0; i < bp->cp_nr_rings; i++) {
6333 struct bnxt_napi *bnapi = bp->bnapi[i];
6334 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6335
6336 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6337 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6338
6339 rc = _hwrm_send_message(bp, &req, sizeof(req),
6340 HWRM_CMD_TIMEOUT);
6341
6342 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6343 }
6344 }
6345 mutex_unlock(&bp->hwrm_cmd_lock);
6346 return rc;
6347 }
6348
6349 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6350 {
6351 int rc = 0, i;
6352 struct hwrm_stat_ctx_alloc_input req = {0};
6353 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6354
6355 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6356 return 0;
6357
6358 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6359
6360 req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6361 req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6362
6363 mutex_lock(&bp->hwrm_cmd_lock);
6364 for (i = 0; i < bp->cp_nr_rings; i++) {
6365 struct bnxt_napi *bnapi = bp->bnapi[i];
6366 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6367
6368 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
6369
6370 rc = _hwrm_send_message(bp, &req, sizeof(req),
6371 HWRM_CMD_TIMEOUT);
6372 if (rc)
6373 break;
6374
6375 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6376
6377 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6378 }
6379 mutex_unlock(&bp->hwrm_cmd_lock);
6380 return rc;
6381 }
6382
6383 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6384 {
6385 struct hwrm_func_qcfg_input req = {0};
6386 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6387 u16 flags;
6388 int rc;
6389
6390 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6391 req.fid = cpu_to_le16(0xffff);
6392 mutex_lock(&bp->hwrm_cmd_lock);
6393 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6394 if (rc)
6395 goto func_qcfg_exit;
6396
6397 #ifdef CONFIG_BNXT_SRIOV
6398 if (BNXT_VF(bp)) {
6399 struct bnxt_vf_info *vf = &bp->vf;
6400
6401 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6402 } else {
6403 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6404 }
6405 #endif
6406 flags = le16_to_cpu(resp->flags);
6407 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6408 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6409 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6410 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6411 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6412 }
6413 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6414 bp->flags |= BNXT_FLAG_MULTI_HOST;
6415
6416 switch (resp->port_partition_type) {
6417 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6418 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6419 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6420 bp->port_partition_type = resp->port_partition_type;
6421 break;
6422 }
6423 if (bp->hwrm_spec_code < 0x10707 ||
6424 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6425 bp->br_mode = BRIDGE_MODE_VEB;
6426 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6427 bp->br_mode = BRIDGE_MODE_VEPA;
6428 else
6429 bp->br_mode = BRIDGE_MODE_UNDEF;
6430
6431 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6432 if (!bp->max_mtu)
6433 bp->max_mtu = BNXT_MAX_MTU;
6434
6435 func_qcfg_exit:
6436 mutex_unlock(&bp->hwrm_cmd_lock);
6437 return rc;
6438 }
6439
6440 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6441 {
6442 struct hwrm_func_backing_store_qcaps_input req = {0};
6443 struct hwrm_func_backing_store_qcaps_output *resp =
6444 bp->hwrm_cmd_resp_addr;
6445 int rc;
6446
6447 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6448 return 0;
6449
6450 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6451 mutex_lock(&bp->hwrm_cmd_lock);
6452 rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6453 if (!rc) {
6454 struct bnxt_ctx_pg_info *ctx_pg;
6455 struct bnxt_ctx_mem_info *ctx;
6456 int i;
6457
6458 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6459 if (!ctx) {
6460 rc = -ENOMEM;
6461 goto ctx_err;
6462 }
6463 ctx_pg = kzalloc(sizeof(*ctx_pg) * (bp->max_q + 1), GFP_KERNEL);
6464 if (!ctx_pg) {
6465 kfree(ctx);
6466 rc = -ENOMEM;
6467 goto ctx_err;
6468 }
6469 for (i = 0; i < bp->max_q + 1; i++, ctx_pg++)
6470 ctx->tqm_mem[i] = ctx_pg;
6471
6472 bp->ctx = ctx;
6473 ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6474 ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6475 ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6476 ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6477 ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6478 ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6479 ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6480 ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6481 ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6482 ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6483 ctx->vnic_max_vnic_entries =
6484 le16_to_cpu(resp->vnic_max_vnic_entries);
6485 ctx->vnic_max_ring_table_entries =
6486 le16_to_cpu(resp->vnic_max_ring_table_entries);
6487 ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6488 ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6489 ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6490 ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6491 ctx->tqm_min_entries_per_ring =
6492 le32_to_cpu(resp->tqm_min_entries_per_ring);
6493 ctx->tqm_max_entries_per_ring =
6494 le32_to_cpu(resp->tqm_max_entries_per_ring);
6495 ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6496 if (!ctx->tqm_entries_multiple)
6497 ctx->tqm_entries_multiple = 1;
6498 ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6499 ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6500 ctx->mrav_num_entries_units =
6501 le16_to_cpu(resp->mrav_num_entries_units);
6502 ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6503 ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6504 ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6505 } else {
6506 rc = 0;
6507 }
6508 ctx_err:
6509 mutex_unlock(&bp->hwrm_cmd_lock);
6510 return rc;
6511 }
6512
6513 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6514 __le64 *pg_dir)
6515 {
6516 u8 pg_size = 0;
6517
6518 if (BNXT_PAGE_SHIFT == 13)
6519 pg_size = 1 << 4;
6520 else if (BNXT_PAGE_SIZE == 16)
6521 pg_size = 2 << 4;
6522
6523 *pg_attr = pg_size;
6524 if (rmem->depth >= 1) {
6525 if (rmem->depth == 2)
6526 *pg_attr |= 2;
6527 else
6528 *pg_attr |= 1;
6529 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6530 } else {
6531 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6532 }
6533 }
6534
6535 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
6536 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
6537 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
6538 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
6539 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
6540 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6541
6542 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6543 {
6544 struct hwrm_func_backing_store_cfg_input req = {0};
6545 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6546 struct bnxt_ctx_pg_info *ctx_pg;
6547 __le32 *num_entries;
6548 __le64 *pg_dir;
6549 u32 flags = 0;
6550 u8 *pg_attr;
6551 int i, rc;
6552 u32 ena;
6553
6554 if (!ctx)
6555 return 0;
6556
6557 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6558 req.enables = cpu_to_le32(enables);
6559
6560 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6561 ctx_pg = &ctx->qp_mem;
6562 req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6563 req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6564 req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6565 req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6566 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6567 &req.qpc_pg_size_qpc_lvl,
6568 &req.qpc_page_dir);
6569 }
6570 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6571 ctx_pg = &ctx->srq_mem;
6572 req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6573 req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6574 req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6575 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6576 &req.srq_pg_size_srq_lvl,
6577 &req.srq_page_dir);
6578 }
6579 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6580 ctx_pg = &ctx->cq_mem;
6581 req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6582 req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6583 req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6584 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6585 &req.cq_page_dir);
6586 }
6587 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6588 ctx_pg = &ctx->vnic_mem;
6589 req.vnic_num_vnic_entries =
6590 cpu_to_le16(ctx->vnic_max_vnic_entries);
6591 req.vnic_num_ring_table_entries =
6592 cpu_to_le16(ctx->vnic_max_ring_table_entries);
6593 req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6594 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6595 &req.vnic_pg_size_vnic_lvl,
6596 &req.vnic_page_dir);
6597 }
6598 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6599 ctx_pg = &ctx->stat_mem;
6600 req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6601 req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6602 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6603 &req.stat_pg_size_stat_lvl,
6604 &req.stat_page_dir);
6605 }
6606 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6607 ctx_pg = &ctx->mrav_mem;
6608 req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6609 if (ctx->mrav_num_entries_units)
6610 flags |=
6611 FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6612 req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6613 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6614 &req.mrav_pg_size_mrav_lvl,
6615 &req.mrav_page_dir);
6616 }
6617 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6618 ctx_pg = &ctx->tim_mem;
6619 req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6620 req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6621 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6622 &req.tim_pg_size_tim_lvl,
6623 &req.tim_page_dir);
6624 }
6625 for (i = 0, num_entries = &req.tqm_sp_num_entries,
6626 pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6627 pg_dir = &req.tqm_sp_page_dir,
6628 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6629 i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6630 if (!(enables & ena))
6631 continue;
6632
6633 req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6634 ctx_pg = ctx->tqm_mem[i];
6635 *num_entries = cpu_to_le32(ctx_pg->entries);
6636 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6637 }
6638 req.flags = cpu_to_le32(flags);
6639 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6640 return rc;
6641 }
6642
6643 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6644 struct bnxt_ctx_pg_info *ctx_pg)
6645 {
6646 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6647
6648 rmem->page_size = BNXT_PAGE_SIZE;
6649 rmem->pg_arr = ctx_pg->ctx_pg_arr;
6650 rmem->dma_arr = ctx_pg->ctx_dma_arr;
6651 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6652 if (rmem->depth >= 1)
6653 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6654 return bnxt_alloc_ring(bp, rmem);
6655 }
6656
6657 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6658 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6659 u8 depth, bool use_init_val)
6660 {
6661 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6662 int rc;
6663
6664 if (!mem_size)
6665 return 0;
6666
6667 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6668 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6669 ctx_pg->nr_pages = 0;
6670 return -EINVAL;
6671 }
6672 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6673 int nr_tbls, i;
6674
6675 rmem->depth = 2;
6676 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6677 GFP_KERNEL);
6678 if (!ctx_pg->ctx_pg_tbl)
6679 return -ENOMEM;
6680 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6681 rmem->nr_pages = nr_tbls;
6682 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6683 if (rc)
6684 return rc;
6685 for (i = 0; i < nr_tbls; i++) {
6686 struct bnxt_ctx_pg_info *pg_tbl;
6687
6688 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6689 if (!pg_tbl)
6690 return -ENOMEM;
6691 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6692 rmem = &pg_tbl->ring_mem;
6693 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6694 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6695 rmem->depth = 1;
6696 rmem->nr_pages = MAX_CTX_PAGES;
6697 if (use_init_val)
6698 rmem->init_val = bp->ctx->ctx_kind_initializer;
6699 if (i == (nr_tbls - 1)) {
6700 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6701
6702 if (rem)
6703 rmem->nr_pages = rem;
6704 }
6705 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6706 if (rc)
6707 break;
6708 }
6709 } else {
6710 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6711 if (rmem->nr_pages > 1 || depth)
6712 rmem->depth = 1;
6713 if (use_init_val)
6714 rmem->init_val = bp->ctx->ctx_kind_initializer;
6715 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6716 }
6717 return rc;
6718 }
6719
6720 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6721 struct bnxt_ctx_pg_info *ctx_pg)
6722 {
6723 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6724
6725 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6726 ctx_pg->ctx_pg_tbl) {
6727 int i, nr_tbls = rmem->nr_pages;
6728
6729 for (i = 0; i < nr_tbls; i++) {
6730 struct bnxt_ctx_pg_info *pg_tbl;
6731 struct bnxt_ring_mem_info *rmem2;
6732
6733 pg_tbl = ctx_pg->ctx_pg_tbl[i];
6734 if (!pg_tbl)
6735 continue;
6736 rmem2 = &pg_tbl->ring_mem;
6737 bnxt_free_ring(bp, rmem2);
6738 ctx_pg->ctx_pg_arr[i] = NULL;
6739 kfree(pg_tbl);
6740 ctx_pg->ctx_pg_tbl[i] = NULL;
6741 }
6742 kfree(ctx_pg->ctx_pg_tbl);
6743 ctx_pg->ctx_pg_tbl = NULL;
6744 }
6745 bnxt_free_ring(bp, rmem);
6746 ctx_pg->nr_pages = 0;
6747 }
6748
6749 static void bnxt_free_ctx_mem(struct bnxt *bp)
6750 {
6751 struct bnxt_ctx_mem_info *ctx = bp->ctx;
6752 int i;
6753
6754 if (!ctx)
6755 return;
6756
6757 if (ctx->tqm_mem[0]) {
6758 for (i = 0; i < bp->max_q + 1; i++)
6759 bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
6760 kfree(ctx->tqm_mem[0]);
6761 ctx->tqm_mem[0] = NULL;
6762 }
6763
6764 bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
6765 bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
6766 bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
6767 bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
6768 bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
6769 bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
6770 bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
6771 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
6772 }
6773
6774 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
6775 {
6776 struct bnxt_ctx_pg_info *ctx_pg;
6777 struct bnxt_ctx_mem_info *ctx;
6778 u32 mem_size, ena, entries;
6779 u32 num_mr, num_ah;
6780 u32 extra_srqs = 0;
6781 u32 extra_qps = 0;
6782 u8 pg_lvl = 1;
6783 int i, rc;
6784
6785 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
6786 if (rc) {
6787 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
6788 rc);
6789 return rc;
6790 }
6791 ctx = bp->ctx;
6792 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
6793 return 0;
6794
6795 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
6796 pg_lvl = 2;
6797 extra_qps = 65536;
6798 extra_srqs = 8192;
6799 }
6800
6801 ctx_pg = &ctx->qp_mem;
6802 ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
6803 extra_qps;
6804 mem_size = ctx->qp_entry_size * ctx_pg->entries;
6805 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
6806 if (rc)
6807 return rc;
6808
6809 ctx_pg = &ctx->srq_mem;
6810 ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
6811 mem_size = ctx->srq_entry_size * ctx_pg->entries;
6812 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
6813 if (rc)
6814 return rc;
6815
6816 ctx_pg = &ctx->cq_mem;
6817 ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
6818 mem_size = ctx->cq_entry_size * ctx_pg->entries;
6819 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
6820 if (rc)
6821 return rc;
6822
6823 ctx_pg = &ctx->vnic_mem;
6824 ctx_pg->entries = ctx->vnic_max_vnic_entries +
6825 ctx->vnic_max_ring_table_entries;
6826 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
6827 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
6828 if (rc)
6829 return rc;
6830
6831 ctx_pg = &ctx->stat_mem;
6832 ctx_pg->entries = ctx->stat_max_entries;
6833 mem_size = ctx->stat_entry_size * ctx_pg->entries;
6834 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
6835 if (rc)
6836 return rc;
6837
6838 ena = 0;
6839 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
6840 goto skip_rdma;
6841
6842 ctx_pg = &ctx->mrav_mem;
6843 /* 128K extra is needed to accommodate static AH context
6844 * allocation by f/w.
6845 */
6846 num_mr = 1024 * 256;
6847 num_ah = 1024 * 128;
6848 ctx_pg->entries = num_mr + num_ah;
6849 mem_size = ctx->mrav_entry_size * ctx_pg->entries;
6850 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
6851 if (rc)
6852 return rc;
6853 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
6854 if (ctx->mrav_num_entries_units)
6855 ctx_pg->entries =
6856 ((num_mr / ctx->mrav_num_entries_units) << 16) |
6857 (num_ah / ctx->mrav_num_entries_units);
6858
6859 ctx_pg = &ctx->tim_mem;
6860 ctx_pg->entries = ctx->qp_mem.entries;
6861 mem_size = ctx->tim_entry_size * ctx_pg->entries;
6862 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
6863 if (rc)
6864 return rc;
6865 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
6866
6867 skip_rdma:
6868 entries = ctx->qp_max_l2_entries + extra_qps;
6869 entries = roundup(entries, ctx->tqm_entries_multiple);
6870 entries = clamp_t(u32, entries, ctx->tqm_min_entries_per_ring,
6871 ctx->tqm_max_entries_per_ring);
6872 for (i = 0; i < bp->max_q + 1; i++) {
6873 ctx_pg = ctx->tqm_mem[i];
6874 ctx_pg->entries = entries;
6875 mem_size = ctx->tqm_entry_size * entries;
6876 rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
6877 if (rc)
6878 return rc;
6879 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
6880 }
6881 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
6882 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
6883 if (rc)
6884 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
6885 rc);
6886 else
6887 ctx->flags |= BNXT_CTX_FLAG_INITED;
6888
6889 return 0;
6890 }
6891
6892 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
6893 {
6894 struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6895 struct hwrm_func_resource_qcaps_input req = {0};
6896 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6897 int rc;
6898
6899 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
6900 req.fid = cpu_to_le16(0xffff);
6901
6902 mutex_lock(&bp->hwrm_cmd_lock);
6903 rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
6904 HWRM_CMD_TIMEOUT);
6905 if (rc)
6906 goto hwrm_func_resc_qcaps_exit;
6907
6908 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
6909 if (!all)
6910 goto hwrm_func_resc_qcaps_exit;
6911
6912 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
6913 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6914 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
6915 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6916 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
6917 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6918 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
6919 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6920 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
6921 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
6922 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
6923 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6924 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
6925 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6926 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
6927 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6928
6929 if (bp->flags & BNXT_FLAG_CHIP_P5) {
6930 u16 max_msix = le16_to_cpu(resp->max_msix);
6931
6932 hw_resc->max_nqs = max_msix;
6933 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
6934 }
6935
6936 if (BNXT_PF(bp)) {
6937 struct bnxt_pf_info *pf = &bp->pf;
6938
6939 pf->vf_resv_strategy =
6940 le16_to_cpu(resp->vf_reservation_strategy);
6941 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
6942 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
6943 }
6944 hwrm_func_resc_qcaps_exit:
6945 mutex_unlock(&bp->hwrm_cmd_lock);
6946 return rc;
6947 }
6948
6949 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
6950 {
6951 int rc = 0;
6952 struct hwrm_func_qcaps_input req = {0};
6953 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6954 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6955 u32 flags;
6956
6957 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
6958 req.fid = cpu_to_le16(0xffff);
6959
6960 mutex_lock(&bp->hwrm_cmd_lock);
6961 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6962 if (rc)
6963 goto hwrm_func_qcaps_exit;
6964
6965 flags = le32_to_cpu(resp->flags);
6966 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
6967 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
6968 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
6969 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
6970 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
6971 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
6972 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
6973 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
6974 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
6975 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
6976 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
6977 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
6978 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
6979 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
6980
6981 bp->tx_push_thresh = 0;
6982 if (flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED)
6983 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
6984
6985 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
6986 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
6987 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
6988 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
6989 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
6990 if (!hw_resc->max_hw_ring_grps)
6991 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
6992 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
6993 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
6994 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
6995
6996 if (BNXT_PF(bp)) {
6997 struct bnxt_pf_info *pf = &bp->pf;
6998
6999 pf->fw_fid = le16_to_cpu(resp->fid);
7000 pf->port_id = le16_to_cpu(resp->port_id);
7001 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7002 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7003 pf->max_vfs = le16_to_cpu(resp->max_vfs);
7004 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7005 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7006 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7007 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7008 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7009 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7010 bp->flags &= ~BNXT_FLAG_WOL_CAP;
7011 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7012 bp->flags |= BNXT_FLAG_WOL_CAP;
7013 } else {
7014 #ifdef CONFIG_BNXT_SRIOV
7015 struct bnxt_vf_info *vf = &bp->vf;
7016
7017 vf->fw_fid = le16_to_cpu(resp->fid);
7018 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7019 #endif
7020 }
7021
7022 hwrm_func_qcaps_exit:
7023 mutex_unlock(&bp->hwrm_cmd_lock);
7024 return rc;
7025 }
7026
7027 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7028
7029 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7030 {
7031 int rc;
7032
7033 rc = __bnxt_hwrm_func_qcaps(bp);
7034 if (rc)
7035 return rc;
7036 rc = bnxt_hwrm_queue_qportcfg(bp);
7037 if (rc) {
7038 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7039 return rc;
7040 }
7041 if (bp->hwrm_spec_code >= 0x10803) {
7042 rc = bnxt_alloc_ctx_mem(bp);
7043 if (rc)
7044 return rc;
7045 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7046 if (!rc)
7047 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7048 }
7049 return 0;
7050 }
7051
7052 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7053 {
7054 struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7055 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7056 int rc = 0;
7057 u32 flags;
7058
7059 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7060 return 0;
7061
7062 resp = bp->hwrm_cmd_resp_addr;
7063 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7064
7065 mutex_lock(&bp->hwrm_cmd_lock);
7066 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7067 if (rc)
7068 goto hwrm_cfa_adv_qcaps_exit;
7069
7070 flags = le32_to_cpu(resp->flags);
7071 if (flags &
7072 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7073 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7074
7075 hwrm_cfa_adv_qcaps_exit:
7076 mutex_unlock(&bp->hwrm_cmd_lock);
7077 return rc;
7078 }
7079
7080 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7081 {
7082 struct bnxt_fw_health *fw_health = bp->fw_health;
7083 u32 reg_base = 0xffffffff;
7084 int i;
7085
7086 /* Only pre-map the monitoring GRC registers using window 3 */
7087 for (i = 0; i < 4; i++) {
7088 u32 reg = fw_health->regs[i];
7089
7090 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7091 continue;
7092 if (reg_base == 0xffffffff)
7093 reg_base = reg & BNXT_GRC_BASE_MASK;
7094 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7095 return -ERANGE;
7096 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE +
7097 (reg & BNXT_GRC_OFFSET_MASK);
7098 }
7099 if (reg_base == 0xffffffff)
7100 return 0;
7101
7102 writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7103 BNXT_FW_HEALTH_WIN_MAP_OFF);
7104 return 0;
7105 }
7106
7107 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7108 {
7109 struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7110 struct bnxt_fw_health *fw_health = bp->fw_health;
7111 struct hwrm_error_recovery_qcfg_input req = {0};
7112 int rc, i;
7113
7114 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7115 return 0;
7116
7117 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7118 mutex_lock(&bp->hwrm_cmd_lock);
7119 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7120 if (rc)
7121 goto err_recovery_out;
7122 fw_health->flags = le32_to_cpu(resp->flags);
7123 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7124 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7125 rc = -EINVAL;
7126 goto err_recovery_out;
7127 }
7128 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7129 fw_health->master_func_wait_dsecs =
7130 le32_to_cpu(resp->master_func_wait_period);
7131 fw_health->normal_func_wait_dsecs =
7132 le32_to_cpu(resp->normal_func_wait_period);
7133 fw_health->post_reset_wait_dsecs =
7134 le32_to_cpu(resp->master_func_wait_period_after_reset);
7135 fw_health->post_reset_max_wait_dsecs =
7136 le32_to_cpu(resp->max_bailout_time_after_reset);
7137 fw_health->regs[BNXT_FW_HEALTH_REG] =
7138 le32_to_cpu(resp->fw_health_status_reg);
7139 fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7140 le32_to_cpu(resp->fw_heartbeat_reg);
7141 fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7142 le32_to_cpu(resp->fw_reset_cnt_reg);
7143 fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7144 le32_to_cpu(resp->reset_inprogress_reg);
7145 fw_health->fw_reset_inprog_reg_mask =
7146 le32_to_cpu(resp->reset_inprogress_reg_mask);
7147 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7148 if (fw_health->fw_reset_seq_cnt >= 16) {
7149 rc = -EINVAL;
7150 goto err_recovery_out;
7151 }
7152 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7153 fw_health->fw_reset_seq_regs[i] =
7154 le32_to_cpu(resp->reset_reg[i]);
7155 fw_health->fw_reset_seq_vals[i] =
7156 le32_to_cpu(resp->reset_reg_val[i]);
7157 fw_health->fw_reset_seq_delay_msec[i] =
7158 resp->delay_after_reset[i];
7159 }
7160 err_recovery_out:
7161 mutex_unlock(&bp->hwrm_cmd_lock);
7162 if (!rc)
7163 rc = bnxt_map_fw_health_regs(bp);
7164 if (rc)
7165 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7166 return rc;
7167 }
7168
7169 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7170 {
7171 struct hwrm_func_reset_input req = {0};
7172
7173 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7174 req.enables = 0;
7175
7176 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7177 }
7178
7179 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7180 {
7181 int rc = 0;
7182 struct hwrm_queue_qportcfg_input req = {0};
7183 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7184 u8 i, j, *qptr;
7185 bool no_rdma;
7186
7187 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7188
7189 mutex_lock(&bp->hwrm_cmd_lock);
7190 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7191 if (rc)
7192 goto qportcfg_exit;
7193
7194 if (!resp->max_configurable_queues) {
7195 rc = -EINVAL;
7196 goto qportcfg_exit;
7197 }
7198 bp->max_tc = resp->max_configurable_queues;
7199 bp->max_lltc = resp->max_configurable_lossless_queues;
7200 if (bp->max_tc > BNXT_MAX_QUEUE)
7201 bp->max_tc = BNXT_MAX_QUEUE;
7202
7203 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7204 qptr = &resp->queue_id0;
7205 for (i = 0, j = 0; i < bp->max_tc; i++) {
7206 bp->q_info[j].queue_id = *qptr;
7207 bp->q_ids[i] = *qptr++;
7208 bp->q_info[j].queue_profile = *qptr++;
7209 bp->tc_to_qidx[j] = j;
7210 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7211 (no_rdma && BNXT_PF(bp)))
7212 j++;
7213 }
7214 bp->max_q = bp->max_tc;
7215 bp->max_tc = max_t(u8, j, 1);
7216
7217 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7218 bp->max_tc = 1;
7219
7220 if (bp->max_lltc > bp->max_tc)
7221 bp->max_lltc = bp->max_tc;
7222
7223 qportcfg_exit:
7224 mutex_unlock(&bp->hwrm_cmd_lock);
7225 return rc;
7226 }
7227
7228 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7229 {
7230 struct hwrm_ver_get_input req = {0};
7231 int rc;
7232
7233 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7234 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7235 req.hwrm_intf_min = HWRM_VERSION_MINOR;
7236 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7237
7238 rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7239 silent);
7240 return rc;
7241 }
7242
7243 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7244 {
7245 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7246 u32 dev_caps_cfg;
7247 int rc;
7248
7249 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7250 mutex_lock(&bp->hwrm_cmd_lock);
7251 rc = __bnxt_hwrm_ver_get(bp, false);
7252 if (rc)
7253 goto hwrm_ver_get_exit;
7254
7255 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7256
7257 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7258 resp->hwrm_intf_min_8b << 8 |
7259 resp->hwrm_intf_upd_8b;
7260 if (resp->hwrm_intf_maj_8b < 1) {
7261 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7262 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7263 resp->hwrm_intf_upd_8b);
7264 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7265 }
7266 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d.%d",
7267 resp->hwrm_fw_maj_8b, resp->hwrm_fw_min_8b,
7268 resp->hwrm_fw_bld_8b, resp->hwrm_fw_rsvd_8b);
7269
7270 if (strlen(resp->active_pkg_name)) {
7271 int fw_ver_len = strlen(bp->fw_ver_str);
7272
7273 snprintf(bp->fw_ver_str + fw_ver_len,
7274 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7275 resp->active_pkg_name);
7276 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7277 }
7278
7279 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7280 if (!bp->hwrm_cmd_timeout)
7281 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7282
7283 if (resp->hwrm_intf_maj_8b >= 1) {
7284 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7285 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7286 }
7287 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7288 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7289
7290 bp->chip_num = le16_to_cpu(resp->chip_num);
7291 bp->chip_rev = resp->chip_rev;
7292 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7293 !resp->chip_metal)
7294 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7295
7296 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7297 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7298 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7299 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7300
7301 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7302 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7303
7304 if (dev_caps_cfg &
7305 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7306 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7307
7308 if (dev_caps_cfg &
7309 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7310 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7311
7312 if (dev_caps_cfg &
7313 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7314 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7315
7316 hwrm_ver_get_exit:
7317 mutex_unlock(&bp->hwrm_cmd_lock);
7318 return rc;
7319 }
7320
7321 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7322 {
7323 struct hwrm_fw_set_time_input req = {0};
7324 struct tm tm;
7325 time64_t now = ktime_get_real_seconds();
7326
7327 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7328 bp->hwrm_spec_code < 0x10400)
7329 return -EOPNOTSUPP;
7330
7331 time64_to_tm(now, 0, &tm);
7332 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7333 req.year = cpu_to_le16(1900 + tm.tm_year);
7334 req.month = 1 + tm.tm_mon;
7335 req.day = tm.tm_mday;
7336 req.hour = tm.tm_hour;
7337 req.minute = tm.tm_min;
7338 req.second = tm.tm_sec;
7339 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7340 }
7341
7342 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
7343 {
7344 int rc;
7345 struct bnxt_pf_info *pf = &bp->pf;
7346 struct hwrm_port_qstats_input req = {0};
7347
7348 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7349 return 0;
7350
7351 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7352 req.port_id = cpu_to_le16(pf->port_id);
7353 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
7354 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
7355 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7356 return rc;
7357 }
7358
7359 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp)
7360 {
7361 struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7362 struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7363 struct hwrm_port_qstats_ext_input req = {0};
7364 struct bnxt_pf_info *pf = &bp->pf;
7365 u32 tx_stat_size;
7366 int rc;
7367
7368 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7369 return 0;
7370
7371 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7372 req.port_id = cpu_to_le16(pf->port_id);
7373 req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7374 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_ext_map);
7375 tx_stat_size = bp->hw_tx_port_stats_ext ?
7376 sizeof(*bp->hw_tx_port_stats_ext) : 0;
7377 req.tx_stat_size = cpu_to_le16(tx_stat_size);
7378 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_ext_map);
7379 mutex_lock(&bp->hwrm_cmd_lock);
7380 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7381 if (!rc) {
7382 bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7383 bp->fw_tx_stats_ext_size = tx_stat_size ?
7384 le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7385 } else {
7386 bp->fw_rx_stats_ext_size = 0;
7387 bp->fw_tx_stats_ext_size = 0;
7388 }
7389 if (bp->fw_tx_stats_ext_size <=
7390 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7391 mutex_unlock(&bp->hwrm_cmd_lock);
7392 bp->pri2cos_valid = 0;
7393 return rc;
7394 }
7395
7396 bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7397 req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7398
7399 rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7400 if (!rc) {
7401 struct hwrm_queue_pri2cos_qcfg_output *resp2;
7402 u8 *pri2cos;
7403 int i, j;
7404
7405 resp2 = bp->hwrm_cmd_resp_addr;
7406 pri2cos = &resp2->pri0_cos_queue_id;
7407 for (i = 0; i < 8; i++) {
7408 u8 queue_id = pri2cos[i];
7409
7410 for (j = 0; j < bp->max_q; j++) {
7411 if (bp->q_ids[j] == queue_id)
7412 bp->pri2cos[i] = j;
7413 }
7414 }
7415 bp->pri2cos_valid = 1;
7416 }
7417 mutex_unlock(&bp->hwrm_cmd_lock);
7418 return rc;
7419 }
7420
7421 static int bnxt_hwrm_pcie_qstats(struct bnxt *bp)
7422 {
7423 struct hwrm_pcie_qstats_input req = {0};
7424
7425 if (!(bp->flags & BNXT_FLAG_PCIE_STATS))
7426 return 0;
7427
7428 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PCIE_QSTATS, -1, -1);
7429 req.pcie_stat_size = cpu_to_le16(sizeof(struct pcie_ctx_hw_stats));
7430 req.pcie_stat_host_addr = cpu_to_le64(bp->hw_pcie_stats_map);
7431 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7432 }
7433
7434 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7435 {
7436 if (bp->vxlan_port_cnt) {
7437 bnxt_hwrm_tunnel_dst_port_free(
7438 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7439 }
7440 bp->vxlan_port_cnt = 0;
7441 if (bp->nge_port_cnt) {
7442 bnxt_hwrm_tunnel_dst_port_free(
7443 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7444 }
7445 bp->nge_port_cnt = 0;
7446 }
7447
7448 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7449 {
7450 int rc, i;
7451 u32 tpa_flags = 0;
7452
7453 if (set_tpa)
7454 tpa_flags = bp->flags & BNXT_FLAG_TPA;
7455 else if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
7456 return 0;
7457 for (i = 0; i < bp->nr_vnics; i++) {
7458 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7459 if (rc) {
7460 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7461 i, rc);
7462 return rc;
7463 }
7464 }
7465 return 0;
7466 }
7467
7468 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7469 {
7470 int i;
7471
7472 for (i = 0; i < bp->nr_vnics; i++)
7473 bnxt_hwrm_vnic_set_rss(bp, i, false);
7474 }
7475
7476 static void bnxt_clear_vnic(struct bnxt *bp)
7477 {
7478 if (!bp->vnic_info)
7479 return;
7480
7481 bnxt_hwrm_clear_vnic_filter(bp);
7482 if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7483 /* clear all RSS setting before free vnic ctx */
7484 bnxt_hwrm_clear_vnic_rss(bp);
7485 bnxt_hwrm_vnic_ctx_free(bp);
7486 }
7487 /* before free the vnic, undo the vnic tpa settings */
7488 if (bp->flags & BNXT_FLAG_TPA)
7489 bnxt_set_tpa(bp, false);
7490 bnxt_hwrm_vnic_free(bp);
7491 if (bp->flags & BNXT_FLAG_CHIP_P5)
7492 bnxt_hwrm_vnic_ctx_free(bp);
7493 }
7494
7495 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7496 bool irq_re_init)
7497 {
7498 bnxt_clear_vnic(bp);
7499 bnxt_hwrm_ring_free(bp, close_path);
7500 bnxt_hwrm_ring_grp_free(bp);
7501 if (irq_re_init) {
7502 bnxt_hwrm_stat_ctx_free(bp);
7503 bnxt_hwrm_free_tunnel_ports(bp);
7504 }
7505 }
7506
7507 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7508 {
7509 struct hwrm_func_cfg_input req = {0};
7510 int rc;
7511
7512 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7513 req.fid = cpu_to_le16(0xffff);
7514 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7515 if (br_mode == BRIDGE_MODE_VEB)
7516 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7517 else if (br_mode == BRIDGE_MODE_VEPA)
7518 req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7519 else
7520 return -EINVAL;
7521 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7522 return rc;
7523 }
7524
7525 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7526 {
7527 struct hwrm_func_cfg_input req = {0};
7528 int rc;
7529
7530 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7531 return 0;
7532
7533 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7534 req.fid = cpu_to_le16(0xffff);
7535 req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
7536 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
7537 if (size == 128)
7538 req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
7539
7540 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7541 return rc;
7542 }
7543
7544 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7545 {
7546 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
7547 int rc;
7548
7549 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7550 goto skip_rss_ctx;
7551
7552 /* allocate context for vnic */
7553 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
7554 if (rc) {
7555 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7556 vnic_id, rc);
7557 goto vnic_setup_err;
7558 }
7559 bp->rsscos_nr_ctxs++;
7560
7561 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7562 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7563 if (rc) {
7564 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7565 vnic_id, rc);
7566 goto vnic_setup_err;
7567 }
7568 bp->rsscos_nr_ctxs++;
7569 }
7570
7571 skip_rss_ctx:
7572 /* configure default vnic, ring grp */
7573 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7574 if (rc) {
7575 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7576 vnic_id, rc);
7577 goto vnic_setup_err;
7578 }
7579
7580 /* Enable RSS hashing on vnic */
7581 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7582 if (rc) {
7583 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7584 vnic_id, rc);
7585 goto vnic_setup_err;
7586 }
7587
7588 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7589 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7590 if (rc) {
7591 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7592 vnic_id, rc);
7593 }
7594 }
7595
7596 vnic_setup_err:
7597 return rc;
7598 }
7599
7600 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7601 {
7602 int rc, i, nr_ctxs;
7603
7604 nr_ctxs = DIV_ROUND_UP(bp->rx_nr_rings, 64);
7605 for (i = 0; i < nr_ctxs; i++) {
7606 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7607 if (rc) {
7608 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7609 vnic_id, i, rc);
7610 break;
7611 }
7612 bp->rsscos_nr_ctxs++;
7613 }
7614 if (i < nr_ctxs)
7615 return -ENOMEM;
7616
7617 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7618 if (rc) {
7619 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7620 vnic_id, rc);
7621 return rc;
7622 }
7623 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7624 if (rc) {
7625 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7626 vnic_id, rc);
7627 return rc;
7628 }
7629 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7630 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7631 if (rc) {
7632 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7633 vnic_id, rc);
7634 }
7635 }
7636 return rc;
7637 }
7638
7639 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7640 {
7641 if (bp->flags & BNXT_FLAG_CHIP_P5)
7642 return __bnxt_setup_vnic_p5(bp, vnic_id);
7643 else
7644 return __bnxt_setup_vnic(bp, vnic_id);
7645 }
7646
7647 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
7648 {
7649 #ifdef CONFIG_RFS_ACCEL
7650 int i, rc = 0;
7651
7652 if (bp->flags & BNXT_FLAG_CHIP_P5)
7653 return 0;
7654
7655 for (i = 0; i < bp->rx_nr_rings; i++) {
7656 struct bnxt_vnic_info *vnic;
7657 u16 vnic_id = i + 1;
7658 u16 ring_id = i;
7659
7660 if (vnic_id >= bp->nr_vnics)
7661 break;
7662
7663 vnic = &bp->vnic_info[vnic_id];
7664 vnic->flags |= BNXT_VNIC_RFS_FLAG;
7665 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
7666 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
7667 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
7668 if (rc) {
7669 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7670 vnic_id, rc);
7671 break;
7672 }
7673 rc = bnxt_setup_vnic(bp, vnic_id);
7674 if (rc)
7675 break;
7676 }
7677 return rc;
7678 #else
7679 return 0;
7680 #endif
7681 }
7682
7683 /* Allow PF and VF with default VLAN to be in promiscuous mode */
7684 static bool bnxt_promisc_ok(struct bnxt *bp)
7685 {
7686 #ifdef CONFIG_BNXT_SRIOV
7687 if (BNXT_VF(bp) && !bp->vf.vlan)
7688 return false;
7689 #endif
7690 return true;
7691 }
7692
7693 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
7694 {
7695 unsigned int rc = 0;
7696
7697 rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
7698 if (rc) {
7699 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7700 rc);
7701 return rc;
7702 }
7703
7704 rc = bnxt_hwrm_vnic_cfg(bp, 1);
7705 if (rc) {
7706 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
7707 rc);
7708 return rc;
7709 }
7710 return rc;
7711 }
7712
7713 static int bnxt_cfg_rx_mode(struct bnxt *);
7714 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
7715
7716 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
7717 {
7718 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
7719 int rc = 0;
7720 unsigned int rx_nr_rings = bp->rx_nr_rings;
7721
7722 if (irq_re_init) {
7723 rc = bnxt_hwrm_stat_ctx_alloc(bp);
7724 if (rc) {
7725 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
7726 rc);
7727 goto err_out;
7728 }
7729 }
7730
7731 rc = bnxt_hwrm_ring_alloc(bp);
7732 if (rc) {
7733 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
7734 goto err_out;
7735 }
7736
7737 rc = bnxt_hwrm_ring_grp_alloc(bp);
7738 if (rc) {
7739 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
7740 goto err_out;
7741 }
7742
7743 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
7744 rx_nr_rings--;
7745
7746 /* default vnic 0 */
7747 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
7748 if (rc) {
7749 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
7750 goto err_out;
7751 }
7752
7753 rc = bnxt_setup_vnic(bp, 0);
7754 if (rc)
7755 goto err_out;
7756
7757 if (bp->flags & BNXT_FLAG_RFS) {
7758 rc = bnxt_alloc_rfs_vnics(bp);
7759 if (rc)
7760 goto err_out;
7761 }
7762
7763 if (bp->flags & BNXT_FLAG_TPA) {
7764 rc = bnxt_set_tpa(bp, true);
7765 if (rc)
7766 goto err_out;
7767 }
7768
7769 if (BNXT_VF(bp))
7770 bnxt_update_vf_mac(bp);
7771
7772 /* Filter for default vnic 0 */
7773 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
7774 if (rc) {
7775 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
7776 goto err_out;
7777 }
7778 vnic->uc_filter_count = 1;
7779
7780 vnic->rx_mask = 0;
7781 if (bp->dev->flags & IFF_BROADCAST)
7782 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
7783
7784 if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
7785 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
7786
7787 if (bp->dev->flags & IFF_ALLMULTI) {
7788 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
7789 vnic->mc_list_count = 0;
7790 } else {
7791 u32 mask = 0;
7792
7793 bnxt_mc_list_updated(bp, &mask);
7794 vnic->rx_mask |= mask;
7795 }
7796
7797 rc = bnxt_cfg_rx_mode(bp);
7798 if (rc)
7799 goto err_out;
7800
7801 rc = bnxt_hwrm_set_coal(bp);
7802 if (rc)
7803 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
7804 rc);
7805
7806 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7807 rc = bnxt_setup_nitroa0_vnic(bp);
7808 if (rc)
7809 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
7810 rc);
7811 }
7812
7813 if (BNXT_VF(bp)) {
7814 bnxt_hwrm_func_qcfg(bp);
7815 netdev_update_features(bp->dev);
7816 }
7817
7818 return 0;
7819
7820 err_out:
7821 bnxt_hwrm_resource_free(bp, 0, true);
7822
7823 return rc;
7824 }
7825
7826 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
7827 {
7828 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
7829 return 0;
7830 }
7831
7832 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
7833 {
7834 bnxt_init_cp_rings(bp);
7835 bnxt_init_rx_rings(bp);
7836 bnxt_init_tx_rings(bp);
7837 bnxt_init_ring_grps(bp, irq_re_init);
7838 bnxt_init_vnics(bp);
7839
7840 return bnxt_init_chip(bp, irq_re_init);
7841 }
7842
7843 static int bnxt_set_real_num_queues(struct bnxt *bp)
7844 {
7845 int rc;
7846 struct net_device *dev = bp->dev;
7847
7848 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
7849 bp->tx_nr_rings_xdp);
7850 if (rc)
7851 return rc;
7852
7853 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
7854 if (rc)
7855 return rc;
7856
7857 #ifdef CONFIG_RFS_ACCEL
7858 if (bp->flags & BNXT_FLAG_RFS)
7859 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
7860 #endif
7861
7862 return rc;
7863 }
7864
7865 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7866 bool shared)
7867 {
7868 int _rx = *rx, _tx = *tx;
7869
7870 if (shared) {
7871 *rx = min_t(int, _rx, max);
7872 *tx = min_t(int, _tx, max);
7873 } else {
7874 if (max < 2)
7875 return -ENOMEM;
7876
7877 while (_rx + _tx > max) {
7878 if (_rx > _tx && _rx > 1)
7879 _rx--;
7880 else if (_tx > 1)
7881 _tx--;
7882 }
7883 *rx = _rx;
7884 *tx = _tx;
7885 }
7886 return 0;
7887 }
7888
7889 static void bnxt_setup_msix(struct bnxt *bp)
7890 {
7891 const int len = sizeof(bp->irq_tbl[0].name);
7892 struct net_device *dev = bp->dev;
7893 int tcs, i;
7894
7895 tcs = netdev_get_num_tc(dev);
7896 if (tcs > 1) {
7897 int i, off, count;
7898
7899 for (i = 0; i < tcs; i++) {
7900 count = bp->tx_nr_rings_per_tc;
7901 off = i * count;
7902 netdev_set_tc_queue(dev, i, count, off);
7903 }
7904 }
7905
7906 for (i = 0; i < bp->cp_nr_rings; i++) {
7907 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
7908 char *attr;
7909
7910 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7911 attr = "TxRx";
7912 else if (i < bp->rx_nr_rings)
7913 attr = "rx";
7914 else
7915 attr = "tx";
7916
7917 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
7918 attr, i);
7919 bp->irq_tbl[map_idx].handler = bnxt_msix;
7920 }
7921 }
7922
7923 static void bnxt_setup_inta(struct bnxt *bp)
7924 {
7925 const int len = sizeof(bp->irq_tbl[0].name);
7926
7927 if (netdev_get_num_tc(bp->dev))
7928 netdev_reset_tc(bp->dev);
7929
7930 snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
7931 0);
7932 bp->irq_tbl[0].handler = bnxt_inta;
7933 }
7934
7935 static int bnxt_setup_int_mode(struct bnxt *bp)
7936 {
7937 int rc;
7938
7939 if (bp->flags & BNXT_FLAG_USING_MSIX)
7940 bnxt_setup_msix(bp);
7941 else
7942 bnxt_setup_inta(bp);
7943
7944 rc = bnxt_set_real_num_queues(bp);
7945 return rc;
7946 }
7947
7948 #ifdef CONFIG_RFS_ACCEL
7949 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
7950 {
7951 return bp->hw_resc.max_rsscos_ctxs;
7952 }
7953
7954 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
7955 {
7956 return bp->hw_resc.max_vnics;
7957 }
7958 #endif
7959
7960 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
7961 {
7962 return bp->hw_resc.max_stat_ctxs;
7963 }
7964
7965 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
7966 {
7967 return bp->hw_resc.max_cp_rings;
7968 }
7969
7970 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
7971 {
7972 unsigned int cp = bp->hw_resc.max_cp_rings;
7973
7974 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
7975 cp -= bnxt_get_ulp_msix_num(bp);
7976
7977 return cp;
7978 }
7979
7980 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
7981 {
7982 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7983
7984 if (bp->flags & BNXT_FLAG_CHIP_P5)
7985 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
7986
7987 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
7988 }
7989
7990 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
7991 {
7992 bp->hw_resc.max_irqs = max_irqs;
7993 }
7994
7995 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
7996 {
7997 unsigned int cp;
7998
7999 cp = bnxt_get_max_func_cp_rings_for_en(bp);
8000 if (bp->flags & BNXT_FLAG_CHIP_P5)
8001 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8002 else
8003 return cp - bp->cp_nr_rings;
8004 }
8005
8006 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8007 {
8008 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8009 }
8010
8011 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8012 {
8013 int max_cp = bnxt_get_max_func_cp_rings(bp);
8014 int max_irq = bnxt_get_max_func_irqs(bp);
8015 int total_req = bp->cp_nr_rings + num;
8016 int max_idx, avail_msix;
8017
8018 max_idx = bp->total_irqs;
8019 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8020 max_idx = min_t(int, bp->total_irqs, max_cp);
8021 avail_msix = max_idx - bp->cp_nr_rings;
8022 if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8023 return avail_msix;
8024
8025 if (max_irq < total_req) {
8026 num = max_irq - bp->cp_nr_rings;
8027 if (num <= 0)
8028 return 0;
8029 }
8030 return num;
8031 }
8032
8033 static int bnxt_get_num_msix(struct bnxt *bp)
8034 {
8035 if (!BNXT_NEW_RM(bp))
8036 return bnxt_get_max_func_irqs(bp);
8037
8038 return bnxt_nq_rings_in_use(bp);
8039 }
8040
8041 static int bnxt_init_msix(struct bnxt *bp)
8042 {
8043 int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8044 struct msix_entry *msix_ent;
8045
8046 total_vecs = bnxt_get_num_msix(bp);
8047 max = bnxt_get_max_func_irqs(bp);
8048 if (total_vecs > max)
8049 total_vecs = max;
8050
8051 if (!total_vecs)
8052 return 0;
8053
8054 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8055 if (!msix_ent)
8056 return -ENOMEM;
8057
8058 for (i = 0; i < total_vecs; i++) {
8059 msix_ent[i].entry = i;
8060 msix_ent[i].vector = 0;
8061 }
8062
8063 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8064 min = 2;
8065
8066 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8067 ulp_msix = bnxt_get_ulp_msix_num(bp);
8068 if (total_vecs < 0 || total_vecs < ulp_msix) {
8069 rc = -ENODEV;
8070 goto msix_setup_exit;
8071 }
8072
8073 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8074 if (bp->irq_tbl) {
8075 for (i = 0; i < total_vecs; i++)
8076 bp->irq_tbl[i].vector = msix_ent[i].vector;
8077
8078 bp->total_irqs = total_vecs;
8079 /* Trim rings based upon num of vectors allocated */
8080 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8081 total_vecs - ulp_msix, min == 1);
8082 if (rc)
8083 goto msix_setup_exit;
8084
8085 bp->cp_nr_rings = (min == 1) ?
8086 max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8087 bp->tx_nr_rings + bp->rx_nr_rings;
8088
8089 } else {
8090 rc = -ENOMEM;
8091 goto msix_setup_exit;
8092 }
8093 bp->flags |= BNXT_FLAG_USING_MSIX;
8094 kfree(msix_ent);
8095 return 0;
8096
8097 msix_setup_exit:
8098 netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8099 kfree(bp->irq_tbl);
8100 bp->irq_tbl = NULL;
8101 pci_disable_msix(bp->pdev);
8102 kfree(msix_ent);
8103 return rc;
8104 }
8105
8106 static int bnxt_init_inta(struct bnxt *bp)
8107 {
8108 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8109 if (!bp->irq_tbl)
8110 return -ENOMEM;
8111
8112 bp->total_irqs = 1;
8113 bp->rx_nr_rings = 1;
8114 bp->tx_nr_rings = 1;
8115 bp->cp_nr_rings = 1;
8116 bp->flags |= BNXT_FLAG_SHARED_RINGS;
8117 bp->irq_tbl[0].vector = bp->pdev->irq;
8118 return 0;
8119 }
8120
8121 static int bnxt_init_int_mode(struct bnxt *bp)
8122 {
8123 int rc = 0;
8124
8125 if (bp->flags & BNXT_FLAG_MSIX_CAP)
8126 rc = bnxt_init_msix(bp);
8127
8128 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8129 /* fallback to INTA */
8130 rc = bnxt_init_inta(bp);
8131 }
8132 return rc;
8133 }
8134
8135 static void bnxt_clear_int_mode(struct bnxt *bp)
8136 {
8137 if (bp->flags & BNXT_FLAG_USING_MSIX)
8138 pci_disable_msix(bp->pdev);
8139
8140 kfree(bp->irq_tbl);
8141 bp->irq_tbl = NULL;
8142 bp->flags &= ~BNXT_FLAG_USING_MSIX;
8143 }
8144
8145 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8146 {
8147 int tcs = netdev_get_num_tc(bp->dev);
8148 bool irq_cleared = false;
8149 int rc;
8150
8151 if (!bnxt_need_reserve_rings(bp))
8152 return 0;
8153
8154 if (irq_re_init && BNXT_NEW_RM(bp) &&
8155 bnxt_get_num_msix(bp) != bp->total_irqs) {
8156 bnxt_ulp_irq_stop(bp);
8157 bnxt_clear_int_mode(bp);
8158 irq_cleared = true;
8159 }
8160 rc = __bnxt_reserve_rings(bp);
8161 if (irq_cleared) {
8162 if (!rc)
8163 rc = bnxt_init_int_mode(bp);
8164 bnxt_ulp_irq_restart(bp, rc);
8165 }
8166 if (rc) {
8167 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8168 return rc;
8169 }
8170 if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8171 netdev_err(bp->dev, "tx ring reservation failure\n");
8172 netdev_reset_tc(bp->dev);
8173 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8174 return -ENOMEM;
8175 }
8176 return 0;
8177 }
8178
8179 static void bnxt_free_irq(struct bnxt *bp)
8180 {
8181 struct bnxt_irq *irq;
8182 int i;
8183
8184 #ifdef CONFIG_RFS_ACCEL
8185 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8186 bp->dev->rx_cpu_rmap = NULL;
8187 #endif
8188 if (!bp->irq_tbl || !bp->bnapi)
8189 return;
8190
8191 for (i = 0; i < bp->cp_nr_rings; i++) {
8192 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8193
8194 irq = &bp->irq_tbl[map_idx];
8195 if (irq->requested) {
8196 if (irq->have_cpumask) {
8197 irq_set_affinity_hint(irq->vector, NULL);
8198 free_cpumask_var(irq->cpu_mask);
8199 irq->have_cpumask = 0;
8200 }
8201 free_irq(irq->vector, bp->bnapi[i]);
8202 }
8203
8204 irq->requested = 0;
8205 }
8206 }
8207
8208 static int bnxt_request_irq(struct bnxt *bp)
8209 {
8210 int i, j, rc = 0;
8211 unsigned long flags = 0;
8212 #ifdef CONFIG_RFS_ACCEL
8213 struct cpu_rmap *rmap;
8214 #endif
8215
8216 rc = bnxt_setup_int_mode(bp);
8217 if (rc) {
8218 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8219 rc);
8220 return rc;
8221 }
8222 #ifdef CONFIG_RFS_ACCEL
8223 rmap = bp->dev->rx_cpu_rmap;
8224 #endif
8225 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8226 flags = IRQF_SHARED;
8227
8228 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8229 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8230 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8231
8232 #ifdef CONFIG_RFS_ACCEL
8233 if (rmap && bp->bnapi[i]->rx_ring) {
8234 rc = irq_cpu_rmap_add(rmap, irq->vector);
8235 if (rc)
8236 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8237 j);
8238 j++;
8239 }
8240 #endif
8241 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8242 bp->bnapi[i]);
8243 if (rc)
8244 break;
8245
8246 irq->requested = 1;
8247
8248 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8249 int numa_node = dev_to_node(&bp->pdev->dev);
8250
8251 irq->have_cpumask = 1;
8252 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8253 irq->cpu_mask);
8254 rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8255 if (rc) {
8256 netdev_warn(bp->dev,
8257 "Set affinity failed, IRQ = %d\n",
8258 irq->vector);
8259 break;
8260 }
8261 }
8262 }
8263 return rc;
8264 }
8265
8266 static void bnxt_del_napi(struct bnxt *bp)
8267 {
8268 int i;
8269
8270 if (!bp->bnapi)
8271 return;
8272
8273 for (i = 0; i < bp->cp_nr_rings; i++) {
8274 struct bnxt_napi *bnapi = bp->bnapi[i];
8275
8276 napi_hash_del(&bnapi->napi);
8277 netif_napi_del(&bnapi->napi);
8278 }
8279 /* We called napi_hash_del() before netif_napi_del(), we need
8280 * to respect an RCU grace period before freeing napi structures.
8281 */
8282 synchronize_net();
8283 }
8284
8285 static void bnxt_init_napi(struct bnxt *bp)
8286 {
8287 int i;
8288 unsigned int cp_nr_rings = bp->cp_nr_rings;
8289 struct bnxt_napi *bnapi;
8290
8291 if (bp->flags & BNXT_FLAG_USING_MSIX) {
8292 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8293
8294 if (bp->flags & BNXT_FLAG_CHIP_P5)
8295 poll_fn = bnxt_poll_p5;
8296 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8297 cp_nr_rings--;
8298 for (i = 0; i < cp_nr_rings; i++) {
8299 bnapi = bp->bnapi[i];
8300 netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8301 }
8302 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8303 bnapi = bp->bnapi[cp_nr_rings];
8304 netif_napi_add(bp->dev, &bnapi->napi,
8305 bnxt_poll_nitroa0, 64);
8306 }
8307 } else {
8308 bnapi = bp->bnapi[0];
8309 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8310 }
8311 }
8312
8313 static void bnxt_disable_napi(struct bnxt *bp)
8314 {
8315 int i;
8316
8317 if (!bp->bnapi)
8318 return;
8319
8320 for (i = 0; i < bp->cp_nr_rings; i++) {
8321 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8322
8323 if (bp->bnapi[i]->rx_ring)
8324 cancel_work_sync(&cpr->dim.work);
8325
8326 napi_disable(&bp->bnapi[i]->napi);
8327 }
8328 }
8329
8330 static void bnxt_enable_napi(struct bnxt *bp)
8331 {
8332 int i;
8333
8334 for (i = 0; i < bp->cp_nr_rings; i++) {
8335 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8336 bp->bnapi[i]->in_reset = false;
8337
8338 if (bp->bnapi[i]->rx_ring) {
8339 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8340 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8341 }
8342 napi_enable(&bp->bnapi[i]->napi);
8343 }
8344 }
8345
8346 void bnxt_tx_disable(struct bnxt *bp)
8347 {
8348 int i;
8349 struct bnxt_tx_ring_info *txr;
8350
8351 if (bp->tx_ring) {
8352 for (i = 0; i < bp->tx_nr_rings; i++) {
8353 txr = &bp->tx_ring[i];
8354 txr->dev_state = BNXT_DEV_STATE_CLOSING;
8355 }
8356 }
8357 /* Stop all TX queues */
8358 netif_tx_disable(bp->dev);
8359 netif_carrier_off(bp->dev);
8360 }
8361
8362 void bnxt_tx_enable(struct bnxt *bp)
8363 {
8364 int i;
8365 struct bnxt_tx_ring_info *txr;
8366
8367 for (i = 0; i < bp->tx_nr_rings; i++) {
8368 txr = &bp->tx_ring[i];
8369 txr->dev_state = 0;
8370 }
8371 netif_tx_wake_all_queues(bp->dev);
8372 if (bp->link_info.link_up)
8373 netif_carrier_on(bp->dev);
8374 }
8375
8376 static void bnxt_report_link(struct bnxt *bp)
8377 {
8378 if (bp->link_info.link_up) {
8379 const char *duplex;
8380 const char *flow_ctrl;
8381 u32 speed;
8382 u16 fec;
8383
8384 netif_carrier_on(bp->dev);
8385 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8386 duplex = "full";
8387 else
8388 duplex = "half";
8389 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8390 flow_ctrl = "ON - receive & transmit";
8391 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8392 flow_ctrl = "ON - transmit";
8393 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8394 flow_ctrl = "ON - receive";
8395 else
8396 flow_ctrl = "none";
8397 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8398 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8399 speed, duplex, flow_ctrl);
8400 if (bp->flags & BNXT_FLAG_EEE_CAP)
8401 netdev_info(bp->dev, "EEE is %s\n",
8402 bp->eee.eee_active ? "active" :
8403 "not active");
8404 fec = bp->link_info.fec_cfg;
8405 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8406 netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8407 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8408 (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8409 (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
8410 } else {
8411 netif_carrier_off(bp->dev);
8412 netdev_err(bp->dev, "NIC Link is Down\n");
8413 }
8414 }
8415
8416 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8417 {
8418 int rc = 0;
8419 struct hwrm_port_phy_qcaps_input req = {0};
8420 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8421 struct bnxt_link_info *link_info = &bp->link_info;
8422
8423 bp->flags &= ~BNXT_FLAG_EEE_CAP;
8424 if (bp->test_info)
8425 bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8426 BNXT_TEST_FL_AN_PHY_LPBK);
8427 if (bp->hwrm_spec_code < 0x10201)
8428 return 0;
8429
8430 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8431
8432 mutex_lock(&bp->hwrm_cmd_lock);
8433 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8434 if (rc)
8435 goto hwrm_phy_qcaps_exit;
8436
8437 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8438 struct ethtool_eee *eee = &bp->eee;
8439 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8440
8441 bp->flags |= BNXT_FLAG_EEE_CAP;
8442 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8443 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8444 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8445 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8446 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8447 }
8448 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8449 if (bp->test_info)
8450 bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8451 }
8452 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8453 if (bp->test_info)
8454 bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8455 }
8456 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8457 if (BNXT_PF(bp))
8458 bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8459 }
8460 if (resp->supported_speeds_auto_mode)
8461 link_info->support_auto_speeds =
8462 le16_to_cpu(resp->supported_speeds_auto_mode);
8463
8464 bp->port_count = resp->port_cnt;
8465
8466 hwrm_phy_qcaps_exit:
8467 mutex_unlock(&bp->hwrm_cmd_lock);
8468 return rc;
8469 }
8470
8471 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8472 {
8473 int rc = 0;
8474 struct bnxt_link_info *link_info = &bp->link_info;
8475 struct hwrm_port_phy_qcfg_input req = {0};
8476 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8477 u8 link_up = link_info->link_up;
8478 u16 diff;
8479
8480 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8481
8482 mutex_lock(&bp->hwrm_cmd_lock);
8483 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8484 if (rc) {
8485 mutex_unlock(&bp->hwrm_cmd_lock);
8486 return rc;
8487 }
8488
8489 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8490 link_info->phy_link_status = resp->link;
8491 link_info->duplex = resp->duplex_cfg;
8492 if (bp->hwrm_spec_code >= 0x10800)
8493 link_info->duplex = resp->duplex_state;
8494 link_info->pause = resp->pause;
8495 link_info->auto_mode = resp->auto_mode;
8496 link_info->auto_pause_setting = resp->auto_pause;
8497 link_info->lp_pause = resp->link_partner_adv_pause;
8498 link_info->force_pause_setting = resp->force_pause;
8499 link_info->duplex_setting = resp->duplex_cfg;
8500 if (link_info->phy_link_status == BNXT_LINK_LINK)
8501 link_info->link_speed = le16_to_cpu(resp->link_speed);
8502 else
8503 link_info->link_speed = 0;
8504 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
8505 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8506 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
8507 link_info->lp_auto_link_speeds =
8508 le16_to_cpu(resp->link_partner_adv_speeds);
8509 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8510 link_info->phy_ver[0] = resp->phy_maj;
8511 link_info->phy_ver[1] = resp->phy_min;
8512 link_info->phy_ver[2] = resp->phy_bld;
8513 link_info->media_type = resp->media_type;
8514 link_info->phy_type = resp->phy_type;
8515 link_info->transceiver = resp->xcvr_pkg_type;
8516 link_info->phy_addr = resp->eee_config_phy_addr &
8517 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
8518 link_info->module_status = resp->module_status;
8519
8520 if (bp->flags & BNXT_FLAG_EEE_CAP) {
8521 struct ethtool_eee *eee = &bp->eee;
8522 u16 fw_speeds;
8523
8524 eee->eee_active = 0;
8525 if (resp->eee_config_phy_addr &
8526 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8527 eee->eee_active = 1;
8528 fw_speeds = le16_to_cpu(
8529 resp->link_partner_adv_eee_link_speed_mask);
8530 eee->lp_advertised =
8531 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8532 }
8533
8534 /* Pull initial EEE config */
8535 if (!chng_link_state) {
8536 if (resp->eee_config_phy_addr &
8537 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8538 eee->eee_enabled = 1;
8539
8540 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8541 eee->advertised =
8542 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8543
8544 if (resp->eee_config_phy_addr &
8545 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8546 __le32 tmr;
8547
8548 eee->tx_lpi_enabled = 1;
8549 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8550 eee->tx_lpi_timer = le32_to_cpu(tmr) &
8551 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8552 }
8553 }
8554 }
8555
8556 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8557 if (bp->hwrm_spec_code >= 0x10504)
8558 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8559
8560 /* TODO: need to add more logic to report VF link */
8561 if (chng_link_state) {
8562 if (link_info->phy_link_status == BNXT_LINK_LINK)
8563 link_info->link_up = 1;
8564 else
8565 link_info->link_up = 0;
8566 if (link_up != link_info->link_up)
8567 bnxt_report_link(bp);
8568 } else {
8569 /* alwasy link down if not require to update link state */
8570 link_info->link_up = 0;
8571 }
8572 mutex_unlock(&bp->hwrm_cmd_lock);
8573
8574 if (!BNXT_PHY_CFG_ABLE(bp))
8575 return 0;
8576
8577 diff = link_info->support_auto_speeds ^ link_info->advertising;
8578 if ((link_info->support_auto_speeds | diff) !=
8579 link_info->support_auto_speeds) {
8580 /* An advertised speed is no longer supported, so we need to
8581 * update the advertisement settings. Caller holds RTNL
8582 * so we can modify link settings.
8583 */
8584 link_info->advertising = link_info->support_auto_speeds;
8585 if (link_info->autoneg & BNXT_AUTONEG_SPEED)
8586 bnxt_hwrm_set_link_setting(bp, true, false);
8587 }
8588 return 0;
8589 }
8590
8591 static void bnxt_get_port_module_status(struct bnxt *bp)
8592 {
8593 struct bnxt_link_info *link_info = &bp->link_info;
8594 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8595 u8 module_status;
8596
8597 if (bnxt_update_link(bp, true))
8598 return;
8599
8600 module_status = link_info->module_status;
8601 switch (module_status) {
8602 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8603 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8604 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8605 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8606 bp->pf.port_id);
8607 if (bp->hwrm_spec_code >= 0x10201) {
8608 netdev_warn(bp->dev, "Module part number %s\n",
8609 resp->phy_vendor_partnumber);
8610 }
8611 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8612 netdev_warn(bp->dev, "TX is disabled\n");
8613 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8614 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8615 }
8616 }
8617
8618 static void
8619 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8620 {
8621 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
8622 if (bp->hwrm_spec_code >= 0x10201)
8623 req->auto_pause =
8624 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
8625 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8626 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8627 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8628 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
8629 req->enables |=
8630 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8631 } else {
8632 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8633 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8634 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8635 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
8636 req->enables |=
8637 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
8638 if (bp->hwrm_spec_code >= 0x10201) {
8639 req->auto_pause = req->force_pause;
8640 req->enables |= cpu_to_le32(
8641 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8642 }
8643 }
8644 }
8645
8646 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
8647 struct hwrm_port_phy_cfg_input *req)
8648 {
8649 u8 autoneg = bp->link_info.autoneg;
8650 u16 fw_link_speed = bp->link_info.req_link_speed;
8651 u16 advertising = bp->link_info.advertising;
8652
8653 if (autoneg & BNXT_AUTONEG_SPEED) {
8654 req->auto_mode |=
8655 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
8656
8657 req->enables |= cpu_to_le32(
8658 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
8659 req->auto_link_speed_mask = cpu_to_le16(advertising);
8660
8661 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
8662 req->flags |=
8663 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
8664 } else {
8665 req->force_link_speed = cpu_to_le16(fw_link_speed);
8666 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
8667 }
8668
8669 /* tell chimp that the setting takes effect immediately */
8670 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
8671 }
8672
8673 int bnxt_hwrm_set_pause(struct bnxt *bp)
8674 {
8675 struct hwrm_port_phy_cfg_input req = {0};
8676 int rc;
8677
8678 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8679 bnxt_hwrm_set_pause_common(bp, &req);
8680
8681 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
8682 bp->link_info.force_link_chng)
8683 bnxt_hwrm_set_link_common(bp, &req);
8684
8685 mutex_lock(&bp->hwrm_cmd_lock);
8686 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8687 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
8688 /* since changing of pause setting doesn't trigger any link
8689 * change event, the driver needs to update the current pause
8690 * result upon successfully return of the phy_cfg command
8691 */
8692 bp->link_info.pause =
8693 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
8694 bp->link_info.auto_pause_setting = 0;
8695 if (!bp->link_info.force_link_chng)
8696 bnxt_report_link(bp);
8697 }
8698 bp->link_info.force_link_chng = false;
8699 mutex_unlock(&bp->hwrm_cmd_lock);
8700 return rc;
8701 }
8702
8703 static void bnxt_hwrm_set_eee(struct bnxt *bp,
8704 struct hwrm_port_phy_cfg_input *req)
8705 {
8706 struct ethtool_eee *eee = &bp->eee;
8707
8708 if (eee->eee_enabled) {
8709 u16 eee_speeds;
8710 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
8711
8712 if (eee->tx_lpi_enabled)
8713 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
8714 else
8715 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
8716
8717 req->flags |= cpu_to_le32(flags);
8718 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
8719 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
8720 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
8721 } else {
8722 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
8723 }
8724 }
8725
8726 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
8727 {
8728 struct hwrm_port_phy_cfg_input req = {0};
8729
8730 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8731 if (set_pause)
8732 bnxt_hwrm_set_pause_common(bp, &req);
8733
8734 bnxt_hwrm_set_link_common(bp, &req);
8735
8736 if (set_eee)
8737 bnxt_hwrm_set_eee(bp, &req);
8738 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8739 }
8740
8741 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
8742 {
8743 struct hwrm_port_phy_cfg_input req = {0};
8744
8745 if (!BNXT_SINGLE_PF(bp))
8746 return 0;
8747
8748 if (pci_num_vf(bp->pdev))
8749 return 0;
8750
8751 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
8752 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
8753 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8754 }
8755
8756 static int bnxt_fw_init_one(struct bnxt *bp);
8757
8758 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
8759 {
8760 struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
8761 struct hwrm_func_drv_if_change_input req = {0};
8762 bool resc_reinit = false, fw_reset = false;
8763 u32 flags = 0;
8764 int rc;
8765
8766 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
8767 return 0;
8768
8769 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
8770 if (up)
8771 req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
8772 mutex_lock(&bp->hwrm_cmd_lock);
8773 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8774 if (!rc)
8775 flags = le32_to_cpu(resp->flags);
8776 mutex_unlock(&bp->hwrm_cmd_lock);
8777 if (rc)
8778 return rc;
8779
8780 if (!up)
8781 return 0;
8782
8783 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
8784 resc_reinit = true;
8785 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
8786 fw_reset = true;
8787
8788 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
8789 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
8790 return -ENODEV;
8791 }
8792 if (resc_reinit || fw_reset) {
8793 if (fw_reset) {
8794 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
8795 bnxt_ulp_stop(bp);
8796 bnxt_free_ctx_mem(bp);
8797 kfree(bp->ctx);
8798 bp->ctx = NULL;
8799 rc = bnxt_fw_init_one(bp);
8800 if (rc) {
8801 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
8802 return rc;
8803 }
8804 bnxt_clear_int_mode(bp);
8805 rc = bnxt_init_int_mode(bp);
8806 if (rc) {
8807 netdev_err(bp->dev, "init int mode failed\n");
8808 return rc;
8809 }
8810 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
8811 }
8812 if (BNXT_NEW_RM(bp)) {
8813 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8814
8815 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
8816 hw_resc->resv_cp_rings = 0;
8817 hw_resc->resv_stat_ctxs = 0;
8818 hw_resc->resv_irqs = 0;
8819 hw_resc->resv_tx_rings = 0;
8820 hw_resc->resv_rx_rings = 0;
8821 hw_resc->resv_hw_ring_grps = 0;
8822 hw_resc->resv_vnics = 0;
8823 if (!fw_reset) {
8824 bp->tx_nr_rings = 0;
8825 bp->rx_nr_rings = 0;
8826 }
8827 }
8828 }
8829 return 0;
8830 }
8831
8832 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
8833 {
8834 struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8835 struct hwrm_port_led_qcaps_input req = {0};
8836 struct bnxt_pf_info *pf = &bp->pf;
8837 int rc;
8838
8839 bp->num_leds = 0;
8840 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
8841 return 0;
8842
8843 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
8844 req.port_id = cpu_to_le16(pf->port_id);
8845 mutex_lock(&bp->hwrm_cmd_lock);
8846 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8847 if (rc) {
8848 mutex_unlock(&bp->hwrm_cmd_lock);
8849 return rc;
8850 }
8851 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
8852 int i;
8853
8854 bp->num_leds = resp->num_leds;
8855 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
8856 bp->num_leds);
8857 for (i = 0; i < bp->num_leds; i++) {
8858 struct bnxt_led_info *led = &bp->leds[i];
8859 __le16 caps = led->led_state_caps;
8860
8861 if (!led->led_group_id ||
8862 !BNXT_LED_ALT_BLINK_CAP(caps)) {
8863 bp->num_leds = 0;
8864 break;
8865 }
8866 }
8867 }
8868 mutex_unlock(&bp->hwrm_cmd_lock);
8869 return 0;
8870 }
8871
8872 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
8873 {
8874 struct hwrm_wol_filter_alloc_input req = {0};
8875 struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
8876 int rc;
8877
8878 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
8879 req.port_id = cpu_to_le16(bp->pf.port_id);
8880 req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
8881 req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
8882 memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
8883 mutex_lock(&bp->hwrm_cmd_lock);
8884 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8885 if (!rc)
8886 bp->wol_filter_id = resp->wol_filter_id;
8887 mutex_unlock(&bp->hwrm_cmd_lock);
8888 return rc;
8889 }
8890
8891 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
8892 {
8893 struct hwrm_wol_filter_free_input req = {0};
8894 int rc;
8895
8896 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
8897 req.port_id = cpu_to_le16(bp->pf.port_id);
8898 req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
8899 req.wol_filter_id = bp->wol_filter_id;
8900 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8901 return rc;
8902 }
8903
8904 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
8905 {
8906 struct hwrm_wol_filter_qcfg_input req = {0};
8907 struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8908 u16 next_handle = 0;
8909 int rc;
8910
8911 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
8912 req.port_id = cpu_to_le16(bp->pf.port_id);
8913 req.handle = cpu_to_le16(handle);
8914 mutex_lock(&bp->hwrm_cmd_lock);
8915 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8916 if (!rc) {
8917 next_handle = le16_to_cpu(resp->next_handle);
8918 if (next_handle != 0) {
8919 if (resp->wol_type ==
8920 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
8921 bp->wol = 1;
8922 bp->wol_filter_id = resp->wol_filter_id;
8923 }
8924 }
8925 }
8926 mutex_unlock(&bp->hwrm_cmd_lock);
8927 return next_handle;
8928 }
8929
8930 static void bnxt_get_wol_settings(struct bnxt *bp)
8931 {
8932 u16 handle = 0;
8933
8934 bp->wol = 0;
8935 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
8936 return;
8937
8938 do {
8939 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
8940 } while (handle && handle != 0xffff);
8941 }
8942
8943 #ifdef CONFIG_BNXT_HWMON
8944 static ssize_t bnxt_show_temp(struct device *dev,
8945 struct device_attribute *devattr, char *buf)
8946 {
8947 struct hwrm_temp_monitor_query_input req = {0};
8948 struct hwrm_temp_monitor_query_output *resp;
8949 struct bnxt *bp = dev_get_drvdata(dev);
8950 u32 temp = 0;
8951
8952 resp = bp->hwrm_cmd_resp_addr;
8953 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
8954 mutex_lock(&bp->hwrm_cmd_lock);
8955 if (!_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
8956 temp = resp->temp * 1000; /* display millidegree */
8957 mutex_unlock(&bp->hwrm_cmd_lock);
8958
8959 return sprintf(buf, "%u\n", temp);
8960 }
8961 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
8962
8963 static struct attribute *bnxt_attrs[] = {
8964 &sensor_dev_attr_temp1_input.dev_attr.attr,
8965 NULL
8966 };
8967 ATTRIBUTE_GROUPS(bnxt);
8968
8969 static void bnxt_hwmon_close(struct bnxt *bp)
8970 {
8971 if (bp->hwmon_dev) {
8972 hwmon_device_unregister(bp->hwmon_dev);
8973 bp->hwmon_dev = NULL;
8974 }
8975 }
8976
8977 static void bnxt_hwmon_open(struct bnxt *bp)
8978 {
8979 struct pci_dev *pdev = bp->pdev;
8980
8981 if (bp->hwmon_dev)
8982 return;
8983
8984 bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
8985 DRV_MODULE_NAME, bp,
8986 bnxt_groups);
8987 if (IS_ERR(bp->hwmon_dev)) {
8988 bp->hwmon_dev = NULL;
8989 dev_warn(&pdev->dev, "Cannot register hwmon device\n");
8990 }
8991 }
8992 #else
8993 static void bnxt_hwmon_close(struct bnxt *bp)
8994 {
8995 }
8996
8997 static void bnxt_hwmon_open(struct bnxt *bp)
8998 {
8999 }
9000 #endif
9001
9002 static bool bnxt_eee_config_ok(struct bnxt *bp)
9003 {
9004 struct ethtool_eee *eee = &bp->eee;
9005 struct bnxt_link_info *link_info = &bp->link_info;
9006
9007 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9008 return true;
9009
9010 if (eee->eee_enabled) {
9011 u32 advertising =
9012 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9013
9014 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9015 eee->eee_enabled = 0;
9016 return false;
9017 }
9018 if (eee->advertised & ~advertising) {
9019 eee->advertised = advertising & eee->supported;
9020 return false;
9021 }
9022 }
9023 return true;
9024 }
9025
9026 static int bnxt_update_phy_setting(struct bnxt *bp)
9027 {
9028 int rc;
9029 bool update_link = false;
9030 bool update_pause = false;
9031 bool update_eee = false;
9032 struct bnxt_link_info *link_info = &bp->link_info;
9033
9034 rc = bnxt_update_link(bp, true);
9035 if (rc) {
9036 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9037 rc);
9038 return rc;
9039 }
9040 if (!BNXT_SINGLE_PF(bp))
9041 return 0;
9042
9043 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9044 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9045 link_info->req_flow_ctrl)
9046 update_pause = true;
9047 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9048 link_info->force_pause_setting != link_info->req_flow_ctrl)
9049 update_pause = true;
9050 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9051 if (BNXT_AUTO_MODE(link_info->auto_mode))
9052 update_link = true;
9053 if (link_info->req_link_speed != link_info->force_link_speed)
9054 update_link = true;
9055 if (link_info->req_duplex != link_info->duplex_setting)
9056 update_link = true;
9057 } else {
9058 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9059 update_link = true;
9060 if (link_info->advertising != link_info->auto_link_speeds)
9061 update_link = true;
9062 }
9063
9064 /* The last close may have shutdown the link, so need to call
9065 * PHY_CFG to bring it back up.
9066 */
9067 if (!bp->link_info.link_up)
9068 update_link = true;
9069
9070 if (!bnxt_eee_config_ok(bp))
9071 update_eee = true;
9072
9073 if (update_link)
9074 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9075 else if (update_pause)
9076 rc = bnxt_hwrm_set_pause(bp);
9077 if (rc) {
9078 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9079 rc);
9080 return rc;
9081 }
9082
9083 return rc;
9084 }
9085
9086 /* Common routine to pre-map certain register block to different GRC window.
9087 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9088 * in PF and 3 windows in VF that can be customized to map in different
9089 * register blocks.
9090 */
9091 static void bnxt_preset_reg_win(struct bnxt *bp)
9092 {
9093 if (BNXT_PF(bp)) {
9094 /* CAG registers map to GRC window #4 */
9095 writel(BNXT_CAG_REG_BASE,
9096 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9097 }
9098 }
9099
9100 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9101
9102 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9103 {
9104 int rc = 0;
9105
9106 bnxt_preset_reg_win(bp);
9107 netif_carrier_off(bp->dev);
9108 if (irq_re_init) {
9109 /* Reserve rings now if none were reserved at driver probe. */
9110 rc = bnxt_init_dflt_ring_mode(bp);
9111 if (rc) {
9112 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9113 return rc;
9114 }
9115 }
9116 rc = bnxt_reserve_rings(bp, irq_re_init);
9117 if (rc)
9118 return rc;
9119 if ((bp->flags & BNXT_FLAG_RFS) &&
9120 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9121 /* disable RFS if falling back to INTA */
9122 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9123 bp->flags &= ~BNXT_FLAG_RFS;
9124 }
9125
9126 rc = bnxt_alloc_mem(bp, irq_re_init);
9127 if (rc) {
9128 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9129 goto open_err_free_mem;
9130 }
9131
9132 if (irq_re_init) {
9133 bnxt_init_napi(bp);
9134 rc = bnxt_request_irq(bp);
9135 if (rc) {
9136 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9137 goto open_err_irq;
9138 }
9139 }
9140
9141 bnxt_enable_napi(bp);
9142 bnxt_debug_dev_init(bp);
9143
9144 rc = bnxt_init_nic(bp, irq_re_init);
9145 if (rc) {
9146 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9147 goto open_err;
9148 }
9149
9150 if (link_re_init) {
9151 mutex_lock(&bp->link_lock);
9152 rc = bnxt_update_phy_setting(bp);
9153 mutex_unlock(&bp->link_lock);
9154 if (rc) {
9155 netdev_warn(bp->dev, "failed to update phy settings\n");
9156 if (BNXT_SINGLE_PF(bp)) {
9157 bp->link_info.phy_retry = true;
9158 bp->link_info.phy_retry_expires =
9159 jiffies + 5 * HZ;
9160 }
9161 }
9162 }
9163
9164 if (irq_re_init)
9165 udp_tunnel_get_rx_info(bp->dev);
9166
9167 set_bit(BNXT_STATE_OPEN, &bp->state);
9168 bnxt_enable_int(bp);
9169 /* Enable TX queues */
9170 bnxt_tx_enable(bp);
9171 mod_timer(&bp->timer, jiffies + bp->current_interval);
9172 /* Poll link status and check for SFP+ module status */
9173 bnxt_get_port_module_status(bp);
9174
9175 /* VF-reps may need to be re-opened after the PF is re-opened */
9176 if (BNXT_PF(bp))
9177 bnxt_vf_reps_open(bp);
9178 return 0;
9179
9180 open_err:
9181 bnxt_debug_dev_exit(bp);
9182 bnxt_disable_napi(bp);
9183
9184 open_err_irq:
9185 bnxt_del_napi(bp);
9186
9187 open_err_free_mem:
9188 bnxt_free_skbs(bp);
9189 bnxt_free_irq(bp);
9190 bnxt_free_mem(bp, true);
9191 return rc;
9192 }
9193
9194 /* rtnl_lock held */
9195 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9196 {
9197 int rc = 0;
9198
9199 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9200 if (rc) {
9201 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9202 dev_close(bp->dev);
9203 }
9204 return rc;
9205 }
9206
9207 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9208 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
9209 * self tests.
9210 */
9211 int bnxt_half_open_nic(struct bnxt *bp)
9212 {
9213 int rc = 0;
9214
9215 rc = bnxt_alloc_mem(bp, false);
9216 if (rc) {
9217 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9218 goto half_open_err;
9219 }
9220 rc = bnxt_init_nic(bp, false);
9221 if (rc) {
9222 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9223 goto half_open_err;
9224 }
9225 return 0;
9226
9227 half_open_err:
9228 bnxt_free_skbs(bp);
9229 bnxt_free_mem(bp, false);
9230 dev_close(bp->dev);
9231 return rc;
9232 }
9233
9234 /* rtnl_lock held, this call can only be made after a previous successful
9235 * call to bnxt_half_open_nic().
9236 */
9237 void bnxt_half_close_nic(struct bnxt *bp)
9238 {
9239 bnxt_hwrm_resource_free(bp, false, false);
9240 bnxt_free_skbs(bp);
9241 bnxt_free_mem(bp, false);
9242 }
9243
9244 static int bnxt_open(struct net_device *dev)
9245 {
9246 struct bnxt *bp = netdev_priv(dev);
9247 int rc;
9248
9249 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9250 netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9251 return -ENODEV;
9252 }
9253
9254 rc = bnxt_hwrm_if_change(bp, true);
9255 if (rc)
9256 return rc;
9257 rc = __bnxt_open_nic(bp, true, true);
9258 if (rc) {
9259 bnxt_hwrm_if_change(bp, false);
9260 } else {
9261 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9262 if (BNXT_PF(bp)) {
9263 struct bnxt_pf_info *pf = &bp->pf;
9264 int n = pf->active_vfs;
9265
9266 if (n)
9267 bnxt_cfg_hw_sriov(bp, &n, true);
9268 }
9269 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9270 bnxt_ulp_start(bp, 0);
9271 }
9272 bnxt_hwmon_open(bp);
9273 }
9274
9275 return rc;
9276 }
9277
9278 static bool bnxt_drv_busy(struct bnxt *bp)
9279 {
9280 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9281 test_bit(BNXT_STATE_READ_STATS, &bp->state));
9282 }
9283
9284 static void bnxt_get_ring_stats(struct bnxt *bp,
9285 struct rtnl_link_stats64 *stats);
9286
9287 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9288 bool link_re_init)
9289 {
9290 /* Close the VF-reps before closing PF */
9291 if (BNXT_PF(bp))
9292 bnxt_vf_reps_close(bp);
9293
9294 /* Change device state to avoid TX queue wake up's */
9295 bnxt_tx_disable(bp);
9296
9297 clear_bit(BNXT_STATE_OPEN, &bp->state);
9298 smp_mb__after_atomic();
9299 while (bnxt_drv_busy(bp))
9300 msleep(20);
9301
9302 /* Flush rings and and disable interrupts */
9303 bnxt_shutdown_nic(bp, irq_re_init);
9304
9305 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9306
9307 bnxt_debug_dev_exit(bp);
9308 bnxt_disable_napi(bp);
9309 del_timer_sync(&bp->timer);
9310 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) &&
9311 pci_is_enabled(bp->pdev))
9312 pci_disable_device(bp->pdev);
9313
9314 bnxt_free_skbs(bp);
9315
9316 /* Save ring stats before shutdown */
9317 if (bp->bnapi)
9318 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9319 if (irq_re_init) {
9320 bnxt_free_irq(bp);
9321 bnxt_del_napi(bp);
9322 }
9323 bnxt_free_mem(bp, irq_re_init);
9324 }
9325
9326 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9327 {
9328 int rc = 0;
9329
9330 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9331 /* If we get here, it means firmware reset is in progress
9332 * while we are trying to close. We can safely proceed with
9333 * the close because we are holding rtnl_lock(). Some firmware
9334 * messages may fail as we proceed to close. We set the
9335 * ABORT_ERR flag here so that the FW reset thread will later
9336 * abort when it gets the rtnl_lock() and sees the flag.
9337 */
9338 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9339 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9340 }
9341
9342 #ifdef CONFIG_BNXT_SRIOV
9343 if (bp->sriov_cfg) {
9344 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9345 !bp->sriov_cfg,
9346 BNXT_SRIOV_CFG_WAIT_TMO);
9347 if (rc)
9348 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9349 }
9350 #endif
9351 __bnxt_close_nic(bp, irq_re_init, link_re_init);
9352 return rc;
9353 }
9354
9355 static int bnxt_close(struct net_device *dev)
9356 {
9357 struct bnxt *bp = netdev_priv(dev);
9358
9359 bnxt_hwmon_close(bp);
9360 bnxt_close_nic(bp, true, true);
9361 bnxt_hwrm_shutdown_link(bp);
9362 bnxt_hwrm_if_change(bp, false);
9363 return 0;
9364 }
9365
9366 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9367 u16 *val)
9368 {
9369 struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9370 struct hwrm_port_phy_mdio_read_input req = {0};
9371 int rc;
9372
9373 if (bp->hwrm_spec_code < 0x10a00)
9374 return -EOPNOTSUPP;
9375
9376 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9377 req.port_id = cpu_to_le16(bp->pf.port_id);
9378 req.phy_addr = phy_addr;
9379 req.reg_addr = cpu_to_le16(reg & 0x1f);
9380 if (mdio_phy_id_is_c45(phy_addr)) {
9381 req.cl45_mdio = 1;
9382 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9383 req.dev_addr = mdio_phy_id_devad(phy_addr);
9384 req.reg_addr = cpu_to_le16(reg);
9385 }
9386
9387 mutex_lock(&bp->hwrm_cmd_lock);
9388 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9389 if (!rc)
9390 *val = le16_to_cpu(resp->reg_data);
9391 mutex_unlock(&bp->hwrm_cmd_lock);
9392 return rc;
9393 }
9394
9395 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9396 u16 val)
9397 {
9398 struct hwrm_port_phy_mdio_write_input req = {0};
9399
9400 if (bp->hwrm_spec_code < 0x10a00)
9401 return -EOPNOTSUPP;
9402
9403 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9404 req.port_id = cpu_to_le16(bp->pf.port_id);
9405 req.phy_addr = phy_addr;
9406 req.reg_addr = cpu_to_le16(reg & 0x1f);
9407 if (mdio_phy_id_is_c45(phy_addr)) {
9408 req.cl45_mdio = 1;
9409 req.phy_addr = mdio_phy_id_prtad(phy_addr);
9410 req.dev_addr = mdio_phy_id_devad(phy_addr);
9411 req.reg_addr = cpu_to_le16(reg);
9412 }
9413 req.reg_data = cpu_to_le16(val);
9414
9415 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9416 }
9417
9418 /* rtnl_lock held */
9419 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9420 {
9421 struct mii_ioctl_data *mdio = if_mii(ifr);
9422 struct bnxt *bp = netdev_priv(dev);
9423 int rc;
9424
9425 switch (cmd) {
9426 case SIOCGMIIPHY:
9427 mdio->phy_id = bp->link_info.phy_addr;
9428
9429 /* fallthru */
9430 case SIOCGMIIREG: {
9431 u16 mii_regval = 0;
9432
9433 if (!netif_running(dev))
9434 return -EAGAIN;
9435
9436 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9437 &mii_regval);
9438 mdio->val_out = mii_regval;
9439 return rc;
9440 }
9441
9442 case SIOCSMIIREG:
9443 if (!netif_running(dev))
9444 return -EAGAIN;
9445
9446 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9447 mdio->val_in);
9448
9449 default:
9450 /* do nothing */
9451 break;
9452 }
9453 return -EOPNOTSUPP;
9454 }
9455
9456 static void bnxt_get_ring_stats(struct bnxt *bp,
9457 struct rtnl_link_stats64 *stats)
9458 {
9459 int i;
9460
9461
9462 for (i = 0; i < bp->cp_nr_rings; i++) {
9463 struct bnxt_napi *bnapi = bp->bnapi[i];
9464 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9465 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
9466
9467 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
9468 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
9469 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
9470
9471 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
9472 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
9473 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
9474
9475 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
9476 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
9477 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
9478
9479 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
9480 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
9481 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
9482
9483 stats->rx_missed_errors +=
9484 le64_to_cpu(hw_stats->rx_discard_pkts);
9485
9486 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
9487
9488 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
9489 }
9490 }
9491
9492 static void bnxt_add_prev_stats(struct bnxt *bp,
9493 struct rtnl_link_stats64 *stats)
9494 {
9495 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9496
9497 stats->rx_packets += prev_stats->rx_packets;
9498 stats->tx_packets += prev_stats->tx_packets;
9499 stats->rx_bytes += prev_stats->rx_bytes;
9500 stats->tx_bytes += prev_stats->tx_bytes;
9501 stats->rx_missed_errors += prev_stats->rx_missed_errors;
9502 stats->multicast += prev_stats->multicast;
9503 stats->tx_dropped += prev_stats->tx_dropped;
9504 }
9505
9506 static void
9507 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9508 {
9509 struct bnxt *bp = netdev_priv(dev);
9510
9511 set_bit(BNXT_STATE_READ_STATS, &bp->state);
9512 /* Make sure bnxt_close_nic() sees that we are reading stats before
9513 * we check the BNXT_STATE_OPEN flag.
9514 */
9515 smp_mb__after_atomic();
9516 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9517 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9518 *stats = bp->net_stats_prev;
9519 return;
9520 }
9521
9522 bnxt_get_ring_stats(bp, stats);
9523 bnxt_add_prev_stats(bp, stats);
9524
9525 if (bp->flags & BNXT_FLAG_PORT_STATS) {
9526 struct rx_port_stats *rx = bp->hw_rx_port_stats;
9527 struct tx_port_stats *tx = bp->hw_tx_port_stats;
9528
9529 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
9530 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
9531 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
9532 le64_to_cpu(rx->rx_ovrsz_frames) +
9533 le64_to_cpu(rx->rx_runt_frames);
9534 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
9535 le64_to_cpu(rx->rx_jbr_frames);
9536 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
9537 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
9538 stats->tx_errors = le64_to_cpu(tx->tx_err);
9539 }
9540 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9541 }
9542
9543 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9544 {
9545 struct net_device *dev = bp->dev;
9546 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9547 struct netdev_hw_addr *ha;
9548 u8 *haddr;
9549 int mc_count = 0;
9550 bool update = false;
9551 int off = 0;
9552
9553 netdev_for_each_mc_addr(ha, dev) {
9554 if (mc_count >= BNXT_MAX_MC_ADDRS) {
9555 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9556 vnic->mc_list_count = 0;
9557 return false;
9558 }
9559 haddr = ha->addr;
9560 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9561 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9562 update = true;
9563 }
9564 off += ETH_ALEN;
9565 mc_count++;
9566 }
9567 if (mc_count)
9568 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9569
9570 if (mc_count != vnic->mc_list_count) {
9571 vnic->mc_list_count = mc_count;
9572 update = true;
9573 }
9574 return update;
9575 }
9576
9577 static bool bnxt_uc_list_updated(struct bnxt *bp)
9578 {
9579 struct net_device *dev = bp->dev;
9580 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9581 struct netdev_hw_addr *ha;
9582 int off = 0;
9583
9584 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9585 return true;
9586
9587 netdev_for_each_uc_addr(ha, dev) {
9588 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9589 return true;
9590
9591 off += ETH_ALEN;
9592 }
9593 return false;
9594 }
9595
9596 static void bnxt_set_rx_mode(struct net_device *dev)
9597 {
9598 struct bnxt *bp = netdev_priv(dev);
9599 struct bnxt_vnic_info *vnic;
9600 bool mc_update = false;
9601 bool uc_update;
9602 u32 mask;
9603
9604 if (!test_bit(BNXT_STATE_OPEN, &bp->state))
9605 return;
9606
9607 vnic = &bp->vnic_info[0];
9608 mask = vnic->rx_mask;
9609 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9610 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
9611 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9612 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
9613
9614 if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
9615 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9616
9617 uc_update = bnxt_uc_list_updated(bp);
9618
9619 if (dev->flags & IFF_BROADCAST)
9620 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9621 if (dev->flags & IFF_ALLMULTI) {
9622 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9623 vnic->mc_list_count = 0;
9624 } else {
9625 mc_update = bnxt_mc_list_updated(bp, &mask);
9626 }
9627
9628 if (mask != vnic->rx_mask || uc_update || mc_update) {
9629 vnic->rx_mask = mask;
9630
9631 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
9632 bnxt_queue_sp_work(bp);
9633 }
9634 }
9635
9636 static int bnxt_cfg_rx_mode(struct bnxt *bp)
9637 {
9638 struct net_device *dev = bp->dev;
9639 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9640 struct netdev_hw_addr *ha;
9641 int i, off = 0, rc;
9642 bool uc_update;
9643
9644 netif_addr_lock_bh(dev);
9645 uc_update = bnxt_uc_list_updated(bp);
9646 netif_addr_unlock_bh(dev);
9647
9648 if (!uc_update)
9649 goto skip_uc;
9650
9651 mutex_lock(&bp->hwrm_cmd_lock);
9652 for (i = 1; i < vnic->uc_filter_count; i++) {
9653 struct hwrm_cfa_l2_filter_free_input req = {0};
9654
9655 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
9656 -1);
9657
9658 req.l2_filter_id = vnic->fw_l2_filter_id[i];
9659
9660 rc = _hwrm_send_message(bp, &req, sizeof(req),
9661 HWRM_CMD_TIMEOUT);
9662 }
9663 mutex_unlock(&bp->hwrm_cmd_lock);
9664
9665 vnic->uc_filter_count = 1;
9666
9667 netif_addr_lock_bh(dev);
9668 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
9669 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9670 } else {
9671 netdev_for_each_uc_addr(ha, dev) {
9672 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
9673 off += ETH_ALEN;
9674 vnic->uc_filter_count++;
9675 }
9676 }
9677 netif_addr_unlock_bh(dev);
9678
9679 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
9680 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
9681 if (rc) {
9682 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
9683 rc);
9684 vnic->uc_filter_count = i;
9685 return rc;
9686 }
9687 }
9688
9689 skip_uc:
9690 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9691 if (rc && vnic->mc_list_count) {
9692 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
9693 rc);
9694 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9695 vnic->mc_list_count = 0;
9696 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
9697 }
9698 if (rc)
9699 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
9700 rc);
9701
9702 return rc;
9703 }
9704
9705 static bool bnxt_can_reserve_rings(struct bnxt *bp)
9706 {
9707 #ifdef CONFIG_BNXT_SRIOV
9708 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
9709 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9710
9711 /* No minimum rings were provisioned by the PF. Don't
9712 * reserve rings by default when device is down.
9713 */
9714 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
9715 return true;
9716
9717 if (!netif_running(bp->dev))
9718 return false;
9719 }
9720 #endif
9721 return true;
9722 }
9723
9724 /* If the chip and firmware supports RFS */
9725 static bool bnxt_rfs_supported(struct bnxt *bp)
9726 {
9727 if (bp->flags & BNXT_FLAG_CHIP_P5) {
9728 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
9729 return true;
9730 return false;
9731 }
9732 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
9733 return true;
9734 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9735 return true;
9736 return false;
9737 }
9738
9739 /* If runtime conditions support RFS */
9740 static bool bnxt_rfs_capable(struct bnxt *bp)
9741 {
9742 #ifdef CONFIG_RFS_ACCEL
9743 int vnics, max_vnics, max_rss_ctxs;
9744
9745 if (bp->flags & BNXT_FLAG_CHIP_P5)
9746 return bnxt_rfs_supported(bp);
9747 if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
9748 return false;
9749
9750 vnics = 1 + bp->rx_nr_rings;
9751 max_vnics = bnxt_get_max_func_vnics(bp);
9752 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
9753
9754 /* RSS contexts not a limiting factor */
9755 if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
9756 max_rss_ctxs = max_vnics;
9757 if (vnics > max_vnics || vnics > max_rss_ctxs) {
9758 if (bp->rx_nr_rings > 1)
9759 netdev_warn(bp->dev,
9760 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
9761 min(max_rss_ctxs - 1, max_vnics - 1));
9762 return false;
9763 }
9764
9765 if (!BNXT_NEW_RM(bp))
9766 return true;
9767
9768 if (vnics == bp->hw_resc.resv_vnics)
9769 return true;
9770
9771 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
9772 if (vnics <= bp->hw_resc.resv_vnics)
9773 return true;
9774
9775 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
9776 bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
9777 return false;
9778 #else
9779 return false;
9780 #endif
9781 }
9782
9783 static netdev_features_t bnxt_fix_features(struct net_device *dev,
9784 netdev_features_t features)
9785 {
9786 struct bnxt *bp = netdev_priv(dev);
9787
9788 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
9789 features &= ~NETIF_F_NTUPLE;
9790
9791 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9792 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
9793
9794 if (!(features & NETIF_F_GRO))
9795 features &= ~NETIF_F_GRO_HW;
9796
9797 if (features & NETIF_F_GRO_HW)
9798 features &= ~NETIF_F_LRO;
9799
9800 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
9801 * turned on or off together.
9802 */
9803 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
9804 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
9805 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
9806 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9807 NETIF_F_HW_VLAN_STAG_RX);
9808 else
9809 features |= NETIF_F_HW_VLAN_CTAG_RX |
9810 NETIF_F_HW_VLAN_STAG_RX;
9811 }
9812 #ifdef CONFIG_BNXT_SRIOV
9813 if (BNXT_VF(bp)) {
9814 if (bp->vf.vlan) {
9815 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
9816 NETIF_F_HW_VLAN_STAG_RX);
9817 }
9818 }
9819 #endif
9820 return features;
9821 }
9822
9823 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
9824 {
9825 struct bnxt *bp = netdev_priv(dev);
9826 u32 flags = bp->flags;
9827 u32 changes;
9828 int rc = 0;
9829 bool re_init = false;
9830 bool update_tpa = false;
9831
9832 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
9833 if (features & NETIF_F_GRO_HW)
9834 flags |= BNXT_FLAG_GRO;
9835 else if (features & NETIF_F_LRO)
9836 flags |= BNXT_FLAG_LRO;
9837
9838 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
9839 flags &= ~BNXT_FLAG_TPA;
9840
9841 if (features & NETIF_F_HW_VLAN_CTAG_RX)
9842 flags |= BNXT_FLAG_STRIP_VLAN;
9843
9844 if (features & NETIF_F_NTUPLE)
9845 flags |= BNXT_FLAG_RFS;
9846
9847 changes = flags ^ bp->flags;
9848 if (changes & BNXT_FLAG_TPA) {
9849 update_tpa = true;
9850 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
9851 (flags & BNXT_FLAG_TPA) == 0 ||
9852 (bp->flags & BNXT_FLAG_CHIP_P5))
9853 re_init = true;
9854 }
9855
9856 if (changes & ~BNXT_FLAG_TPA)
9857 re_init = true;
9858
9859 if (flags != bp->flags) {
9860 u32 old_flags = bp->flags;
9861
9862 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9863 bp->flags = flags;
9864 if (update_tpa)
9865 bnxt_set_ring_params(bp);
9866 return rc;
9867 }
9868
9869 if (re_init) {
9870 bnxt_close_nic(bp, false, false);
9871 bp->flags = flags;
9872 if (update_tpa)
9873 bnxt_set_ring_params(bp);
9874
9875 return bnxt_open_nic(bp, false, false);
9876 }
9877 if (update_tpa) {
9878 bp->flags = flags;
9879 rc = bnxt_set_tpa(bp,
9880 (flags & BNXT_FLAG_TPA) ?
9881 true : false);
9882 if (rc)
9883 bp->flags = old_flags;
9884 }
9885 }
9886 return rc;
9887 }
9888
9889 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
9890 u32 ring_id, u32 *prod, u32 *cons)
9891 {
9892 struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
9893 struct hwrm_dbg_ring_info_get_input req = {0};
9894 int rc;
9895
9896 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
9897 req.ring_type = ring_type;
9898 req.fw_ring_id = cpu_to_le32(ring_id);
9899 mutex_lock(&bp->hwrm_cmd_lock);
9900 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9901 if (!rc) {
9902 *prod = le32_to_cpu(resp->producer_index);
9903 *cons = le32_to_cpu(resp->consumer_index);
9904 }
9905 mutex_unlock(&bp->hwrm_cmd_lock);
9906 return rc;
9907 }
9908
9909 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
9910 {
9911 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
9912 int i = bnapi->index;
9913
9914 if (!txr)
9915 return;
9916
9917 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
9918 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
9919 txr->tx_cons);
9920 }
9921
9922 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
9923 {
9924 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
9925 int i = bnapi->index;
9926
9927 if (!rxr)
9928 return;
9929
9930 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
9931 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
9932 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
9933 rxr->rx_sw_agg_prod);
9934 }
9935
9936 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
9937 {
9938 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9939 int i = bnapi->index;
9940
9941 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
9942 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
9943 }
9944
9945 static void bnxt_dbg_dump_states(struct bnxt *bp)
9946 {
9947 int i;
9948 struct bnxt_napi *bnapi;
9949
9950 for (i = 0; i < bp->cp_nr_rings; i++) {
9951 bnapi = bp->bnapi[i];
9952 if (netif_msg_drv(bp)) {
9953 bnxt_dump_tx_sw_state(bnapi);
9954 bnxt_dump_rx_sw_state(bnapi);
9955 bnxt_dump_cp_sw_state(bnapi);
9956 }
9957 }
9958 }
9959
9960 static void bnxt_reset_task(struct bnxt *bp, bool silent)
9961 {
9962 if (!silent)
9963 bnxt_dbg_dump_states(bp);
9964 if (netif_running(bp->dev)) {
9965 int rc;
9966
9967 if (silent) {
9968 bnxt_close_nic(bp, false, false);
9969 bnxt_open_nic(bp, false, false);
9970 } else {
9971 bnxt_ulp_stop(bp);
9972 bnxt_close_nic(bp, true, false);
9973 rc = bnxt_open_nic(bp, true, false);
9974 bnxt_ulp_start(bp, rc);
9975 }
9976 }
9977 }
9978
9979 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
9980 {
9981 struct bnxt *bp = netdev_priv(dev);
9982
9983 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
9984 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
9985 bnxt_queue_sp_work(bp);
9986 }
9987
9988 static void bnxt_fw_health_check(struct bnxt *bp)
9989 {
9990 struct bnxt_fw_health *fw_health = bp->fw_health;
9991 u32 val;
9992
9993 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9994 return;
9995
9996 if (fw_health->tmr_counter) {
9997 fw_health->tmr_counter--;
9998 return;
9999 }
10000
10001 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10002 if (val == fw_health->last_fw_heartbeat)
10003 goto fw_reset;
10004
10005 fw_health->last_fw_heartbeat = val;
10006
10007 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10008 if (val != fw_health->last_fw_reset_cnt)
10009 goto fw_reset;
10010
10011 fw_health->tmr_counter = fw_health->tmr_multiplier;
10012 return;
10013
10014 fw_reset:
10015 set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10016 bnxt_queue_sp_work(bp);
10017 }
10018
10019 static void bnxt_timer(struct timer_list *t)
10020 {
10021 struct bnxt *bp = from_timer(bp, t, timer);
10022 struct net_device *dev = bp->dev;
10023
10024 if (!netif_running(dev))
10025 return;
10026
10027 if (atomic_read(&bp->intr_sem) != 0)
10028 goto bnxt_restart_timer;
10029
10030 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10031 bnxt_fw_health_check(bp);
10032
10033 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS) &&
10034 bp->stats_coal_ticks) {
10035 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10036 bnxt_queue_sp_work(bp);
10037 }
10038
10039 if (bnxt_tc_flower_enabled(bp)) {
10040 set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10041 bnxt_queue_sp_work(bp);
10042 }
10043
10044 #ifdef CONFIG_RFS_ACCEL
10045 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10046 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10047 bnxt_queue_sp_work(bp);
10048 }
10049 #endif /*CONFIG_RFS_ACCEL*/
10050
10051 if (bp->link_info.phy_retry) {
10052 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10053 bp->link_info.phy_retry = false;
10054 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10055 } else {
10056 set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10057 bnxt_queue_sp_work(bp);
10058 }
10059 }
10060
10061 if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10062 netif_carrier_ok(dev)) {
10063 set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10064 bnxt_queue_sp_work(bp);
10065 }
10066 bnxt_restart_timer:
10067 mod_timer(&bp->timer, jiffies + bp->current_interval);
10068 }
10069
10070 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10071 {
10072 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10073 * set. If the device is being closed, bnxt_close() may be holding
10074 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
10075 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10076 */
10077 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10078 rtnl_lock();
10079 }
10080
10081 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10082 {
10083 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10084 rtnl_unlock();
10085 }
10086
10087 /* Only called from bnxt_sp_task() */
10088 static void bnxt_reset(struct bnxt *bp, bool silent)
10089 {
10090 bnxt_rtnl_lock_sp(bp);
10091 if (test_bit(BNXT_STATE_OPEN, &bp->state))
10092 bnxt_reset_task(bp, silent);
10093 bnxt_rtnl_unlock_sp(bp);
10094 }
10095
10096 static void bnxt_fw_reset_close(struct bnxt *bp)
10097 {
10098 bnxt_ulp_stop(bp);
10099 __bnxt_close_nic(bp, true, false);
10100 bnxt_clear_int_mode(bp);
10101 bnxt_hwrm_func_drv_unrgtr(bp);
10102 bnxt_free_ctx_mem(bp);
10103 kfree(bp->ctx);
10104 bp->ctx = NULL;
10105 }
10106
10107 static bool is_bnxt_fw_ok(struct bnxt *bp)
10108 {
10109 struct bnxt_fw_health *fw_health = bp->fw_health;
10110 bool no_heartbeat = false, has_reset = false;
10111 u32 val;
10112
10113 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10114 if (val == fw_health->last_fw_heartbeat)
10115 no_heartbeat = true;
10116
10117 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10118 if (val != fw_health->last_fw_reset_cnt)
10119 has_reset = true;
10120
10121 if (!no_heartbeat && has_reset)
10122 return true;
10123
10124 return false;
10125 }
10126
10127 /* rtnl_lock is acquired before calling this function */
10128 static void bnxt_force_fw_reset(struct bnxt *bp)
10129 {
10130 struct bnxt_fw_health *fw_health = bp->fw_health;
10131 u32 wait_dsecs;
10132
10133 if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10134 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10135 return;
10136
10137 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10138 bnxt_fw_reset_close(bp);
10139 wait_dsecs = fw_health->master_func_wait_dsecs;
10140 if (fw_health->master) {
10141 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10142 wait_dsecs = 0;
10143 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10144 } else {
10145 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10146 wait_dsecs = fw_health->normal_func_wait_dsecs;
10147 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10148 }
10149
10150 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10151 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10152 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10153 }
10154
10155 void bnxt_fw_exception(struct bnxt *bp)
10156 {
10157 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10158 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10159 bnxt_rtnl_lock_sp(bp);
10160 bnxt_force_fw_reset(bp);
10161 bnxt_rtnl_unlock_sp(bp);
10162 }
10163
10164 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10165 * < 0 on error.
10166 */
10167 static int bnxt_get_registered_vfs(struct bnxt *bp)
10168 {
10169 #ifdef CONFIG_BNXT_SRIOV
10170 int rc;
10171
10172 if (!BNXT_PF(bp))
10173 return 0;
10174
10175 rc = bnxt_hwrm_func_qcfg(bp);
10176 if (rc) {
10177 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10178 return rc;
10179 }
10180 if (bp->pf.registered_vfs)
10181 return bp->pf.registered_vfs;
10182 if (bp->sriov_cfg)
10183 return 1;
10184 #endif
10185 return 0;
10186 }
10187
10188 void bnxt_fw_reset(struct bnxt *bp)
10189 {
10190 bnxt_rtnl_lock_sp(bp);
10191 if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10192 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10193 int n = 0, tmo;
10194
10195 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10196 if (bp->pf.active_vfs &&
10197 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10198 n = bnxt_get_registered_vfs(bp);
10199 if (n < 0) {
10200 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10201 n);
10202 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10203 dev_close(bp->dev);
10204 goto fw_reset_exit;
10205 } else if (n > 0) {
10206 u16 vf_tmo_dsecs = n * 10;
10207
10208 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10209 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10210 bp->fw_reset_state =
10211 BNXT_FW_RESET_STATE_POLL_VF;
10212 bnxt_queue_fw_reset_work(bp, HZ / 10);
10213 goto fw_reset_exit;
10214 }
10215 bnxt_fw_reset_close(bp);
10216 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10217 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10218 tmo = HZ / 10;
10219 } else {
10220 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10221 tmo = bp->fw_reset_min_dsecs * HZ / 10;
10222 }
10223 bnxt_queue_fw_reset_work(bp, tmo);
10224 }
10225 fw_reset_exit:
10226 bnxt_rtnl_unlock_sp(bp);
10227 }
10228
10229 static void bnxt_chk_missed_irq(struct bnxt *bp)
10230 {
10231 int i;
10232
10233 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10234 return;
10235
10236 for (i = 0; i < bp->cp_nr_rings; i++) {
10237 struct bnxt_napi *bnapi = bp->bnapi[i];
10238 struct bnxt_cp_ring_info *cpr;
10239 u32 fw_ring_id;
10240 int j;
10241
10242 if (!bnapi)
10243 continue;
10244
10245 cpr = &bnapi->cp_ring;
10246 for (j = 0; j < 2; j++) {
10247 struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10248 u32 val[2];
10249
10250 if (!cpr2 || cpr2->has_more_work ||
10251 !bnxt_has_work(bp, cpr2))
10252 continue;
10253
10254 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10255 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10256 continue;
10257 }
10258 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10259 bnxt_dbg_hwrm_ring_info_get(bp,
10260 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10261 fw_ring_id, &val[0], &val[1]);
10262 cpr->missed_irqs++;
10263 }
10264 }
10265 }
10266
10267 static void bnxt_cfg_ntp_filters(struct bnxt *);
10268
10269 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10270 {
10271 struct bnxt_link_info *link_info = &bp->link_info;
10272
10273 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10274 link_info->autoneg = BNXT_AUTONEG_SPEED;
10275 if (bp->hwrm_spec_code >= 0x10201) {
10276 if (link_info->auto_pause_setting &
10277 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10278 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10279 } else {
10280 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10281 }
10282 link_info->advertising = link_info->auto_link_speeds;
10283 } else {
10284 link_info->req_link_speed = link_info->force_link_speed;
10285 link_info->req_duplex = link_info->duplex_setting;
10286 }
10287 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10288 link_info->req_flow_ctrl =
10289 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10290 else
10291 link_info->req_flow_ctrl = link_info->force_pause_setting;
10292 }
10293
10294 static void bnxt_sp_task(struct work_struct *work)
10295 {
10296 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10297
10298 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10299 smp_mb__after_atomic();
10300 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10301 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10302 return;
10303 }
10304
10305 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
10306 bnxt_cfg_rx_mode(bp);
10307
10308 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
10309 bnxt_cfg_ntp_filters(bp);
10310 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
10311 bnxt_hwrm_exec_fwd_req(bp);
10312 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
10313 bnxt_hwrm_tunnel_dst_port_alloc(
10314 bp, bp->vxlan_port,
10315 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
10316 }
10317 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
10318 bnxt_hwrm_tunnel_dst_port_free(
10319 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
10320 }
10321 if (test_and_clear_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event)) {
10322 bnxt_hwrm_tunnel_dst_port_alloc(
10323 bp, bp->nge_port,
10324 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
10325 }
10326 if (test_and_clear_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event)) {
10327 bnxt_hwrm_tunnel_dst_port_free(
10328 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
10329 }
10330 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
10331 bnxt_hwrm_port_qstats(bp);
10332 bnxt_hwrm_port_qstats_ext(bp);
10333 bnxt_hwrm_pcie_qstats(bp);
10334 }
10335
10336 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
10337 int rc;
10338
10339 mutex_lock(&bp->link_lock);
10340 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
10341 &bp->sp_event))
10342 bnxt_hwrm_phy_qcaps(bp);
10343
10344 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
10345 &bp->sp_event))
10346 bnxt_init_ethtool_link_settings(bp);
10347
10348 rc = bnxt_update_link(bp, true);
10349 mutex_unlock(&bp->link_lock);
10350 if (rc)
10351 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
10352 rc);
10353 }
10354 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
10355 int rc;
10356
10357 mutex_lock(&bp->link_lock);
10358 rc = bnxt_update_phy_setting(bp);
10359 mutex_unlock(&bp->link_lock);
10360 if (rc) {
10361 netdev_warn(bp->dev, "update phy settings retry failed\n");
10362 } else {
10363 bp->link_info.phy_retry = false;
10364 netdev_info(bp->dev, "update phy settings retry succeeded\n");
10365 }
10366 }
10367 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
10368 mutex_lock(&bp->link_lock);
10369 bnxt_get_port_module_status(bp);
10370 mutex_unlock(&bp->link_lock);
10371 }
10372
10373 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
10374 bnxt_tc_flow_stats_work(bp);
10375
10376 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
10377 bnxt_chk_missed_irq(bp);
10378
10379 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
10380 * must be the last functions to be called before exiting.
10381 */
10382 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
10383 bnxt_reset(bp, false);
10384
10385 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
10386 bnxt_reset(bp, true);
10387
10388 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
10389 bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
10390
10391 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
10392 if (!is_bnxt_fw_ok(bp))
10393 bnxt_devlink_health_report(bp,
10394 BNXT_FW_EXCEPTION_SP_EVENT);
10395 }
10396
10397 smp_mb__before_atomic();
10398 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10399 }
10400
10401 /* Under rtnl_lock */
10402 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
10403 int tx_xdp)
10404 {
10405 int max_rx, max_tx, tx_sets = 1;
10406 int tx_rings_needed, stats;
10407 int rx_rings = rx;
10408 int cp, vnics, rc;
10409
10410 if (tcs)
10411 tx_sets = tcs;
10412
10413 rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
10414 if (rc)
10415 return rc;
10416
10417 if (max_rx < rx)
10418 return -ENOMEM;
10419
10420 tx_rings_needed = tx * tx_sets + tx_xdp;
10421 if (max_tx < tx_rings_needed)
10422 return -ENOMEM;
10423
10424 vnics = 1;
10425 if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
10426 vnics += rx_rings;
10427
10428 if (bp->flags & BNXT_FLAG_AGG_RINGS)
10429 rx_rings <<= 1;
10430 cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
10431 stats = cp;
10432 if (BNXT_NEW_RM(bp)) {
10433 cp += bnxt_get_ulp_msix_num(bp);
10434 stats += bnxt_get_ulp_stat_ctxs(bp);
10435 }
10436 return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
10437 stats, vnics);
10438 }
10439
10440 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
10441 {
10442 if (bp->bar2) {
10443 pci_iounmap(pdev, bp->bar2);
10444 bp->bar2 = NULL;
10445 }
10446
10447 if (bp->bar1) {
10448 pci_iounmap(pdev, bp->bar1);
10449 bp->bar1 = NULL;
10450 }
10451
10452 if (bp->bar0) {
10453 pci_iounmap(pdev, bp->bar0);
10454 bp->bar0 = NULL;
10455 }
10456 }
10457
10458 static void bnxt_cleanup_pci(struct bnxt *bp)
10459 {
10460 bnxt_unmap_bars(bp, bp->pdev);
10461 pci_release_regions(bp->pdev);
10462 if (pci_is_enabled(bp->pdev))
10463 pci_disable_device(bp->pdev);
10464 }
10465
10466 static void bnxt_init_dflt_coal(struct bnxt *bp)
10467 {
10468 struct bnxt_coal *coal;
10469
10470 /* Tick values in micro seconds.
10471 * 1 coal_buf x bufs_per_record = 1 completion record.
10472 */
10473 coal = &bp->rx_coal;
10474 coal->coal_ticks = 10;
10475 coal->coal_bufs = 30;
10476 coal->coal_ticks_irq = 1;
10477 coal->coal_bufs_irq = 2;
10478 coal->idle_thresh = 50;
10479 coal->bufs_per_record = 2;
10480 coal->budget = 64; /* NAPI budget */
10481
10482 coal = &bp->tx_coal;
10483 coal->coal_ticks = 28;
10484 coal->coal_bufs = 30;
10485 coal->coal_ticks_irq = 2;
10486 coal->coal_bufs_irq = 2;
10487 coal->bufs_per_record = 1;
10488
10489 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
10490 }
10491
10492 static void bnxt_alloc_fw_health(struct bnxt *bp)
10493 {
10494 if (bp->fw_health)
10495 return;
10496
10497 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
10498 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
10499 return;
10500
10501 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
10502 if (!bp->fw_health) {
10503 netdev_warn(bp->dev, "Failed to allocate fw_health\n");
10504 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
10505 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
10506 }
10507 }
10508
10509 static int bnxt_fw_init_one_p1(struct bnxt *bp)
10510 {
10511 int rc;
10512
10513 bp->fw_cap = 0;
10514 rc = bnxt_hwrm_ver_get(bp);
10515 if (rc)
10516 return rc;
10517
10518 if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10519 rc = bnxt_alloc_kong_hwrm_resources(bp);
10520 if (rc)
10521 bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10522 }
10523
10524 if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10525 bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10526 rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10527 if (rc)
10528 return rc;
10529 }
10530 rc = bnxt_hwrm_func_reset(bp);
10531 if (rc)
10532 return -ENODEV;
10533
10534 bnxt_hwrm_fw_set_time(bp);
10535 return 0;
10536 }
10537
10538 static int bnxt_fw_init_one_p2(struct bnxt *bp)
10539 {
10540 int rc;
10541
10542 /* Get the MAX capabilities for this function */
10543 rc = bnxt_hwrm_func_qcaps(bp);
10544 if (rc) {
10545 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10546 rc);
10547 return -ENODEV;
10548 }
10549
10550 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
10551 if (rc)
10552 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
10553 rc);
10554
10555 bnxt_alloc_fw_health(bp);
10556 rc = bnxt_hwrm_error_recovery_qcfg(bp);
10557 if (rc)
10558 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
10559 rc);
10560
10561 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
10562 if (rc)
10563 return -ENODEV;
10564
10565 bnxt_hwrm_func_qcfg(bp);
10566 bnxt_hwrm_vnic_qcaps(bp);
10567 bnxt_hwrm_port_led_qcaps(bp);
10568 bnxt_ethtool_init(bp);
10569 bnxt_dcb_init(bp);
10570 return 0;
10571 }
10572
10573 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
10574 {
10575 bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
10576 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10577 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10578 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10579 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10580 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
10581 bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10582 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10583 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10584 }
10585 }
10586
10587 static void bnxt_set_dflt_rfs(struct bnxt *bp)
10588 {
10589 struct net_device *dev = bp->dev;
10590
10591 dev->hw_features &= ~NETIF_F_NTUPLE;
10592 dev->features &= ~NETIF_F_NTUPLE;
10593 bp->flags &= ~BNXT_FLAG_RFS;
10594 if (bnxt_rfs_supported(bp)) {
10595 dev->hw_features |= NETIF_F_NTUPLE;
10596 if (bnxt_rfs_capable(bp)) {
10597 bp->flags |= BNXT_FLAG_RFS;
10598 dev->features |= NETIF_F_NTUPLE;
10599 }
10600 }
10601 }
10602
10603 static void bnxt_fw_init_one_p3(struct bnxt *bp)
10604 {
10605 struct pci_dev *pdev = bp->pdev;
10606
10607 bnxt_set_dflt_rss_hash_type(bp);
10608 bnxt_set_dflt_rfs(bp);
10609
10610 bnxt_get_wol_settings(bp);
10611 if (bp->flags & BNXT_FLAG_WOL_CAP)
10612 device_set_wakeup_enable(&pdev->dev, bp->wol);
10613 else
10614 device_set_wakeup_capable(&pdev->dev, false);
10615
10616 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
10617 bnxt_hwrm_coal_params_qcaps(bp);
10618 }
10619
10620 static int bnxt_fw_init_one(struct bnxt *bp)
10621 {
10622 int rc;
10623
10624 rc = bnxt_fw_init_one_p1(bp);
10625 if (rc) {
10626 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
10627 return rc;
10628 }
10629 rc = bnxt_fw_init_one_p2(bp);
10630 if (rc) {
10631 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
10632 return rc;
10633 }
10634 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
10635 if (rc)
10636 return rc;
10637
10638 /* In case fw capabilities have changed, destroy the unneeded
10639 * reporters and create newly capable ones.
10640 */
10641 bnxt_dl_fw_reporters_destroy(bp, false);
10642 bnxt_dl_fw_reporters_create(bp);
10643 bnxt_fw_init_one_p3(bp);
10644 return 0;
10645 }
10646
10647 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
10648 {
10649 struct bnxt_fw_health *fw_health = bp->fw_health;
10650 u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
10651 u32 val = fw_health->fw_reset_seq_vals[reg_idx];
10652 u32 reg_type, reg_off, delay_msecs;
10653
10654 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
10655 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
10656 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
10657 switch (reg_type) {
10658 case BNXT_FW_HEALTH_REG_TYPE_CFG:
10659 pci_write_config_dword(bp->pdev, reg_off, val);
10660 break;
10661 case BNXT_FW_HEALTH_REG_TYPE_GRC:
10662 writel(reg_off & BNXT_GRC_BASE_MASK,
10663 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
10664 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
10665 /* fall through */
10666 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
10667 writel(val, bp->bar0 + reg_off);
10668 break;
10669 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
10670 writel(val, bp->bar1 + reg_off);
10671 break;
10672 }
10673 if (delay_msecs) {
10674 pci_read_config_dword(bp->pdev, 0, &val);
10675 msleep(delay_msecs);
10676 }
10677 }
10678
10679 static void bnxt_reset_all(struct bnxt *bp)
10680 {
10681 struct bnxt_fw_health *fw_health = bp->fw_health;
10682 int i, rc;
10683
10684 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10685 #ifdef CONFIG_TEE_BNXT_FW
10686 rc = tee_bnxt_fw_load();
10687 if (rc)
10688 netdev_err(bp->dev, "Unable to reset FW rc=%d\n", rc);
10689 bp->fw_reset_timestamp = jiffies;
10690 #endif
10691 return;
10692 }
10693
10694 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
10695 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
10696 bnxt_fw_reset_writel(bp, i);
10697 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
10698 struct hwrm_fw_reset_input req = {0};
10699
10700 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
10701 req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
10702 req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
10703 req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
10704 req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
10705 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10706 if (rc)
10707 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
10708 }
10709 bp->fw_reset_timestamp = jiffies;
10710 }
10711
10712 static void bnxt_fw_reset_task(struct work_struct *work)
10713 {
10714 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
10715 int rc;
10716
10717 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10718 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
10719 return;
10720 }
10721
10722 switch (bp->fw_reset_state) {
10723 case BNXT_FW_RESET_STATE_POLL_VF: {
10724 int n = bnxt_get_registered_vfs(bp);
10725 int tmo;
10726
10727 if (n < 0) {
10728 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
10729 n, jiffies_to_msecs(jiffies -
10730 bp->fw_reset_timestamp));
10731 goto fw_reset_abort;
10732 } else if (n > 0) {
10733 if (time_after(jiffies, bp->fw_reset_timestamp +
10734 (bp->fw_reset_max_dsecs * HZ / 10))) {
10735 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10736 bp->fw_reset_state = 0;
10737 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
10738 n);
10739 return;
10740 }
10741 bnxt_queue_fw_reset_work(bp, HZ / 10);
10742 return;
10743 }
10744 bp->fw_reset_timestamp = jiffies;
10745 rtnl_lock();
10746 bnxt_fw_reset_close(bp);
10747 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10748 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10749 tmo = HZ / 10;
10750 } else {
10751 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10752 tmo = bp->fw_reset_min_dsecs * HZ / 10;
10753 }
10754 rtnl_unlock();
10755 bnxt_queue_fw_reset_work(bp, tmo);
10756 return;
10757 }
10758 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
10759 u32 val;
10760
10761 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
10762 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
10763 !time_after(jiffies, bp->fw_reset_timestamp +
10764 (bp->fw_reset_max_dsecs * HZ / 10))) {
10765 bnxt_queue_fw_reset_work(bp, HZ / 5);
10766 return;
10767 }
10768
10769 if (!bp->fw_health->master) {
10770 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
10771
10772 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10773 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10774 return;
10775 }
10776 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10777 }
10778 /* fall through */
10779 case BNXT_FW_RESET_STATE_RESET_FW:
10780 bnxt_reset_all(bp);
10781 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10782 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
10783 return;
10784 case BNXT_FW_RESET_STATE_ENABLE_DEV:
10785 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
10786 u32 val;
10787
10788 val = bnxt_fw_health_readl(bp,
10789 BNXT_FW_RESET_INPROG_REG);
10790 if (val)
10791 netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
10792 val);
10793 }
10794 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10795 if (pci_enable_device(bp->pdev)) {
10796 netdev_err(bp->dev, "Cannot re-enable PCI device\n");
10797 goto fw_reset_abort;
10798 }
10799 pci_set_master(bp->pdev);
10800 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
10801 /* fall through */
10802 case BNXT_FW_RESET_STATE_POLL_FW:
10803 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
10804 rc = __bnxt_hwrm_ver_get(bp, true);
10805 if (rc) {
10806 if (time_after(jiffies, bp->fw_reset_timestamp +
10807 (bp->fw_reset_max_dsecs * HZ / 10))) {
10808 netdev_err(bp->dev, "Firmware reset aborted\n");
10809 goto fw_reset_abort;
10810 }
10811 bnxt_queue_fw_reset_work(bp, HZ / 5);
10812 return;
10813 }
10814 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
10815 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
10816 /* fall through */
10817 case BNXT_FW_RESET_STATE_OPENING:
10818 while (!rtnl_trylock()) {
10819 bnxt_queue_fw_reset_work(bp, HZ / 10);
10820 return;
10821 }
10822 rc = bnxt_open(bp->dev);
10823 if (rc) {
10824 netdev_err(bp->dev, "bnxt_open_nic() failed\n");
10825 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10826 dev_close(bp->dev);
10827 }
10828
10829 bp->fw_reset_state = 0;
10830 /* Make sure fw_reset_state is 0 before clearing the flag */
10831 smp_mb__before_atomic();
10832 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10833 bnxt_ulp_start(bp, rc);
10834 bnxt_dl_health_recovery_done(bp);
10835 bnxt_dl_health_status_update(bp, true);
10836 rtnl_unlock();
10837 break;
10838 }
10839 return;
10840
10841 fw_reset_abort:
10842 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10843 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
10844 bnxt_dl_health_status_update(bp, false);
10845 bp->fw_reset_state = 0;
10846 rtnl_lock();
10847 dev_close(bp->dev);
10848 rtnl_unlock();
10849 }
10850
10851 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
10852 {
10853 int rc;
10854 struct bnxt *bp = netdev_priv(dev);
10855
10856 SET_NETDEV_DEV(dev, &pdev->dev);
10857
10858 /* enable device (incl. PCI PM wakeup), and bus-mastering */
10859 rc = pci_enable_device(pdev);
10860 if (rc) {
10861 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
10862 goto init_err;
10863 }
10864
10865 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
10866 dev_err(&pdev->dev,
10867 "Cannot find PCI device base address, aborting\n");
10868 rc = -ENODEV;
10869 goto init_err_disable;
10870 }
10871
10872 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
10873 if (rc) {
10874 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
10875 goto init_err_disable;
10876 }
10877
10878 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
10879 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
10880 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
10881 goto init_err_disable;
10882 }
10883
10884 pci_set_master(pdev);
10885
10886 bp->dev = dev;
10887 bp->pdev = pdev;
10888
10889 bp->bar0 = pci_ioremap_bar(pdev, 0);
10890 if (!bp->bar0) {
10891 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
10892 rc = -ENOMEM;
10893 goto init_err_release;
10894 }
10895
10896 bp->bar1 = pci_ioremap_bar(pdev, 2);
10897 if (!bp->bar1) {
10898 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
10899 rc = -ENOMEM;
10900 goto init_err_release;
10901 }
10902
10903 bp->bar2 = pci_ioremap_bar(pdev, 4);
10904 if (!bp->bar2) {
10905 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
10906 rc = -ENOMEM;
10907 goto init_err_release;
10908 }
10909
10910 pci_enable_pcie_error_reporting(pdev);
10911
10912 INIT_WORK(&bp->sp_task, bnxt_sp_task);
10913 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
10914
10915 spin_lock_init(&bp->ntp_fltr_lock);
10916 #if BITS_PER_LONG == 32
10917 spin_lock_init(&bp->db_lock);
10918 #endif
10919
10920 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
10921 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
10922
10923 bnxt_init_dflt_coal(bp);
10924
10925 timer_setup(&bp->timer, bnxt_timer, 0);
10926 bp->current_interval = BNXT_TIMER_INTERVAL;
10927
10928 clear_bit(BNXT_STATE_OPEN, &bp->state);
10929 return 0;
10930
10931 init_err_release:
10932 bnxt_unmap_bars(bp, pdev);
10933 pci_release_regions(pdev);
10934
10935 init_err_disable:
10936 pci_disable_device(pdev);
10937
10938 init_err:
10939 return rc;
10940 }
10941
10942 /* rtnl_lock held */
10943 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
10944 {
10945 struct sockaddr *addr = p;
10946 struct bnxt *bp = netdev_priv(dev);
10947 int rc = 0;
10948
10949 if (!is_valid_ether_addr(addr->sa_data))
10950 return -EADDRNOTAVAIL;
10951
10952 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
10953 return 0;
10954
10955 rc = bnxt_approve_mac(bp, addr->sa_data, true);
10956 if (rc)
10957 return rc;
10958
10959 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
10960 if (netif_running(dev)) {
10961 bnxt_close_nic(bp, false, false);
10962 rc = bnxt_open_nic(bp, false, false);
10963 }
10964
10965 return rc;
10966 }
10967
10968 /* rtnl_lock held */
10969 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
10970 {
10971 struct bnxt *bp = netdev_priv(dev);
10972
10973 if (netif_running(dev))
10974 bnxt_close_nic(bp, false, false);
10975
10976 dev->mtu = new_mtu;
10977 bnxt_set_ring_params(bp);
10978
10979 if (netif_running(dev))
10980 return bnxt_open_nic(bp, false, false);
10981
10982 return 0;
10983 }
10984
10985 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
10986 {
10987 struct bnxt *bp = netdev_priv(dev);
10988 bool sh = false;
10989 int rc;
10990
10991 if (tc > bp->max_tc) {
10992 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
10993 tc, bp->max_tc);
10994 return -EINVAL;
10995 }
10996
10997 if (netdev_get_num_tc(dev) == tc)
10998 return 0;
10999
11000 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11001 sh = true;
11002
11003 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11004 sh, tc, bp->tx_nr_rings_xdp);
11005 if (rc)
11006 return rc;
11007
11008 /* Needs to close the device and do hw resource re-allocations */
11009 if (netif_running(bp->dev))
11010 bnxt_close_nic(bp, true, false);
11011
11012 if (tc) {
11013 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11014 netdev_set_num_tc(dev, tc);
11015 } else {
11016 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11017 netdev_reset_tc(dev);
11018 }
11019 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11020 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11021 bp->tx_nr_rings + bp->rx_nr_rings;
11022
11023 if (netif_running(bp->dev))
11024 return bnxt_open_nic(bp, true, false);
11025
11026 return 0;
11027 }
11028
11029 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11030 void *cb_priv)
11031 {
11032 struct bnxt *bp = cb_priv;
11033
11034 if (!bnxt_tc_flower_enabled(bp) ||
11035 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11036 return -EOPNOTSUPP;
11037
11038 switch (type) {
11039 case TC_SETUP_CLSFLOWER:
11040 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11041 default:
11042 return -EOPNOTSUPP;
11043 }
11044 }
11045
11046 LIST_HEAD(bnxt_block_cb_list);
11047
11048 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11049 void *type_data)
11050 {
11051 struct bnxt *bp = netdev_priv(dev);
11052
11053 switch (type) {
11054 case TC_SETUP_BLOCK:
11055 return flow_block_cb_setup_simple(type_data,
11056 &bnxt_block_cb_list,
11057 bnxt_setup_tc_block_cb,
11058 bp, bp, true);
11059 case TC_SETUP_QDISC_MQPRIO: {
11060 struct tc_mqprio_qopt *mqprio = type_data;
11061
11062 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11063
11064 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11065 }
11066 default:
11067 return -EOPNOTSUPP;
11068 }
11069 }
11070
11071 #ifdef CONFIG_RFS_ACCEL
11072 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11073 struct bnxt_ntuple_filter *f2)
11074 {
11075 struct flow_keys *keys1 = &f1->fkeys;
11076 struct flow_keys *keys2 = &f2->fkeys;
11077
11078 if (keys1->basic.n_proto != keys2->basic.n_proto ||
11079 keys1->basic.ip_proto != keys2->basic.ip_proto)
11080 return false;
11081
11082 if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11083 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11084 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11085 return false;
11086 } else {
11087 if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11088 sizeof(keys1->addrs.v6addrs.src)) ||
11089 memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11090 sizeof(keys1->addrs.v6addrs.dst)))
11091 return false;
11092 }
11093
11094 if (keys1->ports.ports == keys2->ports.ports &&
11095 keys1->control.flags == keys2->control.flags &&
11096 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11097 ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11098 return true;
11099
11100 return false;
11101 }
11102
11103 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11104 u16 rxq_index, u32 flow_id)
11105 {
11106 struct bnxt *bp = netdev_priv(dev);
11107 struct bnxt_ntuple_filter *fltr, *new_fltr;
11108 struct flow_keys *fkeys;
11109 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11110 int rc = 0, idx, bit_id, l2_idx = 0;
11111 struct hlist_head *head;
11112 u32 flags;
11113
11114 if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11115 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11116 int off = 0, j;
11117
11118 netif_addr_lock_bh(dev);
11119 for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11120 if (ether_addr_equal(eth->h_dest,
11121 vnic->uc_list + off)) {
11122 l2_idx = j + 1;
11123 break;
11124 }
11125 }
11126 netif_addr_unlock_bh(dev);
11127 if (!l2_idx)
11128 return -EINVAL;
11129 }
11130 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11131 if (!new_fltr)
11132 return -ENOMEM;
11133
11134 fkeys = &new_fltr->fkeys;
11135 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11136 rc = -EPROTONOSUPPORT;
11137 goto err_free;
11138 }
11139
11140 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11141 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11142 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11143 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11144 rc = -EPROTONOSUPPORT;
11145 goto err_free;
11146 }
11147 if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11148 bp->hwrm_spec_code < 0x10601) {
11149 rc = -EPROTONOSUPPORT;
11150 goto err_free;
11151 }
11152 flags = fkeys->control.flags;
11153 if (((flags & FLOW_DIS_ENCAPSULATION) &&
11154 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11155 rc = -EPROTONOSUPPORT;
11156 goto err_free;
11157 }
11158
11159 memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11160 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11161
11162 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11163 head = &bp->ntp_fltr_hash_tbl[idx];
11164 rcu_read_lock();
11165 hlist_for_each_entry_rcu(fltr, head, hash) {
11166 if (bnxt_fltr_match(fltr, new_fltr)) {
11167 rcu_read_unlock();
11168 rc = 0;
11169 goto err_free;
11170 }
11171 }
11172 rcu_read_unlock();
11173
11174 spin_lock_bh(&bp->ntp_fltr_lock);
11175 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11176 BNXT_NTP_FLTR_MAX_FLTR, 0);
11177 if (bit_id < 0) {
11178 spin_unlock_bh(&bp->ntp_fltr_lock);
11179 rc = -ENOMEM;
11180 goto err_free;
11181 }
11182
11183 new_fltr->sw_id = (u16)bit_id;
11184 new_fltr->flow_id = flow_id;
11185 new_fltr->l2_fltr_idx = l2_idx;
11186 new_fltr->rxq = rxq_index;
11187 hlist_add_head_rcu(&new_fltr->hash, head);
11188 bp->ntp_fltr_count++;
11189 spin_unlock_bh(&bp->ntp_fltr_lock);
11190
11191 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11192 bnxt_queue_sp_work(bp);
11193
11194 return new_fltr->sw_id;
11195
11196 err_free:
11197 kfree(new_fltr);
11198 return rc;
11199 }
11200
11201 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11202 {
11203 int i;
11204
11205 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11206 struct hlist_head *head;
11207 struct hlist_node *tmp;
11208 struct bnxt_ntuple_filter *fltr;
11209 int rc;
11210
11211 head = &bp->ntp_fltr_hash_tbl[i];
11212 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11213 bool del = false;
11214
11215 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11216 if (rps_may_expire_flow(bp->dev, fltr->rxq,
11217 fltr->flow_id,
11218 fltr->sw_id)) {
11219 bnxt_hwrm_cfa_ntuple_filter_free(bp,
11220 fltr);
11221 del = true;
11222 }
11223 } else {
11224 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11225 fltr);
11226 if (rc)
11227 del = true;
11228 else
11229 set_bit(BNXT_FLTR_VALID, &fltr->state);
11230 }
11231
11232 if (del) {
11233 spin_lock_bh(&bp->ntp_fltr_lock);
11234 hlist_del_rcu(&fltr->hash);
11235 bp->ntp_fltr_count--;
11236 spin_unlock_bh(&bp->ntp_fltr_lock);
11237 synchronize_rcu();
11238 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11239 kfree(fltr);
11240 }
11241 }
11242 }
11243 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11244 netdev_info(bp->dev, "Receive PF driver unload event!");
11245 }
11246
11247 #else
11248
11249 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11250 {
11251 }
11252
11253 #endif /* CONFIG_RFS_ACCEL */
11254
11255 static void bnxt_udp_tunnel_add(struct net_device *dev,
11256 struct udp_tunnel_info *ti)
11257 {
11258 struct bnxt *bp = netdev_priv(dev);
11259
11260 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
11261 return;
11262
11263 if (!netif_running(dev))
11264 return;
11265
11266 switch (ti->type) {
11267 case UDP_TUNNEL_TYPE_VXLAN:
11268 if (bp->vxlan_port_cnt && bp->vxlan_port != ti->port)
11269 return;
11270
11271 bp->vxlan_port_cnt++;
11272 if (bp->vxlan_port_cnt == 1) {
11273 bp->vxlan_port = ti->port;
11274 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
11275 bnxt_queue_sp_work(bp);
11276 }
11277 break;
11278 case UDP_TUNNEL_TYPE_GENEVE:
11279 if (bp->nge_port_cnt && bp->nge_port != ti->port)
11280 return;
11281
11282 bp->nge_port_cnt++;
11283 if (bp->nge_port_cnt == 1) {
11284 bp->nge_port = ti->port;
11285 set_bit(BNXT_GENEVE_ADD_PORT_SP_EVENT, &bp->sp_event);
11286 }
11287 break;
11288 default:
11289 return;
11290 }
11291
11292 bnxt_queue_sp_work(bp);
11293 }
11294
11295 static void bnxt_udp_tunnel_del(struct net_device *dev,
11296 struct udp_tunnel_info *ti)
11297 {
11298 struct bnxt *bp = netdev_priv(dev);
11299
11300 if (ti->sa_family != AF_INET6 && ti->sa_family != AF_INET)
11301 return;
11302
11303 if (!netif_running(dev))
11304 return;
11305
11306 switch (ti->type) {
11307 case UDP_TUNNEL_TYPE_VXLAN:
11308 if (!bp->vxlan_port_cnt || bp->vxlan_port != ti->port)
11309 return;
11310 bp->vxlan_port_cnt--;
11311
11312 if (bp->vxlan_port_cnt != 0)
11313 return;
11314
11315 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
11316 break;
11317 case UDP_TUNNEL_TYPE_GENEVE:
11318 if (!bp->nge_port_cnt || bp->nge_port != ti->port)
11319 return;
11320 bp->nge_port_cnt--;
11321
11322 if (bp->nge_port_cnt != 0)
11323 return;
11324
11325 set_bit(BNXT_GENEVE_DEL_PORT_SP_EVENT, &bp->sp_event);
11326 break;
11327 default:
11328 return;
11329 }
11330
11331 bnxt_queue_sp_work(bp);
11332 }
11333
11334 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11335 struct net_device *dev, u32 filter_mask,
11336 int nlflags)
11337 {
11338 struct bnxt *bp = netdev_priv(dev);
11339
11340 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11341 nlflags, filter_mask, NULL);
11342 }
11343
11344 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
11345 u16 flags, struct netlink_ext_ack *extack)
11346 {
11347 struct bnxt *bp = netdev_priv(dev);
11348 struct nlattr *attr, *br_spec;
11349 int rem, rc = 0;
11350
11351 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
11352 return -EOPNOTSUPP;
11353
11354 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11355 if (!br_spec)
11356 return -EINVAL;
11357
11358 nla_for_each_nested(attr, br_spec, rem) {
11359 u16 mode;
11360
11361 if (nla_type(attr) != IFLA_BRIDGE_MODE)
11362 continue;
11363
11364 if (nla_len(attr) < sizeof(mode))
11365 return -EINVAL;
11366
11367 mode = nla_get_u16(attr);
11368 if (mode == bp->br_mode)
11369 break;
11370
11371 rc = bnxt_hwrm_set_br_mode(bp, mode);
11372 if (!rc)
11373 bp->br_mode = mode;
11374 break;
11375 }
11376 return rc;
11377 }
11378
11379 int bnxt_get_port_parent_id(struct net_device *dev,
11380 struct netdev_phys_item_id *ppid)
11381 {
11382 struct bnxt *bp = netdev_priv(dev);
11383
11384 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
11385 return -EOPNOTSUPP;
11386
11387 /* The PF and it's VF-reps only support the switchdev framework */
11388 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
11389 return -EOPNOTSUPP;
11390
11391 ppid->id_len = sizeof(bp->dsn);
11392 memcpy(ppid->id, bp->dsn, ppid->id_len);
11393
11394 return 0;
11395 }
11396
11397 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
11398 {
11399 struct bnxt *bp = netdev_priv(dev);
11400
11401 return &bp->dl_port;
11402 }
11403
11404 static const struct net_device_ops bnxt_netdev_ops = {
11405 .ndo_open = bnxt_open,
11406 .ndo_start_xmit = bnxt_start_xmit,
11407 .ndo_stop = bnxt_close,
11408 .ndo_get_stats64 = bnxt_get_stats64,
11409 .ndo_set_rx_mode = bnxt_set_rx_mode,
11410 .ndo_do_ioctl = bnxt_ioctl,
11411 .ndo_validate_addr = eth_validate_addr,
11412 .ndo_set_mac_address = bnxt_change_mac_addr,
11413 .ndo_change_mtu = bnxt_change_mtu,
11414 .ndo_fix_features = bnxt_fix_features,
11415 .ndo_set_features = bnxt_set_features,
11416 .ndo_tx_timeout = bnxt_tx_timeout,
11417 #ifdef CONFIG_BNXT_SRIOV
11418 .ndo_get_vf_config = bnxt_get_vf_config,
11419 .ndo_set_vf_mac = bnxt_set_vf_mac,
11420 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
11421 .ndo_set_vf_rate = bnxt_set_vf_bw,
11422 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
11423 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
11424 .ndo_set_vf_trust = bnxt_set_vf_trust,
11425 #endif
11426 .ndo_setup_tc = bnxt_setup_tc,
11427 #ifdef CONFIG_RFS_ACCEL
11428 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
11429 #endif
11430 .ndo_udp_tunnel_add = bnxt_udp_tunnel_add,
11431 .ndo_udp_tunnel_del = bnxt_udp_tunnel_del,
11432 .ndo_bpf = bnxt_xdp,
11433 .ndo_xdp_xmit = bnxt_xdp_xmit,
11434 .ndo_bridge_getlink = bnxt_bridge_getlink,
11435 .ndo_bridge_setlink = bnxt_bridge_setlink,
11436 .ndo_get_devlink_port = bnxt_get_devlink_port,
11437 };
11438
11439 static void bnxt_remove_one(struct pci_dev *pdev)
11440 {
11441 struct net_device *dev = pci_get_drvdata(pdev);
11442 struct bnxt *bp = netdev_priv(dev);
11443
11444 if (BNXT_PF(bp))
11445 bnxt_sriov_disable(bp);
11446
11447 bnxt_dl_fw_reporters_destroy(bp, true);
11448 pci_disable_pcie_error_reporting(pdev);
11449 unregister_netdev(dev);
11450 bnxt_dl_unregister(bp);
11451 bnxt_shutdown_tc(bp);
11452 bnxt_cancel_sp_work(bp);
11453 bp->sp_event = 0;
11454
11455 bnxt_clear_int_mode(bp);
11456 bnxt_hwrm_func_drv_unrgtr(bp);
11457 bnxt_free_hwrm_resources(bp);
11458 bnxt_free_hwrm_short_cmd_req(bp);
11459 bnxt_ethtool_free(bp);
11460 bnxt_dcb_free(bp);
11461 kfree(bp->edev);
11462 bp->edev = NULL;
11463 kfree(bp->fw_health);
11464 bp->fw_health = NULL;
11465 bnxt_cleanup_pci(bp);
11466 bnxt_free_ctx_mem(bp);
11467 kfree(bp->ctx);
11468 bp->ctx = NULL;
11469 bnxt_free_port_stats(bp);
11470 free_netdev(dev);
11471 }
11472
11473 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
11474 {
11475 int rc = 0;
11476 struct bnxt_link_info *link_info = &bp->link_info;
11477
11478 rc = bnxt_hwrm_phy_qcaps(bp);
11479 if (rc) {
11480 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
11481 rc);
11482 return rc;
11483 }
11484 if (!fw_dflt)
11485 return 0;
11486
11487 rc = bnxt_update_link(bp, false);
11488 if (rc) {
11489 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
11490 rc);
11491 return rc;
11492 }
11493
11494 /* Older firmware does not have supported_auto_speeds, so assume
11495 * that all supported speeds can be autonegotiated.
11496 */
11497 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
11498 link_info->support_auto_speeds = link_info->support_speeds;
11499
11500 bnxt_init_ethtool_link_settings(bp);
11501 return 0;
11502 }
11503
11504 static int bnxt_get_max_irq(struct pci_dev *pdev)
11505 {
11506 u16 ctrl;
11507
11508 if (!pdev->msix_cap)
11509 return 1;
11510
11511 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
11512 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
11513 }
11514
11515 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11516 int *max_cp)
11517 {
11518 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11519 int max_ring_grps = 0, max_irq;
11520
11521 *max_tx = hw_resc->max_tx_rings;
11522 *max_rx = hw_resc->max_rx_rings;
11523 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
11524 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
11525 bnxt_get_ulp_msix_num(bp),
11526 hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
11527 if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11528 *max_cp = min_t(int, *max_cp, max_irq);
11529 max_ring_grps = hw_resc->max_hw_ring_grps;
11530 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
11531 *max_cp -= 1;
11532 *max_rx -= 2;
11533 }
11534 if (bp->flags & BNXT_FLAG_AGG_RINGS)
11535 *max_rx >>= 1;
11536 if (bp->flags & BNXT_FLAG_CHIP_P5) {
11537 bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
11538 /* On P5 chips, max_cp output param should be available NQs */
11539 *max_cp = max_irq;
11540 }
11541 *max_rx = min_t(int, *max_rx, max_ring_grps);
11542 }
11543
11544 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
11545 {
11546 int rx, tx, cp;
11547
11548 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
11549 *max_rx = rx;
11550 *max_tx = tx;
11551 if (!rx || !tx || !cp)
11552 return -ENOMEM;
11553
11554 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
11555 }
11556
11557 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11558 bool shared)
11559 {
11560 int rc;
11561
11562 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11563 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
11564 /* Not enough rings, try disabling agg rings. */
11565 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
11566 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11567 if (rc) {
11568 /* set BNXT_FLAG_AGG_RINGS back for consistency */
11569 bp->flags |= BNXT_FLAG_AGG_RINGS;
11570 return rc;
11571 }
11572 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
11573 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11574 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11575 bnxt_set_ring_params(bp);
11576 }
11577
11578 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
11579 int max_cp, max_stat, max_irq;
11580
11581 /* Reserve minimum resources for RoCE */
11582 max_cp = bnxt_get_max_func_cp_rings(bp);
11583 max_stat = bnxt_get_max_func_stat_ctxs(bp);
11584 max_irq = bnxt_get_max_func_irqs(bp);
11585 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
11586 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
11587 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
11588 return 0;
11589
11590 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
11591 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
11592 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
11593 max_cp = min_t(int, max_cp, max_irq);
11594 max_cp = min_t(int, max_cp, max_stat);
11595 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
11596 if (rc)
11597 rc = 0;
11598 }
11599 return rc;
11600 }
11601
11602 /* In initial default shared ring setting, each shared ring must have a
11603 * RX/TX ring pair.
11604 */
11605 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
11606 {
11607 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
11608 bp->rx_nr_rings = bp->cp_nr_rings;
11609 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
11610 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11611 }
11612
11613 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
11614 {
11615 int dflt_rings, max_rx_rings, max_tx_rings, rc;
11616
11617 if (!bnxt_can_reserve_rings(bp))
11618 return 0;
11619
11620 if (sh)
11621 bp->flags |= BNXT_FLAG_SHARED_RINGS;
11622 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
11623 /* Reduce default rings on multi-port cards so that total default
11624 * rings do not exceed CPU count.
11625 */
11626 if (bp->port_count > 1) {
11627 int max_rings =
11628 max_t(int, num_online_cpus() / bp->port_count, 1);
11629
11630 dflt_rings = min_t(int, dflt_rings, max_rings);
11631 }
11632 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
11633 if (rc)
11634 return rc;
11635 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
11636 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
11637 if (sh)
11638 bnxt_trim_dflt_sh_rings(bp);
11639 else
11640 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
11641 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11642
11643 rc = __bnxt_reserve_rings(bp);
11644 if (rc)
11645 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
11646 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11647 if (sh)
11648 bnxt_trim_dflt_sh_rings(bp);
11649
11650 /* Rings may have been trimmed, re-reserve the trimmed rings. */
11651 if (bnxt_need_reserve_rings(bp)) {
11652 rc = __bnxt_reserve_rings(bp);
11653 if (rc)
11654 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
11655 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11656 }
11657 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
11658 bp->rx_nr_rings++;
11659 bp->cp_nr_rings++;
11660 }
11661 return rc;
11662 }
11663
11664 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
11665 {
11666 int rc;
11667
11668 if (bp->tx_nr_rings)
11669 return 0;
11670
11671 bnxt_ulp_irq_stop(bp);
11672 bnxt_clear_int_mode(bp);
11673 rc = bnxt_set_dflt_rings(bp, true);
11674 if (rc) {
11675 netdev_err(bp->dev, "Not enough rings available.\n");
11676 goto init_dflt_ring_err;
11677 }
11678 rc = bnxt_init_int_mode(bp);
11679 if (rc)
11680 goto init_dflt_ring_err;
11681
11682 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11683 if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
11684 bp->flags |= BNXT_FLAG_RFS;
11685 bp->dev->features |= NETIF_F_NTUPLE;
11686 }
11687 init_dflt_ring_err:
11688 bnxt_ulp_irq_restart(bp, rc);
11689 return rc;
11690 }
11691
11692 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
11693 {
11694 int rc;
11695
11696 ASSERT_RTNL();
11697 bnxt_hwrm_func_qcaps(bp);
11698
11699 if (netif_running(bp->dev))
11700 __bnxt_close_nic(bp, true, false);
11701
11702 bnxt_ulp_irq_stop(bp);
11703 bnxt_clear_int_mode(bp);
11704 rc = bnxt_init_int_mode(bp);
11705 bnxt_ulp_irq_restart(bp, rc);
11706
11707 if (netif_running(bp->dev)) {
11708 if (rc)
11709 dev_close(bp->dev);
11710 else
11711 rc = bnxt_open_nic(bp, true, false);
11712 }
11713
11714 return rc;
11715 }
11716
11717 static int bnxt_init_mac_addr(struct bnxt *bp)
11718 {
11719 int rc = 0;
11720
11721 if (BNXT_PF(bp)) {
11722 memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
11723 } else {
11724 #ifdef CONFIG_BNXT_SRIOV
11725 struct bnxt_vf_info *vf = &bp->vf;
11726 bool strict_approval = true;
11727
11728 if (is_valid_ether_addr(vf->mac_addr)) {
11729 /* overwrite netdev dev_addr with admin VF MAC */
11730 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
11731 /* Older PF driver or firmware may not approve this
11732 * correctly.
11733 */
11734 strict_approval = false;
11735 } else {
11736 eth_hw_addr_random(bp->dev);
11737 }
11738 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
11739 #endif
11740 }
11741 return rc;
11742 }
11743
11744 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
11745 {
11746 struct pci_dev *pdev = bp->pdev;
11747 int pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
11748 u32 dw;
11749
11750 if (!pos) {
11751 netdev_info(bp->dev, "Unable do read adapter's DSN");
11752 return -EOPNOTSUPP;
11753 }
11754
11755 /* DSN (two dw) is at an offset of 4 from the cap pos */
11756 pos += 4;
11757 pci_read_config_dword(pdev, pos, &dw);
11758 put_unaligned_le32(dw, &dsn[0]);
11759 pci_read_config_dword(pdev, pos + 4, &dw);
11760 put_unaligned_le32(dw, &dsn[4]);
11761 bp->flags |= BNXT_FLAG_DSN_VALID;
11762 return 0;
11763 }
11764
11765 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
11766 {
11767 static int version_printed;
11768 struct net_device *dev;
11769 struct bnxt *bp;
11770 int rc, max_irqs;
11771
11772 if (pci_is_bridge(pdev))
11773 return -ENODEV;
11774
11775 if (version_printed++ == 0)
11776 pr_info("%s", version);
11777
11778 max_irqs = bnxt_get_max_irq(pdev);
11779 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
11780 if (!dev)
11781 return -ENOMEM;
11782
11783 bp = netdev_priv(dev);
11784 bnxt_set_max_func_irqs(bp, max_irqs);
11785
11786 if (bnxt_vf_pciid(ent->driver_data))
11787 bp->flags |= BNXT_FLAG_VF;
11788
11789 if (pdev->msix_cap)
11790 bp->flags |= BNXT_FLAG_MSIX_CAP;
11791
11792 rc = bnxt_init_board(pdev, dev);
11793 if (rc < 0)
11794 goto init_err_free;
11795
11796 dev->netdev_ops = &bnxt_netdev_ops;
11797 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
11798 dev->ethtool_ops = &bnxt_ethtool_ops;
11799 pci_set_drvdata(pdev, dev);
11800
11801 rc = bnxt_alloc_hwrm_resources(bp);
11802 if (rc)
11803 goto init_err_pci_clean;
11804
11805 mutex_init(&bp->hwrm_cmd_lock);
11806 mutex_init(&bp->link_lock);
11807
11808 rc = bnxt_fw_init_one_p1(bp);
11809 if (rc)
11810 goto init_err_pci_clean;
11811
11812 if (BNXT_CHIP_P5(bp))
11813 bp->flags |= BNXT_FLAG_CHIP_P5;
11814
11815 rc = bnxt_fw_init_one_p2(bp);
11816 if (rc)
11817 goto init_err_pci_clean;
11818
11819 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
11820 NETIF_F_TSO | NETIF_F_TSO6 |
11821 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
11822 NETIF_F_GSO_IPXIP4 |
11823 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
11824 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
11825 NETIF_F_RXCSUM | NETIF_F_GRO;
11826
11827 if (BNXT_SUPPORTS_TPA(bp))
11828 dev->hw_features |= NETIF_F_LRO;
11829
11830 dev->hw_enc_features =
11831 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
11832 NETIF_F_TSO | NETIF_F_TSO6 |
11833 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
11834 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
11835 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
11836 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
11837 NETIF_F_GSO_GRE_CSUM;
11838 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
11839 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
11840 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
11841 if (BNXT_SUPPORTS_TPA(bp))
11842 dev->hw_features |= NETIF_F_GRO_HW;
11843 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
11844 if (dev->features & NETIF_F_GRO_HW)
11845 dev->features &= ~NETIF_F_LRO;
11846 dev->priv_flags |= IFF_UNICAST_FLT;
11847
11848 #ifdef CONFIG_BNXT_SRIOV
11849 init_waitqueue_head(&bp->sriov_cfg_wait);
11850 mutex_init(&bp->sriov_lock);
11851 #endif
11852 if (BNXT_SUPPORTS_TPA(bp)) {
11853 bp->gro_func = bnxt_gro_func_5730x;
11854 if (BNXT_CHIP_P4(bp))
11855 bp->gro_func = bnxt_gro_func_5731x;
11856 else if (BNXT_CHIP_P5(bp))
11857 bp->gro_func = bnxt_gro_func_5750x;
11858 }
11859 if (!BNXT_CHIP_P4_PLUS(bp))
11860 bp->flags |= BNXT_FLAG_DOUBLE_DB;
11861
11862 bp->ulp_probe = bnxt_ulp_probe;
11863
11864 rc = bnxt_init_mac_addr(bp);
11865 if (rc) {
11866 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
11867 rc = -EADDRNOTAVAIL;
11868 goto init_err_pci_clean;
11869 }
11870
11871 if (BNXT_PF(bp)) {
11872 /* Read the adapter's DSN to use as the eswitch switch_id */
11873 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
11874 }
11875
11876 /* MTU range: 60 - FW defined max */
11877 dev->min_mtu = ETH_ZLEN;
11878 dev->max_mtu = bp->max_mtu;
11879
11880 rc = bnxt_probe_phy(bp, true);
11881 if (rc)
11882 goto init_err_pci_clean;
11883
11884 bnxt_set_rx_skb_mode(bp, false);
11885 bnxt_set_tpa_flags(bp);
11886 bnxt_set_ring_params(bp);
11887 rc = bnxt_set_dflt_rings(bp, true);
11888 if (rc) {
11889 netdev_err(bp->dev, "Not enough rings available.\n");
11890 rc = -ENOMEM;
11891 goto init_err_pci_clean;
11892 }
11893
11894 bnxt_fw_init_one_p3(bp);
11895
11896 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
11897 bp->flags |= BNXT_FLAG_STRIP_VLAN;
11898
11899 rc = bnxt_init_int_mode(bp);
11900 if (rc)
11901 goto init_err_pci_clean;
11902
11903 /* No TC has been set yet and rings may have been trimmed due to
11904 * limited MSIX, so we re-initialize the TX rings per TC.
11905 */
11906 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11907
11908 if (BNXT_PF(bp)) {
11909 if (!bnxt_pf_wq) {
11910 bnxt_pf_wq =
11911 create_singlethread_workqueue("bnxt_pf_wq");
11912 if (!bnxt_pf_wq) {
11913 dev_err(&pdev->dev, "Unable to create workqueue.\n");
11914 goto init_err_pci_clean;
11915 }
11916 }
11917 bnxt_init_tc(bp);
11918 }
11919
11920 bnxt_dl_register(bp);
11921
11922 rc = register_netdev(dev);
11923 if (rc)
11924 goto init_err_cleanup;
11925
11926 if (BNXT_PF(bp))
11927 devlink_port_type_eth_set(&bp->dl_port, bp->dev);
11928 bnxt_dl_fw_reporters_create(bp);
11929
11930 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
11931 board_info[ent->driver_data].name,
11932 (long)pci_resource_start(pdev, 0), dev->dev_addr);
11933 pcie_print_link_status(pdev);
11934
11935 return 0;
11936
11937 init_err_cleanup:
11938 bnxt_dl_unregister(bp);
11939 bnxt_shutdown_tc(bp);
11940 bnxt_clear_int_mode(bp);
11941
11942 init_err_pci_clean:
11943 bnxt_hwrm_func_drv_unrgtr(bp);
11944 bnxt_free_hwrm_short_cmd_req(bp);
11945 bnxt_free_hwrm_resources(bp);
11946 bnxt_free_ctx_mem(bp);
11947 kfree(bp->ctx);
11948 bp->ctx = NULL;
11949 kfree(bp->fw_health);
11950 bp->fw_health = NULL;
11951 bnxt_cleanup_pci(bp);
11952
11953 init_err_free:
11954 free_netdev(dev);
11955 return rc;
11956 }
11957
11958 static void bnxt_shutdown(struct pci_dev *pdev)
11959 {
11960 struct net_device *dev = pci_get_drvdata(pdev);
11961 struct bnxt *bp;
11962
11963 if (!dev)
11964 return;
11965
11966 rtnl_lock();
11967 bp = netdev_priv(dev);
11968 if (!bp)
11969 goto shutdown_exit;
11970
11971 if (netif_running(dev))
11972 dev_close(dev);
11973
11974 bnxt_ulp_shutdown(bp);
11975
11976 if (system_state == SYSTEM_POWER_OFF) {
11977 bnxt_clear_int_mode(bp);
11978 pci_disable_device(pdev);
11979 pci_wake_from_d3(pdev, bp->wol);
11980 pci_set_power_state(pdev, PCI_D3hot);
11981 }
11982
11983 shutdown_exit:
11984 rtnl_unlock();
11985 }
11986
11987 #ifdef CONFIG_PM_SLEEP
11988 static int bnxt_suspend(struct device *device)
11989 {
11990 struct net_device *dev = dev_get_drvdata(device);
11991 struct bnxt *bp = netdev_priv(dev);
11992 int rc = 0;
11993
11994 rtnl_lock();
11995 bnxt_ulp_stop(bp);
11996 if (netif_running(dev)) {
11997 netif_device_detach(dev);
11998 rc = bnxt_close(dev);
11999 }
12000 bnxt_hwrm_func_drv_unrgtr(bp);
12001 pci_disable_device(bp->pdev);
12002 bnxt_free_ctx_mem(bp);
12003 kfree(bp->ctx);
12004 bp->ctx = NULL;
12005 rtnl_unlock();
12006 return rc;
12007 }
12008
12009 static int bnxt_resume(struct device *device)
12010 {
12011 struct net_device *dev = dev_get_drvdata(device);
12012 struct bnxt *bp = netdev_priv(dev);
12013 int rc = 0;
12014
12015 rtnl_lock();
12016 rc = pci_enable_device(bp->pdev);
12017 if (rc) {
12018 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12019 rc);
12020 goto resume_exit;
12021 }
12022 pci_set_master(bp->pdev);
12023 if (bnxt_hwrm_ver_get(bp)) {
12024 rc = -ENODEV;
12025 goto resume_exit;
12026 }
12027 rc = bnxt_hwrm_func_reset(bp);
12028 if (rc) {
12029 rc = -EBUSY;
12030 goto resume_exit;
12031 }
12032
12033 if (bnxt_hwrm_queue_qportcfg(bp)) {
12034 rc = -ENODEV;
12035 goto resume_exit;
12036 }
12037
12038 if (bp->hwrm_spec_code >= 0x10803) {
12039 if (bnxt_alloc_ctx_mem(bp)) {
12040 rc = -ENODEV;
12041 goto resume_exit;
12042 }
12043 }
12044 if (BNXT_NEW_RM(bp))
12045 bnxt_hwrm_func_resc_qcaps(bp, false);
12046
12047 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12048 rc = -ENODEV;
12049 goto resume_exit;
12050 }
12051
12052 bnxt_get_wol_settings(bp);
12053 if (netif_running(dev)) {
12054 rc = bnxt_open(dev);
12055 if (!rc)
12056 netif_device_attach(dev);
12057 }
12058
12059 resume_exit:
12060 bnxt_ulp_start(bp, rc);
12061 rtnl_unlock();
12062 return rc;
12063 }
12064
12065 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12066 #define BNXT_PM_OPS (&bnxt_pm_ops)
12067
12068 #else
12069
12070 #define BNXT_PM_OPS NULL
12071
12072 #endif /* CONFIG_PM_SLEEP */
12073
12074 /**
12075 * bnxt_io_error_detected - called when PCI error is detected
12076 * @pdev: Pointer to PCI device
12077 * @state: The current pci connection state
12078 *
12079 * This function is called after a PCI bus error affecting
12080 * this device has been detected.
12081 */
12082 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12083 pci_channel_state_t state)
12084 {
12085 struct net_device *netdev = pci_get_drvdata(pdev);
12086 struct bnxt *bp = netdev_priv(netdev);
12087
12088 netdev_info(netdev, "PCI I/O error detected\n");
12089
12090 rtnl_lock();
12091 netif_device_detach(netdev);
12092
12093 bnxt_ulp_stop(bp);
12094
12095 if (state == pci_channel_io_perm_failure) {
12096 rtnl_unlock();
12097 return PCI_ERS_RESULT_DISCONNECT;
12098 }
12099
12100 if (netif_running(netdev))
12101 bnxt_close(netdev);
12102
12103 pci_disable_device(pdev);
12104 rtnl_unlock();
12105
12106 /* Request a slot slot reset. */
12107 return PCI_ERS_RESULT_NEED_RESET;
12108 }
12109
12110 /**
12111 * bnxt_io_slot_reset - called after the pci bus has been reset.
12112 * @pdev: Pointer to PCI device
12113 *
12114 * Restart the card from scratch, as if from a cold-boot.
12115 * At this point, the card has exprienced a hard reset,
12116 * followed by fixups by BIOS, and has its config space
12117 * set up identically to what it was at cold boot.
12118 */
12119 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12120 {
12121 struct net_device *netdev = pci_get_drvdata(pdev);
12122 struct bnxt *bp = netdev_priv(netdev);
12123 int err = 0;
12124 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12125
12126 netdev_info(bp->dev, "PCI Slot Reset\n");
12127
12128 rtnl_lock();
12129
12130 if (pci_enable_device(pdev)) {
12131 dev_err(&pdev->dev,
12132 "Cannot re-enable PCI device after reset.\n");
12133 } else {
12134 pci_set_master(pdev);
12135
12136 err = bnxt_hwrm_func_reset(bp);
12137 if (!err && netif_running(netdev))
12138 err = bnxt_open(netdev);
12139
12140 if (!err)
12141 result = PCI_ERS_RESULT_RECOVERED;
12142 bnxt_ulp_start(bp, err);
12143 }
12144
12145 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
12146 dev_close(netdev);
12147
12148 rtnl_unlock();
12149
12150 return PCI_ERS_RESULT_RECOVERED;
12151 }
12152
12153 /**
12154 * bnxt_io_resume - called when traffic can start flowing again.
12155 * @pdev: Pointer to PCI device
12156 *
12157 * This callback is called when the error recovery driver tells
12158 * us that its OK to resume normal operation.
12159 */
12160 static void bnxt_io_resume(struct pci_dev *pdev)
12161 {
12162 struct net_device *netdev = pci_get_drvdata(pdev);
12163
12164 rtnl_lock();
12165
12166 netif_device_attach(netdev);
12167
12168 rtnl_unlock();
12169 }
12170
12171 static const struct pci_error_handlers bnxt_err_handler = {
12172 .error_detected = bnxt_io_error_detected,
12173 .slot_reset = bnxt_io_slot_reset,
12174 .resume = bnxt_io_resume
12175 };
12176
12177 static struct pci_driver bnxt_pci_driver = {
12178 .name = DRV_MODULE_NAME,
12179 .id_table = bnxt_pci_tbl,
12180 .probe = bnxt_init_one,
12181 .remove = bnxt_remove_one,
12182 .shutdown = bnxt_shutdown,
12183 .driver.pm = BNXT_PM_OPS,
12184 .err_handler = &bnxt_err_handler,
12185 #if defined(CONFIG_BNXT_SRIOV)
12186 .sriov_configure = bnxt_sriov_configure,
12187 #endif
12188 };
12189
12190 static int __init bnxt_init(void)
12191 {
12192 bnxt_debug_init();
12193 return pci_register_driver(&bnxt_pci_driver);
12194 }
12195
12196 static void __exit bnxt_exit(void)
12197 {
12198 pci_unregister_driver(&bnxt_pci_driver);
12199 if (bnxt_pf_wq)
12200 destroy_workqueue(bnxt_pf_wq);
12201 bnxt_debug_exit();
12202 }
12203
12204 module_init(bnxt_init);
12205 module_exit(bnxt_exit);
Linux with added FPC-III support