?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'net-6.13-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/netdev/net
[drm/drm-misc.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
blob4ec4934a4eddd34c67463ddd06a7bc680adf1230
1 /* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 * Copyright (c) 2016-2019 Broadcom Limited
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 */
10
11 #include <linux/module.h>
12
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/gro.h>
41 #include <net/ip.h>
42 #include <net/tcp.h>
43 #include <net/udp.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46 #include <net/udp_tunnel.h>
47 #include <linux/workqueue.h>
48 #include <linux/prefetch.h>
49 #include <linux/cache.h>
50 #include <linux/log2.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <net/page_pool/helpers.h>
56 #include <linux/align.h>
57 #include <net/netdev_queues.h>
58
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_hwrm.h"
62 #include "bnxt_ulp.h"
63 #include "bnxt_sriov.h"
64 #include "bnxt_ethtool.h"
65 #include "bnxt_dcb.h"
66 #include "bnxt_xdp.h"
67 #include "bnxt_ptp.h"
68 #include "bnxt_vfr.h"
69 #include "bnxt_tc.h"
70 #include "bnxt_devlink.h"
71 #include "bnxt_debugfs.h"
72 #include "bnxt_coredump.h"
73 #include "bnxt_hwmon.h"
74
75 #define BNXT_TX_TIMEOUT (5 * HZ)
76 #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \
77 NETIF_MSG_TX_ERR)
78
79 MODULE_LICENSE("GPL");
80 MODULE_DESCRIPTION("Broadcom NetXtreme network driver");
81
82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
84 #define BNXT_RX_COPY_THRESH 256
85
86 #define BNXT_TX_PUSH_THRESH 164
87
88 /* indexed by enum board_idx */
89 static const struct {
90 char *name;
91 } board_info[] = {
92 [BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
93 [BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
94 [BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
95 [BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
96 [BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
97 [BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
98 [BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
99 [BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
100 [BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
101 [BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
102 [BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
103 [BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
104 [BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
105 [BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
106 [BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
107 [BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
108 [BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
109 [BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
110 [BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
111 [BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
112 [BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
113 [BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
114 [BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
115 [BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
116 [BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
117 [BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
118 [BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
119 [BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
120 [BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
121 [BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
122 [BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
123 [BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
124 [BCM57608] = { "Broadcom BCM57608 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
125 [BCM57604] = { "Broadcom BCM57604 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
126 [BCM57602] = { "Broadcom BCM57602 NetXtreme-E 10Gb/25Gb/50Gb/100Gb Ethernet" },
127 [BCM57601] = { "Broadcom BCM57601 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb/400Gb Ethernet" },
128 [BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
129 [BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
130 [BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
131 [BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
132 [BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
133 [BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
134 [NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
135 [NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
136 [NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
137 [NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
138 [NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
139 [NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
140 [NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
141 [NETXTREME_E_P7_VF] = { "Broadcom BCM5760X Virtual Function" },
142 };
143
144 static const struct pci_device_id bnxt_pci_tbl[] = {
145 { PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
146 { PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
147 { PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
148 { PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
149 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
150 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
151 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
152 { PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
153 { PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
154 { PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
155 { PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
156 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
157 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
158 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
159 { PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
160 { PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
161 { PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
162 { PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
163 { PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
164 { PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
165 { PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
166 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
167 { PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
168 { PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
169 { PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
170 { PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
171 { PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
172 { PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
173 { PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
174 { PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
175 { PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
176 { PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
177 { PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
178 { PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
179 { PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
180 { PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
181 { PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
182 { PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
183 { PCI_VDEVICE(BROADCOM, 0x1760), .driver_data = BCM57608 },
184 { PCI_VDEVICE(BROADCOM, 0x1761), .driver_data = BCM57604 },
185 { PCI_VDEVICE(BROADCOM, 0x1762), .driver_data = BCM57602 },
186 { PCI_VDEVICE(BROADCOM, 0x1763), .driver_data = BCM57601 },
187 { PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
188 { PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
189 { PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
190 { PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
191 { PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
192 { PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
193 { PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
194 { PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
195 #ifdef CONFIG_BNXT_SRIOV
196 { PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
197 { PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
198 { PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
199 { PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
200 { PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
201 { PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
202 { PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
203 { PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
204 { PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
205 { PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
206 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
207 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
208 { PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
209 { PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
210 { PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
211 { PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
212 { PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
213 { PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
214 { PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
215 { PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
216 { PCI_VDEVICE(BROADCOM, 0x1819), .driver_data = NETXTREME_E_P7_VF },
217 { PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
218 #endif
219 { 0 }
220 };
221
222 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
223
224 static const u16 bnxt_vf_req_snif[] = {
225 HWRM_FUNC_CFG,
226 HWRM_FUNC_VF_CFG,
227 HWRM_PORT_PHY_QCFG,
228 HWRM_CFA_L2_FILTER_ALLOC,
229 };
230
231 static const u16 bnxt_async_events_arr[] = {
232 ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
233 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
234 ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
235 ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
236 ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
237 ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
238 ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
239 ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
240 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
241 ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
242 ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
243 ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
244 ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
245 ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
246 ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
247 ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
248 ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER,
249 };
250
251 const u16 bnxt_bstore_to_trace[] = {
252 [BNXT_CTX_SRT] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT_TRACE,
253 [BNXT_CTX_SRT2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_SRT2_TRACE,
254 [BNXT_CTX_CRT] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT_TRACE,
255 [BNXT_CTX_CRT2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CRT2_TRACE,
256 [BNXT_CTX_RIGP0] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP0_TRACE,
257 [BNXT_CTX_L2HWRM] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_L2_HWRM_TRACE,
258 [BNXT_CTX_REHWRM] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_ROCE_HWRM_TRACE,
259 [BNXT_CTX_CA0] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA0_TRACE,
260 [BNXT_CTX_CA1] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA1_TRACE,
261 [BNXT_CTX_CA2] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_CA2_TRACE,
262 [BNXT_CTX_RIGP1] = DBG_LOG_BUFFER_FLUSH_REQ_TYPE_RIGP1_TRACE,
263 };
264
265 static struct workqueue_struct *bnxt_pf_wq;
266
267 #define BNXT_IPV6_MASK_ALL {{{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, \
268 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }}}
269 #define BNXT_IPV6_MASK_NONE {{{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}}
270
271 const struct bnxt_flow_masks BNXT_FLOW_MASK_NONE = {
272 .ports = {
273 .src = 0,
274 .dst = 0,
275 },
276 .addrs = {
277 .v6addrs = {
278 .src = BNXT_IPV6_MASK_NONE,
279 .dst = BNXT_IPV6_MASK_NONE,
280 },
281 },
282 };
283
284 const struct bnxt_flow_masks BNXT_FLOW_IPV6_MASK_ALL = {
285 .ports = {
286 .src = cpu_to_be16(0xffff),
287 .dst = cpu_to_be16(0xffff),
288 },
289 .addrs = {
290 .v6addrs = {
291 .src = BNXT_IPV6_MASK_ALL,
292 .dst = BNXT_IPV6_MASK_ALL,
293 },
294 },
295 };
296
297 const struct bnxt_flow_masks BNXT_FLOW_IPV4_MASK_ALL = {
298 .ports = {
299 .src = cpu_to_be16(0xffff),
300 .dst = cpu_to_be16(0xffff),
301 },
302 .addrs = {
303 .v4addrs = {
304 .src = cpu_to_be32(0xffffffff),
305 .dst = cpu_to_be32(0xffffffff),
306 },
307 },
308 };
309
310 static bool bnxt_vf_pciid(enum board_idx idx)
311 {
312 return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
313 idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
314 idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
315 idx == NETXTREME_E_P5_VF_HV || idx == NETXTREME_E_P7_VF);
316 }
317
318 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
319 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
320
321 #define BNXT_DB_CQ(db, idx) \
322 writel(DB_CP_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
323
324 #define BNXT_DB_NQ_P5(db, idx) \
325 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\
326 (db)->doorbell)
327
328 #define BNXT_DB_NQ_P7(db, idx) \
329 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_MASK | \
330 DB_RING_IDX(db, idx), (db)->doorbell)
331
332 #define BNXT_DB_CQ_ARM(db, idx) \
333 writel(DB_CP_REARM_FLAGS | DB_RING_IDX(db, idx), (db)->doorbell)
334
335 #define BNXT_DB_NQ_ARM_P5(db, idx) \
336 bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | \
337 DB_RING_IDX(db, idx), (db)->doorbell)
338
339 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
340 {
341 if (bp->flags & BNXT_FLAG_CHIP_P7)
342 BNXT_DB_NQ_P7(db, idx);
343 else if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
344 BNXT_DB_NQ_P5(db, idx);
345 else
346 BNXT_DB_CQ(db, idx);
347 }
348
349 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
350 {
351 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
352 BNXT_DB_NQ_ARM_P5(db, idx);
353 else
354 BNXT_DB_CQ_ARM(db, idx);
355 }
356
357 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
358 {
359 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
360 bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
361 DB_RING_IDX(db, idx), db->doorbell);
362 else
363 BNXT_DB_CQ(db, idx);
364 }
365
366 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
367 {
368 if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
369 return;
370
371 if (BNXT_PF(bp))
372 queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
373 else
374 schedule_delayed_work(&bp->fw_reset_task, delay);
375 }
376
377 static void __bnxt_queue_sp_work(struct bnxt *bp)
378 {
379 if (BNXT_PF(bp))
380 queue_work(bnxt_pf_wq, &bp->sp_task);
381 else
382 schedule_work(&bp->sp_task);
383 }
384
385 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
386 {
387 set_bit(event, &bp->sp_event);
388 __bnxt_queue_sp_work(bp);
389 }
390
391 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
392 {
393 if (!rxr->bnapi->in_reset) {
394 rxr->bnapi->in_reset = true;
395 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
396 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
397 else
398 set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
399 __bnxt_queue_sp_work(bp);
400 }
401 rxr->rx_next_cons = 0xffff;
402 }
403
404 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
405 u16 curr)
406 {
407 struct bnxt_napi *bnapi = txr->bnapi;
408
409 if (bnapi->tx_fault)
410 return;
411
412 netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
413 txr->txq_index, txr->tx_hw_cons,
414 txr->tx_cons, txr->tx_prod, curr);
415 WARN_ON_ONCE(1);
416 bnapi->tx_fault = 1;
417 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
418 }
419
420 const u16 bnxt_lhint_arr[] = {
421 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
422 TX_BD_FLAGS_LHINT_512_TO_1023,
423 TX_BD_FLAGS_LHINT_1024_TO_2047,
424 TX_BD_FLAGS_LHINT_1024_TO_2047,
425 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
426 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
427 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
428 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
429 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
430 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
431 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
432 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
433 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
434 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
435 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
436 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
437 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
438 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
439 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
440 };
441
442 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
443 {
444 struct metadata_dst *md_dst = skb_metadata_dst(skb);
445
446 if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
447 return 0;
448
449 return md_dst->u.port_info.port_id;
450 }
451
452 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
453 u16 prod)
454 {
455 /* Sync BD data before updating doorbell */
456 wmb();
457 bnxt_db_write(bp, &txr->tx_db, prod);
458 txr->kick_pending = 0;
459 }
460
461 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
462 {
463 struct bnxt *bp = netdev_priv(dev);
464 struct tx_bd *txbd, *txbd0;
465 struct tx_bd_ext *txbd1;
466 struct netdev_queue *txq;
467 int i;
468 dma_addr_t mapping;
469 unsigned int length, pad = 0;
470 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
471 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
472 struct pci_dev *pdev = bp->pdev;
473 u16 prod, last_frag, txts_prod;
474 struct bnxt_tx_ring_info *txr;
475 struct bnxt_sw_tx_bd *tx_buf;
476 __le32 lflags = 0;
477
478 i = skb_get_queue_mapping(skb);
479 if (unlikely(i >= bp->tx_nr_rings)) {
480 dev_kfree_skb_any(skb);
481 dev_core_stats_tx_dropped_inc(dev);
482 return NETDEV_TX_OK;
483 }
484
485 txq = netdev_get_tx_queue(dev, i);
486 txr = &bp->tx_ring[bp->tx_ring_map[i]];
487 prod = txr->tx_prod;
488
489 free_size = bnxt_tx_avail(bp, txr);
490 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
491 /* We must have raced with NAPI cleanup */
492 if (net_ratelimit() && txr->kick_pending)
493 netif_warn(bp, tx_err, dev,
494 "bnxt: ring busy w/ flush pending!\n");
495 if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
496 bp->tx_wake_thresh))
497 return NETDEV_TX_BUSY;
498 }
499
500 if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
501 goto tx_free;
502
503 length = skb->len;
504 len = skb_headlen(skb);
505 last_frag = skb_shinfo(skb)->nr_frags;
506
507 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
508
509 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
510 tx_buf->skb = skb;
511 tx_buf->nr_frags = last_frag;
512
513 vlan_tag_flags = 0;
514 cfa_action = bnxt_xmit_get_cfa_action(skb);
515 if (skb_vlan_tag_present(skb)) {
516 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
517 skb_vlan_tag_get(skb);
518 /* Currently supports 8021Q, 8021AD vlan offloads
519 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
520 */
521 if (skb->vlan_proto == htons(ETH_P_8021Q))
522 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
523 }
524
525 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && ptp &&
526 ptp->tx_tstamp_en) {
527 if (bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP) {
528 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
529 tx_buf->is_ts_pkt = 1;
530 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
531 } else if (!skb_is_gso(skb)) {
532 u16 seq_id, hdr_off;
533
534 if (!bnxt_ptp_parse(skb, &seq_id, &hdr_off) &&
535 !bnxt_ptp_get_txts_prod(ptp, &txts_prod)) {
536 if (vlan_tag_flags)
537 hdr_off += VLAN_HLEN;
538 lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
539 tx_buf->is_ts_pkt = 1;
540 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
541
542 ptp->txts_req[txts_prod].tx_seqid = seq_id;
543 ptp->txts_req[txts_prod].tx_hdr_off = hdr_off;
544 tx_buf->txts_prod = txts_prod;
545 }
546 }
547 }
548 if (unlikely(skb->no_fcs))
549 lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
550
551 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
552 !lflags) {
553 struct tx_push_buffer *tx_push_buf = txr->tx_push;
554 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
555 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
556 void __iomem *db = txr->tx_db.doorbell;
557 void *pdata = tx_push_buf->data;
558 u64 *end;
559 int j, push_len;
560
561 /* Set COAL_NOW to be ready quickly for the next push */
562 tx_push->tx_bd_len_flags_type =
563 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
564 TX_BD_TYPE_LONG_TX_BD |
565 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
566 TX_BD_FLAGS_COAL_NOW |
567 TX_BD_FLAGS_PACKET_END |
568 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
569
570 if (skb->ip_summed == CHECKSUM_PARTIAL)
571 tx_push1->tx_bd_hsize_lflags =
572 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
573 else
574 tx_push1->tx_bd_hsize_lflags = 0;
575
576 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
577 tx_push1->tx_bd_cfa_action =
578 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
579
580 end = pdata + length;
581 end = PTR_ALIGN(end, 8) - 1;
582 *end = 0;
583
584 skb_copy_from_linear_data(skb, pdata, len);
585 pdata += len;
586 for (j = 0; j < last_frag; j++) {
587 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
588 void *fptr;
589
590 fptr = skb_frag_address_safe(frag);
591 if (!fptr)
592 goto normal_tx;
593
594 memcpy(pdata, fptr, skb_frag_size(frag));
595 pdata += skb_frag_size(frag);
596 }
597
598 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
599 txbd->tx_bd_haddr = txr->data_mapping;
600 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2);
601 prod = NEXT_TX(prod);
602 tx_push->tx_bd_opaque = txbd->tx_bd_opaque;
603 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
604 memcpy(txbd, tx_push1, sizeof(*txbd));
605 prod = NEXT_TX(prod);
606 tx_push->doorbell =
607 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH |
608 DB_RING_IDX(&txr->tx_db, prod));
609 WRITE_ONCE(txr->tx_prod, prod);
610
611 tx_buf->is_push = 1;
612 netdev_tx_sent_queue(txq, skb->len);
613 wmb(); /* Sync is_push and byte queue before pushing data */
614
615 push_len = (length + sizeof(*tx_push) + 7) / 8;
616 if (push_len > 16) {
617 __iowrite64_copy(db, tx_push_buf, 16);
618 __iowrite32_copy(db + 4, tx_push_buf + 1,
619 (push_len - 16) << 1);
620 } else {
621 __iowrite64_copy(db, tx_push_buf, push_len);
622 }
623
624 goto tx_done;
625 }
626
627 normal_tx:
628 if (length < BNXT_MIN_PKT_SIZE) {
629 pad = BNXT_MIN_PKT_SIZE - length;
630 if (skb_pad(skb, pad))
631 /* SKB already freed. */
632 goto tx_kick_pending;
633 length = BNXT_MIN_PKT_SIZE;
634 }
635
636 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
637
638 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
639 goto tx_free;
640
641 dma_unmap_addr_set(tx_buf, mapping, mapping);
642 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
643 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
644
645 txbd->tx_bd_haddr = cpu_to_le64(mapping);
646 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag);
647
648 prod = NEXT_TX(prod);
649 txbd1 = (struct tx_bd_ext *)
650 &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
651
652 txbd1->tx_bd_hsize_lflags = lflags;
653 if (skb_is_gso(skb)) {
654 bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4);
655 u32 hdr_len;
656
657 if (skb->encapsulation) {
658 if (udp_gso)
659 hdr_len = skb_inner_transport_offset(skb) +
660 sizeof(struct udphdr);
661 else
662 hdr_len = skb_inner_tcp_all_headers(skb);
663 } else if (udp_gso) {
664 hdr_len = skb_transport_offset(skb) +
665 sizeof(struct udphdr);
666 } else {
667 hdr_len = skb_tcp_all_headers(skb);
668 }
669
670 txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
671 TX_BD_FLAGS_T_IPID |
672 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
673 length = skb_shinfo(skb)->gso_size;
674 txbd1->tx_bd_mss = cpu_to_le32(length);
675 length += hdr_len;
676 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
677 txbd1->tx_bd_hsize_lflags |=
678 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
679 txbd1->tx_bd_mss = 0;
680 }
681
682 length >>= 9;
683 if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
684 dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
685 skb->len);
686 i = 0;
687 goto tx_dma_error;
688 }
689 flags |= bnxt_lhint_arr[length];
690 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
691
692 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
693 txbd1->tx_bd_cfa_action =
694 cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
695 txbd0 = txbd;
696 for (i = 0; i < last_frag; i++) {
697 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
698
699 prod = NEXT_TX(prod);
700 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];
701
702 len = skb_frag_size(frag);
703 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
704 DMA_TO_DEVICE);
705
706 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
707 goto tx_dma_error;
708
709 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
710 dma_unmap_addr_set(tx_buf, mapping, mapping);
711
712 txbd->tx_bd_haddr = cpu_to_le64(mapping);
713
714 flags = len << TX_BD_LEN_SHIFT;
715 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
716 }
717
718 flags &= ~TX_BD_LEN;
719 txbd->tx_bd_len_flags_type =
720 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
721 TX_BD_FLAGS_PACKET_END);
722
723 netdev_tx_sent_queue(txq, skb->len);
724
725 skb_tx_timestamp(skb);
726
727 prod = NEXT_TX(prod);
728 WRITE_ONCE(txr->tx_prod, prod);
729
730 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
731 bnxt_txr_db_kick(bp, txr, prod);
732 } else {
733 if (free_size >= bp->tx_wake_thresh)
734 txbd0->tx_bd_len_flags_type |=
735 cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
736 txr->kick_pending = 1;
737 }
738
739 tx_done:
740
741 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
742 if (netdev_xmit_more() && !tx_buf->is_push) {
743 txbd0->tx_bd_len_flags_type &=
744 cpu_to_le32(~TX_BD_FLAGS_NO_CMPL);
745 bnxt_txr_db_kick(bp, txr, prod);
746 }
747
748 netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
749 bp->tx_wake_thresh);
750 }
751 return NETDEV_TX_OK;
752
753 tx_dma_error:
754 last_frag = i;
755
756 /* start back at beginning and unmap skb */
757 prod = txr->tx_prod;
758 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
759 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
760 skb_headlen(skb), DMA_TO_DEVICE);
761 prod = NEXT_TX(prod);
762
763 /* unmap remaining mapped pages */
764 for (i = 0; i < last_frag; i++) {
765 prod = NEXT_TX(prod);
766 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];
767 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
768 skb_frag_size(&skb_shinfo(skb)->frags[i]),
769 DMA_TO_DEVICE);
770 }
771
772 tx_free:
773 dev_kfree_skb_any(skb);
774 tx_kick_pending:
775 if (BNXT_TX_PTP_IS_SET(lflags)) {
776 txr->tx_buf_ring[txr->tx_prod].is_ts_pkt = 0;
777 atomic64_inc(&bp->ptp_cfg->stats.ts_err);
778 if (!(bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP))
779 /* set SKB to err so PTP worker will clean up */
780 ptp->txts_req[txts_prod].tx_skb = ERR_PTR(-EIO);
781 }
782 if (txr->kick_pending)
783 bnxt_txr_db_kick(bp, txr, txr->tx_prod);
784 txr->tx_buf_ring[txr->tx_prod].skb = NULL;
785 dev_core_stats_tx_dropped_inc(dev);
786 return NETDEV_TX_OK;
787 }
788
789 /* Returns true if some remaining TX packets not processed. */
790 static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
791 int budget)
792 {
793 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
794 struct pci_dev *pdev = bp->pdev;
795 u16 hw_cons = txr->tx_hw_cons;
796 unsigned int tx_bytes = 0;
797 u16 cons = txr->tx_cons;
798 int tx_pkts = 0;
799 bool rc = false;
800
801 while (RING_TX(bp, cons) != hw_cons) {
802 struct bnxt_sw_tx_bd *tx_buf;
803 struct sk_buff *skb;
804 bool is_ts_pkt;
805 int j, last;
806
807 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
808 skb = tx_buf->skb;
809
810 if (unlikely(!skb)) {
811 bnxt_sched_reset_txr(bp, txr, cons);
812 return rc;
813 }
814
815 is_ts_pkt = tx_buf->is_ts_pkt;
816 if (is_ts_pkt && (bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP)) {
817 rc = true;
818 break;
819 }
820
821 cons = NEXT_TX(cons);
822 tx_pkts++;
823 tx_bytes += skb->len;
824 tx_buf->skb = NULL;
825 tx_buf->is_ts_pkt = 0;
826
827 if (tx_buf->is_push) {
828 tx_buf->is_push = 0;
829 goto next_tx_int;
830 }
831
832 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
833 skb_headlen(skb), DMA_TO_DEVICE);
834 last = tx_buf->nr_frags;
835
836 for (j = 0; j < last; j++) {
837 cons = NEXT_TX(cons);
838 tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)];
839 dma_unmap_page(
840 &pdev->dev,
841 dma_unmap_addr(tx_buf, mapping),
842 skb_frag_size(&skb_shinfo(skb)->frags[j]),
843 DMA_TO_DEVICE);
844 }
845 if (unlikely(is_ts_pkt)) {
846 if (BNXT_CHIP_P5(bp)) {
847 /* PTP worker takes ownership of the skb */
848 bnxt_get_tx_ts_p5(bp, skb, tx_buf->txts_prod);
849 skb = NULL;
850 }
851 }
852
853 next_tx_int:
854 cons = NEXT_TX(cons);
855
856 dev_consume_skb_any(skb);
857 }
858
859 WRITE_ONCE(txr->tx_cons, cons);
860
861 __netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
862 bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
863 READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
864
865 return rc;
866 }
867
868 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
869 {
870 struct bnxt_tx_ring_info *txr;
871 bool more = false;
872 int i;
873
874 bnxt_for_each_napi_tx(i, bnapi, txr) {
875 if (txr->tx_hw_cons != RING_TX(bp, txr->tx_cons))
876 more |= __bnxt_tx_int(bp, txr, budget);
877 }
878 if (!more)
879 bnapi->events &= ~BNXT_TX_CMP_EVENT;
880 }
881
882 static bool bnxt_separate_head_pool(void)
883 {
884 return PAGE_SIZE > BNXT_RX_PAGE_SIZE;
885 }
886
887 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
888 struct bnxt_rx_ring_info *rxr,
889 unsigned int *offset,
890 gfp_t gfp)
891 {
892 struct page *page;
893
894 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
895 page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
896 BNXT_RX_PAGE_SIZE);
897 } else {
898 page = page_pool_dev_alloc_pages(rxr->page_pool);
899 *offset = 0;
900 }
901 if (!page)
902 return NULL;
903
904 *mapping = page_pool_get_dma_addr(page) + *offset;
905 return page;
906 }
907
908 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
909 struct bnxt_rx_ring_info *rxr,
910 gfp_t gfp)
911 {
912 unsigned int offset;
913 struct page *page;
914
915 page = page_pool_alloc_frag(rxr->head_pool, &offset,
916 bp->rx_buf_size, gfp);
917 if (!page)
918 return NULL;
919
920 *mapping = page_pool_get_dma_addr(page) + bp->rx_dma_offset + offset;
921 return page_address(page) + offset;
922 }
923
924 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
925 u16 prod, gfp_t gfp)
926 {
927 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
928 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
929 dma_addr_t mapping;
930
931 if (BNXT_RX_PAGE_MODE(bp)) {
932 unsigned int offset;
933 struct page *page =
934 __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
935
936 if (!page)
937 return -ENOMEM;
938
939 mapping += bp->rx_dma_offset;
940 rx_buf->data = page;
941 rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
942 } else {
943 u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp);
944
945 if (!data)
946 return -ENOMEM;
947
948 rx_buf->data = data;
949 rx_buf->data_ptr = data + bp->rx_offset;
950 }
951 rx_buf->mapping = mapping;
952
953 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
954 return 0;
955 }
956
957 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
958 {
959 u16 prod = rxr->rx_prod;
960 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
961 struct bnxt *bp = rxr->bnapi->bp;
962 struct rx_bd *cons_bd, *prod_bd;
963
964 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
965 cons_rx_buf = &rxr->rx_buf_ring[cons];
966
967 prod_rx_buf->data = data;
968 prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
969
970 prod_rx_buf->mapping = cons_rx_buf->mapping;
971
972 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
973 cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)];
974
975 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
976 }
977
978 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
979 {
980 u16 next, max = rxr->rx_agg_bmap_size;
981
982 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
983 if (next >= max)
984 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
985 return next;
986 }
987
988 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
989 struct bnxt_rx_ring_info *rxr,
990 u16 prod, gfp_t gfp)
991 {
992 struct rx_bd *rxbd =
993 &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
994 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
995 struct page *page;
996 dma_addr_t mapping;
997 u16 sw_prod = rxr->rx_sw_agg_prod;
998 unsigned int offset = 0;
999
1000 page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
1001
1002 if (!page)
1003 return -ENOMEM;
1004
1005 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
1006 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
1007
1008 __set_bit(sw_prod, rxr->rx_agg_bmap);
1009 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
1010 rxr->rx_sw_agg_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
1011
1012 rx_agg_buf->page = page;
1013 rx_agg_buf->offset = offset;
1014 rx_agg_buf->mapping = mapping;
1015 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
1016 rxbd->rx_bd_opaque = sw_prod;
1017 return 0;
1018 }
1019
1020 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
1021 struct bnxt_cp_ring_info *cpr,
1022 u16 cp_cons, u16 curr)
1023 {
1024 struct rx_agg_cmp *agg;
1025
1026 cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
1027 agg = (struct rx_agg_cmp *)
1028 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1029 return agg;
1030 }
1031
1032 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
1033 struct bnxt_rx_ring_info *rxr,
1034 u16 agg_id, u16 curr)
1035 {
1036 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
1037
1038 return &tpa_info->agg_arr[curr];
1039 }
1040
1041 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
1042 u16 start, u32 agg_bufs, bool tpa)
1043 {
1044 struct bnxt_napi *bnapi = cpr->bnapi;
1045 struct bnxt *bp = bnapi->bp;
1046 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1047 u16 prod = rxr->rx_agg_prod;
1048 u16 sw_prod = rxr->rx_sw_agg_prod;
1049 bool p5_tpa = false;
1050 u32 i;
1051
1052 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1053 p5_tpa = true;
1054
1055 for (i = 0; i < agg_bufs; i++) {
1056 u16 cons;
1057 struct rx_agg_cmp *agg;
1058 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
1059 struct rx_bd *prod_bd;
1060 struct page *page;
1061
1062 if (p5_tpa)
1063 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
1064 else
1065 agg = bnxt_get_agg(bp, cpr, idx, start + i);
1066 cons = agg->rx_agg_cmp_opaque;
1067 __clear_bit(cons, rxr->rx_agg_bmap);
1068
1069 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
1070 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
1071
1072 __set_bit(sw_prod, rxr->rx_agg_bmap);
1073 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
1074 cons_rx_buf = &rxr->rx_agg_ring[cons];
1075
1076 /* It is possible for sw_prod to be equal to cons, so
1077 * set cons_rx_buf->page to NULL first.
1078 */
1079 page = cons_rx_buf->page;
1080 cons_rx_buf->page = NULL;
1081 prod_rx_buf->page = page;
1082 prod_rx_buf->offset = cons_rx_buf->offset;
1083
1084 prod_rx_buf->mapping = cons_rx_buf->mapping;
1085
1086 prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bp, prod)][RX_IDX(prod)];
1087
1088 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
1089 prod_bd->rx_bd_opaque = sw_prod;
1090
1091 prod = NEXT_RX_AGG(prod);
1092 sw_prod = RING_RX_AGG(bp, NEXT_RX_AGG(sw_prod));
1093 }
1094 rxr->rx_agg_prod = prod;
1095 rxr->rx_sw_agg_prod = sw_prod;
1096 }
1097
1098 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
1099 struct bnxt_rx_ring_info *rxr,
1100 u16 cons, void *data, u8 *data_ptr,
1101 dma_addr_t dma_addr,
1102 unsigned int offset_and_len)
1103 {
1104 unsigned int len = offset_and_len & 0xffff;
1105 struct page *page = data;
1106 u16 prod = rxr->rx_prod;
1107 struct sk_buff *skb;
1108 int err;
1109
1110 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1111 if (unlikely(err)) {
1112 bnxt_reuse_rx_data(rxr, cons, data);
1113 return NULL;
1114 }
1115 dma_addr -= bp->rx_dma_offset;
1116 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1117 bp->rx_dir);
1118 skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
1119 if (!skb) {
1120 page_pool_recycle_direct(rxr->page_pool, page);
1121 return NULL;
1122 }
1123 skb_mark_for_recycle(skb);
1124 skb_reserve(skb, bp->rx_offset);
1125 __skb_put(skb, len);
1126
1127 return skb;
1128 }
1129
1130 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1131 struct bnxt_rx_ring_info *rxr,
1132 u16 cons, void *data, u8 *data_ptr,
1133 dma_addr_t dma_addr,
1134 unsigned int offset_and_len)
1135 {
1136 unsigned int payload = offset_and_len >> 16;
1137 unsigned int len = offset_and_len & 0xffff;
1138 skb_frag_t *frag;
1139 struct page *page = data;
1140 u16 prod = rxr->rx_prod;
1141 struct sk_buff *skb;
1142 int off, err;
1143
1144 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1145 if (unlikely(err)) {
1146 bnxt_reuse_rx_data(rxr, cons, data);
1147 return NULL;
1148 }
1149 dma_addr -= bp->rx_dma_offset;
1150 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1151 bp->rx_dir);
1152
1153 if (unlikely(!payload))
1154 payload = eth_get_headlen(bp->dev, data_ptr, len);
1155
1156 skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1157 if (!skb) {
1158 page_pool_recycle_direct(rxr->page_pool, page);
1159 return NULL;
1160 }
1161
1162 skb_mark_for_recycle(skb);
1163 off = (void *)data_ptr - page_address(page);
1164 skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1165 memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1166 payload + NET_IP_ALIGN);
1167
1168 frag = &skb_shinfo(skb)->frags[0];
1169 skb_frag_size_sub(frag, payload);
1170 skb_frag_off_add(frag, payload);
1171 skb->data_len -= payload;
1172 skb->tail += payload;
1173
1174 return skb;
1175 }
1176
1177 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1178 struct bnxt_rx_ring_info *rxr, u16 cons,
1179 void *data, u8 *data_ptr,
1180 dma_addr_t dma_addr,
1181 unsigned int offset_and_len)
1182 {
1183 u16 prod = rxr->rx_prod;
1184 struct sk_buff *skb;
1185 int err;
1186
1187 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1188 if (unlikely(err)) {
1189 bnxt_reuse_rx_data(rxr, cons, data);
1190 return NULL;
1191 }
1192
1193 skb = napi_build_skb(data, bp->rx_buf_size);
1194 dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1195 bp->rx_dir);
1196 if (!skb) {
1197 page_pool_free_va(rxr->head_pool, data, true);
1198 return NULL;
1199 }
1200
1201 skb_mark_for_recycle(skb);
1202 skb_reserve(skb, bp->rx_offset);
1203 skb_put(skb, offset_and_len & 0xffff);
1204 return skb;
1205 }
1206
1207 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1208 struct bnxt_cp_ring_info *cpr,
1209 struct skb_shared_info *shinfo,
1210 u16 idx, u32 agg_bufs, bool tpa,
1211 struct xdp_buff *xdp)
1212 {
1213 struct bnxt_napi *bnapi = cpr->bnapi;
1214 struct pci_dev *pdev = bp->pdev;
1215 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1216 u16 prod = rxr->rx_agg_prod;
1217 u32 i, total_frag_len = 0;
1218 bool p5_tpa = false;
1219
1220 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && tpa)
1221 p5_tpa = true;
1222
1223 for (i = 0; i < agg_bufs; i++) {
1224 skb_frag_t *frag = &shinfo->frags[i];
1225 u16 cons, frag_len;
1226 struct rx_agg_cmp *agg;
1227 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1228 struct page *page;
1229 dma_addr_t mapping;
1230
1231 if (p5_tpa)
1232 agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1233 else
1234 agg = bnxt_get_agg(bp, cpr, idx, i);
1235 cons = agg->rx_agg_cmp_opaque;
1236 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1237 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1238
1239 cons_rx_buf = &rxr->rx_agg_ring[cons];
1240 skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1241 cons_rx_buf->offset, frag_len);
1242 shinfo->nr_frags = i + 1;
1243 __clear_bit(cons, rxr->rx_agg_bmap);
1244
1245 /* It is possible for bnxt_alloc_rx_page() to allocate
1246 * a sw_prod index that equals the cons index, so we
1247 * need to clear the cons entry now.
1248 */
1249 mapping = cons_rx_buf->mapping;
1250 page = cons_rx_buf->page;
1251 cons_rx_buf->page = NULL;
1252
1253 if (xdp && page_is_pfmemalloc(page))
1254 xdp_buff_set_frag_pfmemalloc(xdp);
1255
1256 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1257 --shinfo->nr_frags;
1258 cons_rx_buf->page = page;
1259
1260 /* Update prod since possibly some pages have been
1261 * allocated already.
1262 */
1263 rxr->rx_agg_prod = prod;
1264 bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1265 return 0;
1266 }
1267
1268 dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1269 bp->rx_dir);
1270
1271 total_frag_len += frag_len;
1272 prod = NEXT_RX_AGG(prod);
1273 }
1274 rxr->rx_agg_prod = prod;
1275 return total_frag_len;
1276 }
1277
1278 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1279 struct bnxt_cp_ring_info *cpr,
1280 struct sk_buff *skb, u16 idx,
1281 u32 agg_bufs, bool tpa)
1282 {
1283 struct skb_shared_info *shinfo = skb_shinfo(skb);
1284 u32 total_frag_len = 0;
1285
1286 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1287 agg_bufs, tpa, NULL);
1288 if (!total_frag_len) {
1289 skb_mark_for_recycle(skb);
1290 dev_kfree_skb(skb);
1291 return NULL;
1292 }
1293
1294 skb->data_len += total_frag_len;
1295 skb->len += total_frag_len;
1296 skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1297 return skb;
1298 }
1299
1300 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1301 struct bnxt_cp_ring_info *cpr,
1302 struct xdp_buff *xdp, u16 idx,
1303 u32 agg_bufs, bool tpa)
1304 {
1305 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1306 u32 total_frag_len = 0;
1307
1308 if (!xdp_buff_has_frags(xdp))
1309 shinfo->nr_frags = 0;
1310
1311 total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1312 idx, agg_bufs, tpa, xdp);
1313 if (total_frag_len) {
1314 xdp_buff_set_frags_flag(xdp);
1315 shinfo->nr_frags = agg_bufs;
1316 shinfo->xdp_frags_size = total_frag_len;
1317 }
1318 return total_frag_len;
1319 }
1320
1321 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1322 u8 agg_bufs, u32 *raw_cons)
1323 {
1324 u16 last;
1325 struct rx_agg_cmp *agg;
1326
1327 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1328 last = RING_CMP(*raw_cons);
1329 agg = (struct rx_agg_cmp *)
1330 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1331 return RX_AGG_CMP_VALID(agg, *raw_cons);
1332 }
1333
1334 static struct sk_buff *bnxt_copy_data(struct bnxt_napi *bnapi, u8 *data,
1335 unsigned int len,
1336 dma_addr_t mapping)
1337 {
1338 struct bnxt *bp = bnapi->bp;
1339 struct pci_dev *pdev = bp->pdev;
1340 struct sk_buff *skb;
1341
1342 skb = napi_alloc_skb(&bnapi->napi, len);
1343 if (!skb)
1344 return NULL;
1345
1346 dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1347 bp->rx_dir);
1348
1349 memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1350 len + NET_IP_ALIGN);
1351
1352 dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1353 bp->rx_dir);
1354
1355 skb_put(skb, len);
1356
1357 return skb;
1358 }
1359
1360 static struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1361 unsigned int len,
1362 dma_addr_t mapping)
1363 {
1364 return bnxt_copy_data(bnapi, data, len, mapping);
1365 }
1366
1367 static struct sk_buff *bnxt_copy_xdp(struct bnxt_napi *bnapi,
1368 struct xdp_buff *xdp,
1369 unsigned int len,
1370 dma_addr_t mapping)
1371 {
1372 unsigned int metasize = 0;
1373 u8 *data = xdp->data;
1374 struct sk_buff *skb;
1375
1376 len = xdp->data_end - xdp->data_meta;
1377 metasize = xdp->data - xdp->data_meta;
1378 data = xdp->data_meta;
1379
1380 skb = bnxt_copy_data(bnapi, data, len, mapping);
1381 if (!skb)
1382 return skb;
1383
1384 if (metasize) {
1385 skb_metadata_set(skb, metasize);
1386 __skb_pull(skb, metasize);
1387 }
1388
1389 return skb;
1390 }
1391
1392 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1393 u32 *raw_cons, void *cmp)
1394 {
1395 struct rx_cmp *rxcmp = cmp;
1396 u32 tmp_raw_cons = *raw_cons;
1397 u8 cmp_type, agg_bufs = 0;
1398
1399 cmp_type = RX_CMP_TYPE(rxcmp);
1400
1401 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1402 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1403 RX_CMP_AGG_BUFS) >>
1404 RX_CMP_AGG_BUFS_SHIFT;
1405 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1406 struct rx_tpa_end_cmp *tpa_end = cmp;
1407
1408 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1409 return 0;
1410
1411 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1412 }
1413
1414 if (agg_bufs) {
1415 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1416 return -EBUSY;
1417 }
1418 *raw_cons = tmp_raw_cons;
1419 return 0;
1420 }
1421
1422 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1423 {
1424 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1425 u16 idx = agg_id & MAX_TPA_P5_MASK;
1426
1427 if (test_bit(idx, map->agg_idx_bmap))
1428 idx = find_first_zero_bit(map->agg_idx_bmap,
1429 BNXT_AGG_IDX_BMAP_SIZE);
1430 __set_bit(idx, map->agg_idx_bmap);
1431 map->agg_id_tbl[agg_id] = idx;
1432 return idx;
1433 }
1434
1435 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1436 {
1437 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1438
1439 __clear_bit(idx, map->agg_idx_bmap);
1440 }
1441
1442 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1443 {
1444 struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1445
1446 return map->agg_id_tbl[agg_id];
1447 }
1448
1449 static void bnxt_tpa_metadata(struct bnxt_tpa_info *tpa_info,
1450 struct rx_tpa_start_cmp *tpa_start,
1451 struct rx_tpa_start_cmp_ext *tpa_start1)
1452 {
1453 tpa_info->cfa_code_valid = 1;
1454 tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1455 tpa_info->vlan_valid = 0;
1456 if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
1457 tpa_info->vlan_valid = 1;
1458 tpa_info->metadata =
1459 le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1460 }
1461 }
1462
1463 static void bnxt_tpa_metadata_v2(struct bnxt_tpa_info *tpa_info,
1464 struct rx_tpa_start_cmp *tpa_start,
1465 struct rx_tpa_start_cmp_ext *tpa_start1)
1466 {
1467 tpa_info->vlan_valid = 0;
1468 if (TPA_START_VLAN_VALID(tpa_start)) {
1469 u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
1470 u32 vlan_proto = ETH_P_8021Q;
1471
1472 tpa_info->vlan_valid = 1;
1473 if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
1474 vlan_proto = ETH_P_8021AD;
1475 tpa_info->metadata = vlan_proto << 16 |
1476 TPA_START_METADATA0_TCI(tpa_start1);
1477 }
1478 }
1479
1480 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1481 u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
1482 struct rx_tpa_start_cmp_ext *tpa_start1)
1483 {
1484 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1485 struct bnxt_tpa_info *tpa_info;
1486 u16 cons, prod, agg_id;
1487 struct rx_bd *prod_bd;
1488 dma_addr_t mapping;
1489
1490 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1491 agg_id = TPA_START_AGG_ID_P5(tpa_start);
1492 agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1493 } else {
1494 agg_id = TPA_START_AGG_ID(tpa_start);
1495 }
1496 cons = tpa_start->rx_tpa_start_cmp_opaque;
1497 prod = rxr->rx_prod;
1498 cons_rx_buf = &rxr->rx_buf_ring[cons];
1499 prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)];
1500 tpa_info = &rxr->rx_tpa[agg_id];
1501
1502 if (unlikely(cons != rxr->rx_next_cons ||
1503 TPA_START_ERROR(tpa_start))) {
1504 netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1505 cons, rxr->rx_next_cons,
1506 TPA_START_ERROR_CODE(tpa_start1));
1507 bnxt_sched_reset_rxr(bp, rxr);
1508 return;
1509 }
1510 prod_rx_buf->data = tpa_info->data;
1511 prod_rx_buf->data_ptr = tpa_info->data_ptr;
1512
1513 mapping = tpa_info->mapping;
1514 prod_rx_buf->mapping = mapping;
1515
1516 prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)];
1517
1518 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1519
1520 tpa_info->data = cons_rx_buf->data;
1521 tpa_info->data_ptr = cons_rx_buf->data_ptr;
1522 cons_rx_buf->data = NULL;
1523 tpa_info->mapping = cons_rx_buf->mapping;
1524
1525 tpa_info->len =
1526 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1527 RX_TPA_START_CMP_LEN_SHIFT;
1528 if (likely(TPA_START_HASH_VALID(tpa_start))) {
1529 tpa_info->hash_type = PKT_HASH_TYPE_L4;
1530 tpa_info->gso_type = SKB_GSO_TCPV4;
1531 if (TPA_START_IS_IPV6(tpa_start1))
1532 tpa_info->gso_type = SKB_GSO_TCPV6;
1533 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1534 else if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP &&
1535 TPA_START_HASH_TYPE(tpa_start) == 3)
1536 tpa_info->gso_type = SKB_GSO_TCPV6;
1537 tpa_info->rss_hash =
1538 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1539 } else {
1540 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1541 tpa_info->gso_type = 0;
1542 netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1543 }
1544 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1545 tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1546 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
1547 bnxt_tpa_metadata(tpa_info, tpa_start, tpa_start1);
1548 else
1549 bnxt_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
1550 tpa_info->agg_count = 0;
1551
1552 rxr->rx_prod = NEXT_RX(prod);
1553 cons = RING_RX(bp, NEXT_RX(cons));
1554 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
1555 cons_rx_buf = &rxr->rx_buf_ring[cons];
1556
1557 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1558 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1559 cons_rx_buf->data = NULL;
1560 }
1561
1562 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1563 {
1564 if (agg_bufs)
1565 bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1566 }
1567
1568 #ifdef CONFIG_INET
1569 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1570 {
1571 struct udphdr *uh = NULL;
1572
1573 if (ip_proto == htons(ETH_P_IP)) {
1574 struct iphdr *iph = (struct iphdr *)skb->data;
1575
1576 if (iph->protocol == IPPROTO_UDP)
1577 uh = (struct udphdr *)(iph + 1);
1578 } else {
1579 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1580
1581 if (iph->nexthdr == IPPROTO_UDP)
1582 uh = (struct udphdr *)(iph + 1);
1583 }
1584 if (uh) {
1585 if (uh->check)
1586 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1587 else
1588 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1589 }
1590 }
1591 #endif
1592
1593 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1594 int payload_off, int tcp_ts,
1595 struct sk_buff *skb)
1596 {
1597 #ifdef CONFIG_INET
1598 struct tcphdr *th;
1599 int len, nw_off;
1600 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1601 u32 hdr_info = tpa_info->hdr_info;
1602 bool loopback = false;
1603
1604 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1605 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1606 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1607
1608 /* If the packet is an internal loopback packet, the offsets will
1609 * have an extra 4 bytes.
1610 */
1611 if (inner_mac_off == 4) {
1612 loopback = true;
1613 } else if (inner_mac_off > 4) {
1614 __be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1615 ETH_HLEN - 2));
1616
1617 /* We only support inner iPv4/ipv6. If we don't see the
1618 * correct protocol ID, it must be a loopback packet where
1619 * the offsets are off by 4.
1620 */
1621 if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1622 loopback = true;
1623 }
1624 if (loopback) {
1625 /* internal loopback packet, subtract all offsets by 4 */
1626 inner_ip_off -= 4;
1627 inner_mac_off -= 4;
1628 outer_ip_off -= 4;
1629 }
1630
1631 nw_off = inner_ip_off - ETH_HLEN;
1632 skb_set_network_header(skb, nw_off);
1633 if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1634 struct ipv6hdr *iph = ipv6_hdr(skb);
1635
1636 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1637 len = skb->len - skb_transport_offset(skb);
1638 th = tcp_hdr(skb);
1639 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1640 } else {
1641 struct iphdr *iph = ip_hdr(skb);
1642
1643 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1644 len = skb->len - skb_transport_offset(skb);
1645 th = tcp_hdr(skb);
1646 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1647 }
1648
1649 if (inner_mac_off) { /* tunnel */
1650 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1651 ETH_HLEN - 2));
1652
1653 bnxt_gro_tunnel(skb, proto);
1654 }
1655 #endif
1656 return skb;
1657 }
1658
1659 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1660 int payload_off, int tcp_ts,
1661 struct sk_buff *skb)
1662 {
1663 #ifdef CONFIG_INET
1664 u16 outer_ip_off, inner_ip_off, inner_mac_off;
1665 u32 hdr_info = tpa_info->hdr_info;
1666 int iphdr_len, nw_off;
1667
1668 inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1669 inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1670 outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1671
1672 nw_off = inner_ip_off - ETH_HLEN;
1673 skb_set_network_header(skb, nw_off);
1674 iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1675 sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1676 skb_set_transport_header(skb, nw_off + iphdr_len);
1677
1678 if (inner_mac_off) { /* tunnel */
1679 __be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1680 ETH_HLEN - 2));
1681
1682 bnxt_gro_tunnel(skb, proto);
1683 }
1684 #endif
1685 return skb;
1686 }
1687
1688 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
1689 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1690
1691 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1692 int payload_off, int tcp_ts,
1693 struct sk_buff *skb)
1694 {
1695 #ifdef CONFIG_INET
1696 struct tcphdr *th;
1697 int len, nw_off, tcp_opt_len = 0;
1698
1699 if (tcp_ts)
1700 tcp_opt_len = 12;
1701
1702 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1703 struct iphdr *iph;
1704
1705 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1706 ETH_HLEN;
1707 skb_set_network_header(skb, nw_off);
1708 iph = ip_hdr(skb);
1709 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1710 len = skb->len - skb_transport_offset(skb);
1711 th = tcp_hdr(skb);
1712 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1713 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1714 struct ipv6hdr *iph;
1715
1716 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1717 ETH_HLEN;
1718 skb_set_network_header(skb, nw_off);
1719 iph = ipv6_hdr(skb);
1720 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1721 len = skb->len - skb_transport_offset(skb);
1722 th = tcp_hdr(skb);
1723 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1724 } else {
1725 dev_kfree_skb_any(skb);
1726 return NULL;
1727 }
1728
1729 if (nw_off) /* tunnel */
1730 bnxt_gro_tunnel(skb, skb->protocol);
1731 #endif
1732 return skb;
1733 }
1734
1735 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1736 struct bnxt_tpa_info *tpa_info,
1737 struct rx_tpa_end_cmp *tpa_end,
1738 struct rx_tpa_end_cmp_ext *tpa_end1,
1739 struct sk_buff *skb)
1740 {
1741 #ifdef CONFIG_INET
1742 int payload_off;
1743 u16 segs;
1744
1745 segs = TPA_END_TPA_SEGS(tpa_end);
1746 if (segs == 1)
1747 return skb;
1748
1749 NAPI_GRO_CB(skb)->count = segs;
1750 skb_shinfo(skb)->gso_size =
1751 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1752 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1753 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
1754 payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1755 else
1756 payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1757 skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1758 if (likely(skb))
1759 tcp_gro_complete(skb);
1760 #endif
1761 return skb;
1762 }
1763
1764 /* Given the cfa_code of a received packet determine which
1765 * netdev (vf-rep or PF) the packet is destined to.
1766 */
1767 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1768 {
1769 struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1770
1771 /* if vf-rep dev is NULL, the must belongs to the PF */
1772 return dev ? dev : bp->dev;
1773 }
1774
1775 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1776 struct bnxt_cp_ring_info *cpr,
1777 u32 *raw_cons,
1778 struct rx_tpa_end_cmp *tpa_end,
1779 struct rx_tpa_end_cmp_ext *tpa_end1,
1780 u8 *event)
1781 {
1782 struct bnxt_napi *bnapi = cpr->bnapi;
1783 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1784 struct net_device *dev = bp->dev;
1785 u8 *data_ptr, agg_bufs;
1786 unsigned int len;
1787 struct bnxt_tpa_info *tpa_info;
1788 dma_addr_t mapping;
1789 struct sk_buff *skb;
1790 u16 idx = 0, agg_id;
1791 void *data;
1792 bool gro;
1793
1794 if (unlikely(bnapi->in_reset)) {
1795 int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1796
1797 if (rc < 0)
1798 return ERR_PTR(-EBUSY);
1799 return NULL;
1800 }
1801
1802 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
1803 agg_id = TPA_END_AGG_ID_P5(tpa_end);
1804 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1805 agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1806 tpa_info = &rxr->rx_tpa[agg_id];
1807 if (unlikely(agg_bufs != tpa_info->agg_count)) {
1808 netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1809 agg_bufs, tpa_info->agg_count);
1810 agg_bufs = tpa_info->agg_count;
1811 }
1812 tpa_info->agg_count = 0;
1813 *event |= BNXT_AGG_EVENT;
1814 bnxt_free_agg_idx(rxr, agg_id);
1815 idx = agg_id;
1816 gro = !!(bp->flags & BNXT_FLAG_GRO);
1817 } else {
1818 agg_id = TPA_END_AGG_ID(tpa_end);
1819 agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1820 tpa_info = &rxr->rx_tpa[agg_id];
1821 idx = RING_CMP(*raw_cons);
1822 if (agg_bufs) {
1823 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1824 return ERR_PTR(-EBUSY);
1825
1826 *event |= BNXT_AGG_EVENT;
1827 idx = NEXT_CMP(idx);
1828 }
1829 gro = !!TPA_END_GRO(tpa_end);
1830 }
1831 data = tpa_info->data;
1832 data_ptr = tpa_info->data_ptr;
1833 prefetch(data_ptr);
1834 len = tpa_info->len;
1835 mapping = tpa_info->mapping;
1836
1837 if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1838 bnxt_abort_tpa(cpr, idx, agg_bufs);
1839 if (agg_bufs > MAX_SKB_FRAGS)
1840 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1841 agg_bufs, (int)MAX_SKB_FRAGS);
1842 return NULL;
1843 }
1844
1845 if (len <= bp->rx_copy_thresh) {
1846 skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1847 if (!skb) {
1848 bnxt_abort_tpa(cpr, idx, agg_bufs);
1849 cpr->sw_stats->rx.rx_oom_discards += 1;
1850 return NULL;
1851 }
1852 } else {
1853 u8 *new_data;
1854 dma_addr_t new_mapping;
1855
1856 new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, rxr,
1857 GFP_ATOMIC);
1858 if (!new_data) {
1859 bnxt_abort_tpa(cpr, idx, agg_bufs);
1860 cpr->sw_stats->rx.rx_oom_discards += 1;
1861 return NULL;
1862 }
1863
1864 tpa_info->data = new_data;
1865 tpa_info->data_ptr = new_data + bp->rx_offset;
1866 tpa_info->mapping = new_mapping;
1867
1868 skb = napi_build_skb(data, bp->rx_buf_size);
1869 dma_sync_single_for_cpu(&bp->pdev->dev, mapping,
1870 bp->rx_buf_use_size, bp->rx_dir);
1871
1872 if (!skb) {
1873 page_pool_free_va(rxr->head_pool, data, true);
1874 bnxt_abort_tpa(cpr, idx, agg_bufs);
1875 cpr->sw_stats->rx.rx_oom_discards += 1;
1876 return NULL;
1877 }
1878 skb_mark_for_recycle(skb);
1879 skb_reserve(skb, bp->rx_offset);
1880 skb_put(skb, len);
1881 }
1882
1883 if (agg_bufs) {
1884 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1885 if (!skb) {
1886 /* Page reuse already handled by bnxt_rx_pages(). */
1887 cpr->sw_stats->rx.rx_oom_discards += 1;
1888 return NULL;
1889 }
1890 }
1891
1892 if (tpa_info->cfa_code_valid)
1893 dev = bnxt_get_pkt_dev(bp, tpa_info->cfa_code);
1894 skb->protocol = eth_type_trans(skb, dev);
1895
1896 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1897 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1898
1899 if (tpa_info->vlan_valid &&
1900 (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1901 __be16 vlan_proto = htons(tpa_info->metadata >>
1902 RX_CMP_FLAGS2_METADATA_TPID_SFT);
1903 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1904
1905 if (eth_type_vlan(vlan_proto)) {
1906 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1907 } else {
1908 dev_kfree_skb(skb);
1909 return NULL;
1910 }
1911 }
1912
1913 skb_checksum_none_assert(skb);
1914 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1915 skb->ip_summed = CHECKSUM_UNNECESSARY;
1916 skb->csum_level =
1917 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1918 }
1919
1920 if (gro)
1921 skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1922
1923 return skb;
1924 }
1925
1926 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1927 struct rx_agg_cmp *rx_agg)
1928 {
1929 u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1930 struct bnxt_tpa_info *tpa_info;
1931
1932 agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1933 tpa_info = &rxr->rx_tpa[agg_id];
1934 BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1935 tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1936 }
1937
1938 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1939 struct sk_buff *skb)
1940 {
1941 skb_mark_for_recycle(skb);
1942
1943 if (skb->dev != bp->dev) {
1944 /* this packet belongs to a vf-rep */
1945 bnxt_vf_rep_rx(bp, skb);
1946 return;
1947 }
1948 skb_record_rx_queue(skb, bnapi->index);
1949 napi_gro_receive(&bnapi->napi, skb);
1950 }
1951
1952 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1953 struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1954 {
1955 u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1956
1957 if (BNXT_PTP_RX_TS_VALID(flags))
1958 goto ts_valid;
1959 if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1960 return false;
1961
1962 ts_valid:
1963 *cmpl_ts = ts;
1964 return true;
1965 }
1966
1967 static struct sk_buff *bnxt_rx_vlan(struct sk_buff *skb, u8 cmp_type,
1968 struct rx_cmp *rxcmp,
1969 struct rx_cmp_ext *rxcmp1)
1970 {
1971 __be16 vlan_proto;
1972 u16 vtag;
1973
1974 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1975 __le32 flags2 = rxcmp1->rx_cmp_flags2;
1976 u32 meta_data;
1977
1978 if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)))
1979 return skb;
1980
1981 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1982 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1983 vlan_proto = htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT);
1984 if (eth_type_vlan(vlan_proto))
1985 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1986 else
1987 goto vlan_err;
1988 } else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
1989 if (RX_CMP_VLAN_VALID(rxcmp)) {
1990 u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp);
1991
1992 if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q)
1993 vlan_proto = htons(ETH_P_8021Q);
1994 else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD)
1995 vlan_proto = htons(ETH_P_8021AD);
1996 else
1997 goto vlan_err;
1998 vtag = RX_CMP_METADATA0_TCI(rxcmp1);
1999 __vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
2000 }
2001 }
2002 return skb;
2003 vlan_err:
2004 dev_kfree_skb(skb);
2005 return NULL;
2006 }
2007
2008 static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp,
2009 struct rx_cmp *rxcmp)
2010 {
2011 u8 ext_op;
2012
2013 ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp);
2014 switch (ext_op) {
2015 case EXT_OP_INNER_4:
2016 case EXT_OP_OUTER_4:
2017 case EXT_OP_INNFL_3:
2018 case EXT_OP_OUTFL_3:
2019 return PKT_HASH_TYPE_L4;
2020 default:
2021 return PKT_HASH_TYPE_L3;
2022 }
2023 }
2024
2025 /* returns the following:
2026 * 1 - 1 packet successfully received
2027 * 0 - successful TPA_START, packet not completed yet
2028 * -EBUSY - completion ring does not have all the agg buffers yet
2029 * -ENOMEM - packet aborted due to out of memory
2030 * -EIO - packet aborted due to hw error indicated in BD
2031 */
2032 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2033 u32 *raw_cons, u8 *event)
2034 {
2035 struct bnxt_napi *bnapi = cpr->bnapi;
2036 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2037 struct net_device *dev = bp->dev;
2038 struct rx_cmp *rxcmp;
2039 struct rx_cmp_ext *rxcmp1;
2040 u32 tmp_raw_cons = *raw_cons;
2041 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
2042 struct bnxt_sw_rx_bd *rx_buf;
2043 unsigned int len;
2044 u8 *data_ptr, agg_bufs, cmp_type;
2045 bool xdp_active = false;
2046 dma_addr_t dma_addr;
2047 struct sk_buff *skb;
2048 struct xdp_buff xdp;
2049 u32 flags, misc;
2050 u32 cmpl_ts;
2051 void *data;
2052 int rc = 0;
2053
2054 rxcmp = (struct rx_cmp *)
2055 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2056
2057 cmp_type = RX_CMP_TYPE(rxcmp);
2058
2059 if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
2060 bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
2061 goto next_rx_no_prod_no_len;
2062 }
2063
2064 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2065 cp_cons = RING_CMP(tmp_raw_cons);
2066 rxcmp1 = (struct rx_cmp_ext *)
2067 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2068
2069 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2070 return -EBUSY;
2071
2072 /* The valid test of the entry must be done first before
2073 * reading any further.
2074 */
2075 dma_rmb();
2076 prod = rxr->rx_prod;
2077
2078 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
2079 cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2080 bnxt_tpa_start(bp, rxr, cmp_type,
2081 (struct rx_tpa_start_cmp *)rxcmp,
2082 (struct rx_tpa_start_cmp_ext *)rxcmp1);
2083
2084 *event |= BNXT_RX_EVENT;
2085 goto next_rx_no_prod_no_len;
2086
2087 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2088 skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
2089 (struct rx_tpa_end_cmp *)rxcmp,
2090 (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
2091
2092 if (IS_ERR(skb))
2093 return -EBUSY;
2094
2095 rc = -ENOMEM;
2096 if (likely(skb)) {
2097 bnxt_deliver_skb(bp, bnapi, skb);
2098 rc = 1;
2099 }
2100 *event |= BNXT_RX_EVENT;
2101 goto next_rx_no_prod_no_len;
2102 }
2103
2104 cons = rxcmp->rx_cmp_opaque;
2105 if (unlikely(cons != rxr->rx_next_cons)) {
2106 int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
2107
2108 /* 0xffff is forced error, don't print it */
2109 if (rxr->rx_next_cons != 0xffff)
2110 netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
2111 cons, rxr->rx_next_cons);
2112 bnxt_sched_reset_rxr(bp, rxr);
2113 if (rc1)
2114 return rc1;
2115 goto next_rx_no_prod_no_len;
2116 }
2117 rx_buf = &rxr->rx_buf_ring[cons];
2118 data = rx_buf->data;
2119 data_ptr = rx_buf->data_ptr;
2120 prefetch(data_ptr);
2121
2122 misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
2123 agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
2124
2125 if (agg_bufs) {
2126 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
2127 return -EBUSY;
2128
2129 cp_cons = NEXT_CMP(cp_cons);
2130 *event |= BNXT_AGG_EVENT;
2131 }
2132 *event |= BNXT_RX_EVENT;
2133
2134 rx_buf->data = NULL;
2135 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
2136 u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
2137
2138 bnxt_reuse_rx_data(rxr, cons, data);
2139 if (agg_bufs)
2140 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
2141 false);
2142
2143 rc = -EIO;
2144 if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
2145 bnapi->cp_ring.sw_stats->rx.rx_buf_errors++;
2146 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
2147 !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
2148 netdev_warn_once(bp->dev, "RX buffer error %x\n",
2149 rx_err);
2150 bnxt_sched_reset_rxr(bp, rxr);
2151 }
2152 }
2153 goto next_rx_no_len;
2154 }
2155
2156 flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
2157 len = flags >> RX_CMP_LEN_SHIFT;
2158 dma_addr = rx_buf->mapping;
2159
2160 if (bnxt_xdp_attached(bp, rxr)) {
2161 bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
2162 if (agg_bufs) {
2163 u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
2164 cp_cons, agg_bufs,
2165 false);
2166 if (!frag_len)
2167 goto oom_next_rx;
2168 }
2169 xdp_active = true;
2170 }
2171
2172 if (xdp_active) {
2173 if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) {
2174 rc = 1;
2175 goto next_rx;
2176 }
2177 }
2178
2179 if (len <= bp->rx_copy_thresh) {
2180 if (!xdp_active)
2181 skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
2182 else
2183 skb = bnxt_copy_xdp(bnapi, &xdp, len, dma_addr);
2184 bnxt_reuse_rx_data(rxr, cons, data);
2185 if (!skb) {
2186 if (agg_bufs) {
2187 if (!xdp_active)
2188 bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
2189 agg_bufs, false);
2190 else
2191 bnxt_xdp_buff_frags_free(rxr, &xdp);
2192 }
2193 goto oom_next_rx;
2194 }
2195 } else {
2196 u32 payload;
2197
2198 if (rx_buf->data_ptr == data_ptr)
2199 payload = misc & RX_CMP_PAYLOAD_OFFSET;
2200 else
2201 payload = 0;
2202 skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
2203 payload | len);
2204 if (!skb)
2205 goto oom_next_rx;
2206 }
2207
2208 if (agg_bufs) {
2209 if (!xdp_active) {
2210 skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
2211 if (!skb)
2212 goto oom_next_rx;
2213 } else {
2214 skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
2215 if (!skb) {
2216 /* we should be able to free the old skb here */
2217 bnxt_xdp_buff_frags_free(rxr, &xdp);
2218 goto oom_next_rx;
2219 }
2220 }
2221 }
2222
2223 if (RX_CMP_HASH_VALID(rxcmp)) {
2224 enum pkt_hash_types type;
2225
2226 if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2227 type = bnxt_rss_ext_op(bp, rxcmp);
2228 } else {
2229 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
2230
2231 /* RSS profiles 1 and 3 with extract code 0 for inner
2232 * 4-tuple
2233 */
2234 if (hash_type != 1 && hash_type != 3)
2235 type = PKT_HASH_TYPE_L3;
2236 else
2237 type = PKT_HASH_TYPE_L4;
2238 }
2239 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
2240 }
2241
2242 if (cmp_type == CMP_TYPE_RX_L2_CMP)
2243 dev = bnxt_get_pkt_dev(bp, RX_CMP_CFA_CODE(rxcmp1));
2244 skb->protocol = eth_type_trans(skb, dev);
2245
2246 if (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX) {
2247 skb = bnxt_rx_vlan(skb, cmp_type, rxcmp, rxcmp1);
2248 if (!skb)
2249 goto next_rx;
2250 }
2251
2252 skb_checksum_none_assert(skb);
2253 if (RX_CMP_L4_CS_OK(rxcmp1)) {
2254 if (dev->features & NETIF_F_RXCSUM) {
2255 skb->ip_summed = CHECKSUM_UNNECESSARY;
2256 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2257 }
2258 } else {
2259 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2260 if (dev->features & NETIF_F_RXCSUM)
2261 bnapi->cp_ring.sw_stats->rx.rx_l4_csum_errors++;
2262 }
2263 }
2264
2265 if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2266 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
2267 u64 ns, ts;
2268
2269 if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2270 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2271
2272 ns = bnxt_timecounter_cyc2time(ptp, ts);
2273 memset(skb_hwtstamps(skb), 0,
2274 sizeof(*skb_hwtstamps(skb)));
2275 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2276 }
2277 }
2278 }
2279 bnxt_deliver_skb(bp, bnapi, skb);
2280 rc = 1;
2281
2282 next_rx:
2283 cpr->rx_packets += 1;
2284 cpr->rx_bytes += len;
2285
2286 next_rx_no_len:
2287 rxr->rx_prod = NEXT_RX(prod);
2288 rxr->rx_next_cons = RING_RX(bp, NEXT_RX(cons));
2289
2290 next_rx_no_prod_no_len:
2291 *raw_cons = tmp_raw_cons;
2292
2293 return rc;
2294
2295 oom_next_rx:
2296 cpr->sw_stats->rx.rx_oom_discards += 1;
2297 rc = -ENOMEM;
2298 goto next_rx;
2299 }
2300
2301 /* In netpoll mode, if we are using a combined completion ring, we need to
2302 * discard the rx packets and recycle the buffers.
2303 */
2304 static int bnxt_force_rx_discard(struct bnxt *bp,
2305 struct bnxt_cp_ring_info *cpr,
2306 u32 *raw_cons, u8 *event)
2307 {
2308 u32 tmp_raw_cons = *raw_cons;
2309 struct rx_cmp_ext *rxcmp1;
2310 struct rx_cmp *rxcmp;
2311 u16 cp_cons;
2312 u8 cmp_type;
2313 int rc;
2314
2315 cp_cons = RING_CMP(tmp_raw_cons);
2316 rxcmp = (struct rx_cmp *)
2317 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2318
2319 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2320 cp_cons = RING_CMP(tmp_raw_cons);
2321 rxcmp1 = (struct rx_cmp_ext *)
2322 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2323
2324 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2325 return -EBUSY;
2326
2327 /* The valid test of the entry must be done first before
2328 * reading any further.
2329 */
2330 dma_rmb();
2331 cmp_type = RX_CMP_TYPE(rxcmp);
2332 if (cmp_type == CMP_TYPE_RX_L2_CMP ||
2333 cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
2334 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2335 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2336 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2337 struct rx_tpa_end_cmp_ext *tpa_end1;
2338
2339 tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2340 tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2341 cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2342 }
2343 rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2344 if (rc && rc != -EBUSY)
2345 cpr->sw_stats->rx.rx_netpoll_discards += 1;
2346 return rc;
2347 }
2348
2349 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2350 {
2351 struct bnxt_fw_health *fw_health = bp->fw_health;
2352 u32 reg = fw_health->regs[reg_idx];
2353 u32 reg_type, reg_off, val = 0;
2354
2355 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2356 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2357 switch (reg_type) {
2358 case BNXT_FW_HEALTH_REG_TYPE_CFG:
2359 pci_read_config_dword(bp->pdev, reg_off, &val);
2360 break;
2361 case BNXT_FW_HEALTH_REG_TYPE_GRC:
2362 reg_off = fw_health->mapped_regs[reg_idx];
2363 fallthrough;
2364 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2365 val = readl(bp->bar0 + reg_off);
2366 break;
2367 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2368 val = readl(bp->bar1 + reg_off);
2369 break;
2370 }
2371 if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2372 val &= fw_health->fw_reset_inprog_reg_mask;
2373 return val;
2374 }
2375
2376 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2377 {
2378 int i;
2379
2380 for (i = 0; i < bp->rx_nr_rings; i++) {
2381 u16 grp_idx = bp->rx_ring[i].bnapi->index;
2382 struct bnxt_ring_grp_info *grp_info;
2383
2384 grp_info = &bp->grp_info[grp_idx];
2385 if (grp_info->agg_fw_ring_id == ring_id)
2386 return grp_idx;
2387 }
2388 return INVALID_HW_RING_ID;
2389 }
2390
2391 static u16 bnxt_get_force_speed(struct bnxt_link_info *link_info)
2392 {
2393 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2394
2395 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
2396 return link_info->force_link_speed2;
2397 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4)
2398 return link_info->force_pam4_link_speed;
2399 return link_info->force_link_speed;
2400 }
2401
2402 static void bnxt_set_force_speed(struct bnxt_link_info *link_info)
2403 {
2404 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2405
2406 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2407 link_info->req_link_speed = link_info->force_link_speed2;
2408 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2409 switch (link_info->req_link_speed) {
2410 case BNXT_LINK_SPEED_50GB_PAM4:
2411 case BNXT_LINK_SPEED_100GB_PAM4:
2412 case BNXT_LINK_SPEED_200GB_PAM4:
2413 case BNXT_LINK_SPEED_400GB_PAM4:
2414 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2415 break;
2416 case BNXT_LINK_SPEED_100GB_PAM4_112:
2417 case BNXT_LINK_SPEED_200GB_PAM4_112:
2418 case BNXT_LINK_SPEED_400GB_PAM4_112:
2419 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4_112;
2420 break;
2421 default:
2422 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2423 }
2424 return;
2425 }
2426 link_info->req_link_speed = link_info->force_link_speed;
2427 link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
2428 if (link_info->force_pam4_link_speed) {
2429 link_info->req_link_speed = link_info->force_pam4_link_speed;
2430 link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
2431 }
2432 }
2433
2434 static void bnxt_set_auto_speed(struct bnxt_link_info *link_info)
2435 {
2436 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2437
2438 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2439 link_info->advertising = link_info->auto_link_speeds2;
2440 return;
2441 }
2442 link_info->advertising = link_info->auto_link_speeds;
2443 link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
2444 }
2445
2446 static bool bnxt_force_speed_updated(struct bnxt_link_info *link_info)
2447 {
2448 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2449
2450 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2451 if (link_info->req_link_speed != link_info->force_link_speed2)
2452 return true;
2453 return false;
2454 }
2455 if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
2456 link_info->req_link_speed != link_info->force_link_speed)
2457 return true;
2458 if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
2459 link_info->req_link_speed != link_info->force_pam4_link_speed)
2460 return true;
2461 return false;
2462 }
2463
2464 static bool bnxt_auto_speed_updated(struct bnxt_link_info *link_info)
2465 {
2466 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
2467
2468 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
2469 if (link_info->advertising != link_info->auto_link_speeds2)
2470 return true;
2471 return false;
2472 }
2473 if (link_info->advertising != link_info->auto_link_speeds ||
2474 link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
2475 return true;
2476 return false;
2477 }
2478
2479 bool bnxt_bs_trace_avail(struct bnxt *bp, u16 type)
2480 {
2481 u32 flags = bp->ctx->ctx_arr[type].flags;
2482
2483 return (flags & BNXT_CTX_MEM_TYPE_VALID) &&
2484 ((flags & BNXT_CTX_MEM_FW_TRACE) ||
2485 (flags & BNXT_CTX_MEM_FW_BIN_TRACE));
2486 }
2487
2488 static void bnxt_bs_trace_init(struct bnxt *bp, struct bnxt_ctx_mem_type *ctxm)
2489 {
2490 u32 mem_size, pages, rem_bytes, magic_byte_offset;
2491 u16 trace_type = bnxt_bstore_to_trace[ctxm->type];
2492 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
2493 struct bnxt_ring_mem_info *rmem, *rmem_pg_tbl;
2494 struct bnxt_bs_trace_info *bs_trace;
2495 int last_pg;
2496
2497 if (ctxm->instance_bmap && ctxm->instance_bmap > 1)
2498 return;
2499
2500 mem_size = ctxm->max_entries * ctxm->entry_size;
2501 rem_bytes = mem_size % BNXT_PAGE_SIZE;
2502 pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
2503
2504 last_pg = (pages - 1) & (MAX_CTX_PAGES - 1);
2505 magic_byte_offset = (rem_bytes ? rem_bytes : BNXT_PAGE_SIZE) - 1;
2506
2507 rmem = &ctx_pg[0].ring_mem;
2508 bs_trace = &bp->bs_trace[trace_type];
2509 bs_trace->ctx_type = ctxm->type;
2510 bs_trace->trace_type = trace_type;
2511 if (pages > MAX_CTX_PAGES) {
2512 int last_pg_dir = rmem->nr_pages - 1;
2513
2514 rmem_pg_tbl = &ctx_pg[0].ctx_pg_tbl[last_pg_dir]->ring_mem;
2515 bs_trace->magic_byte = rmem_pg_tbl->pg_arr[last_pg];
2516 } else {
2517 bs_trace->magic_byte = rmem->pg_arr[last_pg];
2518 }
2519 bs_trace->magic_byte += magic_byte_offset;
2520 *bs_trace->magic_byte = BNXT_TRACE_BUF_MAGIC_BYTE;
2521 }
2522
2523 #define BNXT_EVENT_BUF_PRODUCER_TYPE(data1) \
2524 (((data1) & ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_MASK) >>\
2525 ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA1_TYPE_SFT)
2526
2527 #define BNXT_EVENT_BUF_PRODUCER_OFFSET(data2) \
2528 (((data2) & \
2529 ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_MASK) >>\
2530 ASYNC_EVENT_CMPL_DBG_BUF_PRODUCER_EVENT_DATA2_CURR_OFF_SFT)
2531
2532 #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \
2533 ((data2) & \
2534 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK)
2535
2536 #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \
2537 (((data2) & \
2538 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >>\
2539 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT)
2540
2541 #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \
2542 ((data1) & \
2543 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK)
2544
2545 #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \
2546 (((data1) & \
2547 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) ==\
2548 ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING)
2549
2550 /* Return true if the workqueue has to be scheduled */
2551 static bool bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2552 {
2553 u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2554
2555 switch (err_type) {
2556 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2557 netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2558 BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2559 break;
2560 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2561 netdev_warn(bp->dev, "Pause Storm detected!\n");
2562 break;
2563 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2564 netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2565 break;
2566 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: {
2567 u32 type = EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1);
2568 char *threshold_type;
2569 bool notify = false;
2570 char *dir_str;
2571
2572 switch (type) {
2573 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN:
2574 threshold_type = "warning";
2575 break;
2576 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL:
2577 threshold_type = "critical";
2578 break;
2579 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL:
2580 threshold_type = "fatal";
2581 break;
2582 case ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN:
2583 threshold_type = "shutdown";
2584 break;
2585 default:
2586 netdev_err(bp->dev, "Unknown Thermal threshold type event\n");
2587 return false;
2588 }
2589 if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) {
2590 dir_str = "above";
2591 notify = true;
2592 } else {
2593 dir_str = "below";
2594 }
2595 netdev_warn(bp->dev, "Chip temperature has gone %s the %s thermal threshold!\n",
2596 dir_str, threshold_type);
2597 netdev_warn(bp->dev, "Temperature (In Celsius), Current: %lu, threshold: %lu\n",
2598 BNXT_EVENT_THERMAL_CURRENT_TEMP(data2),
2599 BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2));
2600 if (notify) {
2601 bp->thermal_threshold_type = type;
2602 set_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event);
2603 return true;
2604 }
2605 return false;
2606 }
2607 case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED:
2608 netdev_warn(bp->dev, "Speed change not supported with dual rate transceivers on this board\n");
2609 break;
2610 default:
2611 netdev_err(bp->dev, "FW reported unknown error type %u\n",
2612 err_type);
2613 break;
2614 }
2615 return false;
2616 }
2617
2618 #define BNXT_GET_EVENT_PORT(data) \
2619 ((data) & \
2620 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2621
2622 #define BNXT_EVENT_RING_TYPE(data2) \
2623 ((data2) & \
2624 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2625
2626 #define BNXT_EVENT_RING_TYPE_RX(data2) \
2627 (BNXT_EVENT_RING_TYPE(data2) == \
2628 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2629
2630 #define BNXT_EVENT_PHC_EVENT_TYPE(data1) \
2631 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2632 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2633
2634 #define BNXT_EVENT_PHC_RTC_UPDATE(data1) \
2635 (((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2636 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2637
2638 #define BNXT_PHC_BITS 48
2639
2640 static int bnxt_async_event_process(struct bnxt *bp,
2641 struct hwrm_async_event_cmpl *cmpl)
2642 {
2643 u16 event_id = le16_to_cpu(cmpl->event_id);
2644 u32 data1 = le32_to_cpu(cmpl->event_data1);
2645 u32 data2 = le32_to_cpu(cmpl->event_data2);
2646
2647 netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2648 event_id, data1, data2);
2649
2650 /* TODO CHIMP_FW: Define event id's for link change, error etc */
2651 switch (event_id) {
2652 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2653 struct bnxt_link_info *link_info = &bp->link_info;
2654
2655 if (BNXT_VF(bp))
2656 goto async_event_process_exit;
2657
2658 /* print unsupported speed warning in forced speed mode only */
2659 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2660 (data1 & 0x20000)) {
2661 u16 fw_speed = bnxt_get_force_speed(link_info);
2662 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2663
2664 if (speed != SPEED_UNKNOWN)
2665 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2666 speed);
2667 }
2668 set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2669 }
2670 fallthrough;
2671 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2672 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2673 set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2674 fallthrough;
2675 case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2676 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2677 break;
2678 case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2679 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2680 break;
2681 case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2682 u16 port_id = BNXT_GET_EVENT_PORT(data1);
2683
2684 if (BNXT_VF(bp))
2685 break;
2686
2687 if (bp->pf.port_id != port_id)
2688 break;
2689
2690 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2691 break;
2692 }
2693 case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2694 if (BNXT_PF(bp))
2695 goto async_event_process_exit;
2696 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2697 break;
2698 case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2699 char *type_str = "Solicited";
2700
2701 if (!bp->fw_health)
2702 goto async_event_process_exit;
2703
2704 bp->fw_reset_timestamp = jiffies;
2705 bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2706 if (!bp->fw_reset_min_dsecs)
2707 bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2708 bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2709 if (!bp->fw_reset_max_dsecs)
2710 bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2711 if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2712 set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2713 } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2714 type_str = "Fatal";
2715 bp->fw_health->fatalities++;
2716 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2717 } else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2718 EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2719 type_str = "Non-fatal";
2720 bp->fw_health->survivals++;
2721 set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2722 }
2723 netif_warn(bp, hw, bp->dev,
2724 "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2725 type_str, data1, data2,
2726 bp->fw_reset_min_dsecs * 100,
2727 bp->fw_reset_max_dsecs * 100);
2728 set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2729 break;
2730 }
2731 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2732 struct bnxt_fw_health *fw_health = bp->fw_health;
2733 char *status_desc = "healthy";
2734 u32 status;
2735
2736 if (!fw_health)
2737 goto async_event_process_exit;
2738
2739 if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2740 fw_health->enabled = false;
2741 netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2742 break;
2743 }
2744 fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2745 fw_health->tmr_multiplier =
2746 DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2747 bp->current_interval * 10);
2748 fw_health->tmr_counter = fw_health->tmr_multiplier;
2749 if (!fw_health->enabled)
2750 fw_health->last_fw_heartbeat =
2751 bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2752 fw_health->last_fw_reset_cnt =
2753 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2754 status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2755 if (status != BNXT_FW_STATUS_HEALTHY)
2756 status_desc = "unhealthy";
2757 netif_info(bp, drv, bp->dev,
2758 "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2759 fw_health->primary ? "primary" : "backup", status,
2760 status_desc, fw_health->last_fw_reset_cnt);
2761 if (!fw_health->enabled) {
2762 /* Make sure tmr_counter is set and visible to
2763 * bnxt_health_check() before setting enabled to true.
2764 */
2765 smp_wmb();
2766 fw_health->enabled = true;
2767 }
2768 goto async_event_process_exit;
2769 }
2770 case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2771 netif_notice(bp, hw, bp->dev,
2772 "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2773 data1, data2);
2774 goto async_event_process_exit;
2775 case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2776 struct bnxt_rx_ring_info *rxr;
2777 u16 grp_idx;
2778
2779 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
2780 goto async_event_process_exit;
2781
2782 netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2783 BNXT_EVENT_RING_TYPE(data2), data1);
2784 if (!BNXT_EVENT_RING_TYPE_RX(data2))
2785 goto async_event_process_exit;
2786
2787 grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2788 if (grp_idx == INVALID_HW_RING_ID) {
2789 netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2790 data1);
2791 goto async_event_process_exit;
2792 }
2793 rxr = bp->bnapi[grp_idx]->rx_ring;
2794 bnxt_sched_reset_rxr(bp, rxr);
2795 goto async_event_process_exit;
2796 }
2797 case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2798 struct bnxt_fw_health *fw_health = bp->fw_health;
2799
2800 netif_notice(bp, hw, bp->dev,
2801 "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2802 data1, data2);
2803 if (fw_health) {
2804 fw_health->echo_req_data1 = data1;
2805 fw_health->echo_req_data2 = data2;
2806 set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2807 break;
2808 }
2809 goto async_event_process_exit;
2810 }
2811 case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2812 bnxt_ptp_pps_event(bp, data1, data2);
2813 goto async_event_process_exit;
2814 }
2815 case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2816 if (bnxt_event_error_report(bp, data1, data2))
2817 break;
2818 goto async_event_process_exit;
2819 }
2820 case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2821 switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2822 case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2823 if (BNXT_PTP_USE_RTC(bp)) {
2824 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2825 unsigned long flags;
2826 u64 ns;
2827
2828 if (!ptp)
2829 goto async_event_process_exit;
2830
2831 bnxt_ptp_update_current_time(bp);
2832 ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2833 BNXT_PHC_BITS) | ptp->current_time);
2834 write_seqlock_irqsave(&ptp->ptp_lock, flags);
2835 bnxt_ptp_rtc_timecounter_init(ptp, ns);
2836 write_sequnlock_irqrestore(&ptp->ptp_lock, flags);
2837 }
2838 break;
2839 }
2840 goto async_event_process_exit;
2841 }
2842 case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2843 u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2844
2845 hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2846 goto async_event_process_exit;
2847 }
2848 case ASYNC_EVENT_CMPL_EVENT_ID_DBG_BUF_PRODUCER: {
2849 u16 type = (u16)BNXT_EVENT_BUF_PRODUCER_TYPE(data1);
2850 u32 offset = BNXT_EVENT_BUF_PRODUCER_OFFSET(data2);
2851
2852 bnxt_bs_trace_check_wrap(&bp->bs_trace[type], offset);
2853 goto async_event_process_exit;
2854 }
2855 default:
2856 goto async_event_process_exit;
2857 }
2858 __bnxt_queue_sp_work(bp);
2859 async_event_process_exit:
2860 return 0;
2861 }
2862
2863 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2864 {
2865 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2866 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2867 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2868 (struct hwrm_fwd_req_cmpl *)txcmp;
2869
2870 switch (cmpl_type) {
2871 case CMPL_BASE_TYPE_HWRM_DONE:
2872 seq_id = le16_to_cpu(h_cmpl->sequence_id);
2873 hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2874 break;
2875
2876 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2877 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2878
2879 if ((vf_id < bp->pf.first_vf_id) ||
2880 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2881 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2882 vf_id);
2883 return -EINVAL;
2884 }
2885
2886 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2887 bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2888 break;
2889
2890 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2891 bnxt_async_event_process(bp,
2892 (struct hwrm_async_event_cmpl *)txcmp);
2893 break;
2894
2895 default:
2896 break;
2897 }
2898
2899 return 0;
2900 }
2901
2902 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2903 {
2904 struct bnxt_napi *bnapi = dev_instance;
2905 struct bnxt *bp = bnapi->bp;
2906 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2907 u32 cons = RING_CMP(cpr->cp_raw_cons);
2908
2909 cpr->event_ctr++;
2910 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2911 napi_schedule(&bnapi->napi);
2912 return IRQ_HANDLED;
2913 }
2914
2915 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2916 {
2917 u32 raw_cons = cpr->cp_raw_cons;
2918 u16 cons = RING_CMP(raw_cons);
2919 struct tx_cmp *txcmp;
2920
2921 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2922
2923 return TX_CMP_VALID(txcmp, raw_cons);
2924 }
2925
2926 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2927 int budget)
2928 {
2929 struct bnxt_napi *bnapi = cpr->bnapi;
2930 u32 raw_cons = cpr->cp_raw_cons;
2931 u32 cons;
2932 int rx_pkts = 0;
2933 u8 event = 0;
2934 struct tx_cmp *txcmp;
2935
2936 cpr->has_more_work = 0;
2937 cpr->had_work_done = 1;
2938 while (1) {
2939 u8 cmp_type;
2940 int rc;
2941
2942 cons = RING_CMP(raw_cons);
2943 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2944
2945 if (!TX_CMP_VALID(txcmp, raw_cons))
2946 break;
2947
2948 /* The valid test of the entry must be done first before
2949 * reading any further.
2950 */
2951 dma_rmb();
2952 cmp_type = TX_CMP_TYPE(txcmp);
2953 if (cmp_type == CMP_TYPE_TX_L2_CMP ||
2954 cmp_type == CMP_TYPE_TX_L2_COAL_CMP) {
2955 u32 opaque = txcmp->tx_cmp_opaque;
2956 struct bnxt_tx_ring_info *txr;
2957 u16 tx_freed;
2958
2959 txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
2960 event |= BNXT_TX_CMP_EVENT;
2961 if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP)
2962 txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp);
2963 else
2964 txr->tx_hw_cons = TX_OPAQUE_PROD(bp, opaque);
2965 tx_freed = (txr->tx_hw_cons - txr->tx_cons) &
2966 bp->tx_ring_mask;
2967 /* return full budget so NAPI will complete. */
2968 if (unlikely(tx_freed >= bp->tx_wake_thresh)) {
2969 rx_pkts = budget;
2970 raw_cons = NEXT_RAW_CMP(raw_cons);
2971 if (budget)
2972 cpr->has_more_work = 1;
2973 break;
2974 }
2975 } else if (cmp_type == CMP_TYPE_TX_L2_PKT_TS_CMP) {
2976 bnxt_tx_ts_cmp(bp, bnapi, (struct tx_ts_cmp *)txcmp);
2977 } else if (cmp_type >= CMP_TYPE_RX_L2_CMP &&
2978 cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
2979 if (likely(budget))
2980 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2981 else
2982 rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2983 &event);
2984 if (likely(rc >= 0))
2985 rx_pkts += rc;
2986 /* Increment rx_pkts when rc is -ENOMEM to count towards
2987 * the NAPI budget. Otherwise, we may potentially loop
2988 * here forever if we consistently cannot allocate
2989 * buffers.
2990 */
2991 else if (rc == -ENOMEM && budget)
2992 rx_pkts++;
2993 else if (rc == -EBUSY) /* partial completion */
2994 break;
2995 } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE ||
2996 cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ ||
2997 cmp_type == CMPL_BASE_TYPE_HWRM_ASYNC_EVENT)) {
2998 bnxt_hwrm_handler(bp, txcmp);
2999 }
3000 raw_cons = NEXT_RAW_CMP(raw_cons);
3001
3002 if (rx_pkts && rx_pkts == budget) {
3003 cpr->has_more_work = 1;
3004 break;
3005 }
3006 }
3007
3008 if (event & BNXT_REDIRECT_EVENT) {
3009 xdp_do_flush();
3010 event &= ~BNXT_REDIRECT_EVENT;
3011 }
3012
3013 if (event & BNXT_TX_EVENT) {
3014 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];
3015 u16 prod = txr->tx_prod;
3016
3017 /* Sync BD data before updating doorbell */
3018 wmb();
3019
3020 bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
3021 event &= ~BNXT_TX_EVENT;
3022 }
3023
3024 cpr->cp_raw_cons = raw_cons;
3025 bnapi->events |= event;
3026 return rx_pkts;
3027 }
3028
3029 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
3030 int budget)
3031 {
3032 if ((bnapi->events & BNXT_TX_CMP_EVENT) && !bnapi->tx_fault)
3033 bnapi->tx_int(bp, bnapi, budget);
3034
3035 if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
3036 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
3037
3038 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
3039 bnapi->events &= ~BNXT_RX_EVENT;
3040 }
3041 if (bnapi->events & BNXT_AGG_EVENT) {
3042 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
3043
3044 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
3045 bnapi->events &= ~BNXT_AGG_EVENT;
3046 }
3047 }
3048
3049 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
3050 int budget)
3051 {
3052 struct bnxt_napi *bnapi = cpr->bnapi;
3053 int rx_pkts;
3054
3055 rx_pkts = __bnxt_poll_work(bp, cpr, budget);
3056
3057 /* ACK completion ring before freeing tx ring and producing new
3058 * buffers in rx/agg rings to prevent overflowing the completion
3059 * ring.
3060 */
3061 bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
3062
3063 __bnxt_poll_work_done(bp, bnapi, budget);
3064 return rx_pkts;
3065 }
3066
3067 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
3068 {
3069 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3070 struct bnxt *bp = bnapi->bp;
3071 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3072 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
3073 struct tx_cmp *txcmp;
3074 struct rx_cmp_ext *rxcmp1;
3075 u32 cp_cons, tmp_raw_cons;
3076 u32 raw_cons = cpr->cp_raw_cons;
3077 bool flush_xdp = false;
3078 u32 rx_pkts = 0;
3079 u8 event = 0;
3080
3081 while (1) {
3082 int rc;
3083
3084 cp_cons = RING_CMP(raw_cons);
3085 txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
3086
3087 if (!TX_CMP_VALID(txcmp, raw_cons))
3088 break;
3089
3090 /* The valid test of the entry must be done first before
3091 * reading any further.
3092 */
3093 dma_rmb();
3094 if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
3095 tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
3096 cp_cons = RING_CMP(tmp_raw_cons);
3097 rxcmp1 = (struct rx_cmp_ext *)
3098 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
3099
3100 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
3101 break;
3102
3103 /* force an error to recycle the buffer */
3104 rxcmp1->rx_cmp_cfa_code_errors_v2 |=
3105 cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
3106
3107 rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
3108 if (likely(rc == -EIO) && budget)
3109 rx_pkts++;
3110 else if (rc == -EBUSY) /* partial completion */
3111 break;
3112 if (event & BNXT_REDIRECT_EVENT)
3113 flush_xdp = true;
3114 } else if (unlikely(TX_CMP_TYPE(txcmp) ==
3115 CMPL_BASE_TYPE_HWRM_DONE)) {
3116 bnxt_hwrm_handler(bp, txcmp);
3117 } else {
3118 netdev_err(bp->dev,
3119 "Invalid completion received on special ring\n");
3120 }
3121 raw_cons = NEXT_RAW_CMP(raw_cons);
3122
3123 if (rx_pkts == budget)
3124 break;
3125 }
3126
3127 cpr->cp_raw_cons = raw_cons;
3128 BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
3129 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
3130
3131 if (event & BNXT_AGG_EVENT)
3132 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
3133 if (flush_xdp)
3134 xdp_do_flush();
3135
3136 if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
3137 napi_complete_done(napi, rx_pkts);
3138 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3139 }
3140 return rx_pkts;
3141 }
3142
3143 static int bnxt_poll(struct napi_struct *napi, int budget)
3144 {
3145 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3146 struct bnxt *bp = bnapi->bp;
3147 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3148 int work_done = 0;
3149
3150 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3151 napi_complete(napi);
3152 return 0;
3153 }
3154 while (1) {
3155 work_done += bnxt_poll_work(bp, cpr, budget - work_done);
3156
3157 if (work_done >= budget) {
3158 if (!budget)
3159 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3160 break;
3161 }
3162
3163 if (!bnxt_has_work(bp, cpr)) {
3164 if (napi_complete_done(napi, work_done))
3165 BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
3166 break;
3167 }
3168 }
3169 if (bp->flags & BNXT_FLAG_DIM) {
3170 struct dim_sample dim_sample = {};
3171
3172 dim_update_sample(cpr->event_ctr,
3173 cpr->rx_packets,
3174 cpr->rx_bytes,
3175 &dim_sample);
3176 net_dim(&cpr->dim, &dim_sample);
3177 }
3178 return work_done;
3179 }
3180
3181 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
3182 {
3183 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3184 int i, work_done = 0;
3185
3186 for (i = 0; i < cpr->cp_ring_count; i++) {
3187 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3188
3189 if (cpr2->had_nqe_notify) {
3190 work_done += __bnxt_poll_work(bp, cpr2,
3191 budget - work_done);
3192 cpr->has_more_work |= cpr2->has_more_work;
3193 }
3194 }
3195 return work_done;
3196 }
3197
3198 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
3199 u64 dbr_type, int budget)
3200 {
3201 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3202 int i;
3203
3204 for (i = 0; i < cpr->cp_ring_count; i++) {
3205 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[i];
3206 struct bnxt_db_info *db;
3207
3208 if (cpr2->had_work_done) {
3209 u32 tgl = 0;
3210
3211 if (dbr_type == DBR_TYPE_CQ_ARMALL) {
3212 cpr2->had_nqe_notify = 0;
3213 tgl = cpr2->toggle;
3214 }
3215 db = &cpr2->cp_db;
3216 bnxt_writeq(bp,
3217 db->db_key64 | dbr_type | DB_TOGGLE(tgl) |
3218 DB_RING_IDX(db, cpr2->cp_raw_cons),
3219 db->doorbell);
3220 cpr2->had_work_done = 0;
3221 }
3222 }
3223 __bnxt_poll_work_done(bp, bnapi, budget);
3224 }
3225
3226 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
3227 {
3228 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
3229 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3230 struct bnxt_cp_ring_info *cpr_rx;
3231 u32 raw_cons = cpr->cp_raw_cons;
3232 struct bnxt *bp = bnapi->bp;
3233 struct nqe_cn *nqcmp;
3234 int work_done = 0;
3235 u32 cons;
3236
3237 if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
3238 napi_complete(napi);
3239 return 0;
3240 }
3241 if (cpr->has_more_work) {
3242 cpr->has_more_work = 0;
3243 work_done = __bnxt_poll_cqs(bp, bnapi, budget);
3244 }
3245 while (1) {
3246 u16 type;
3247
3248 cons = RING_CMP(raw_cons);
3249 nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
3250
3251 if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
3252 if (cpr->has_more_work)
3253 break;
3254
3255 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
3256 budget);
3257 cpr->cp_raw_cons = raw_cons;
3258 if (napi_complete_done(napi, work_done))
3259 BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
3260 cpr->cp_raw_cons);
3261 goto poll_done;
3262 }
3263
3264 /* The valid test of the entry must be done first before
3265 * reading any further.
3266 */
3267 dma_rmb();
3268
3269 type = le16_to_cpu(nqcmp->type);
3270 if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
3271 u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
3272 u32 cq_type = BNXT_NQ_HDL_TYPE(idx);
3273 struct bnxt_cp_ring_info *cpr2;
3274
3275 /* No more budget for RX work */
3276 if (budget && work_done >= budget &&
3277 cq_type == BNXT_NQ_HDL_TYPE_RX)
3278 break;
3279
3280 idx = BNXT_NQ_HDL_IDX(idx);
3281 cpr2 = &cpr->cp_ring_arr[idx];
3282 cpr2->had_nqe_notify = 1;
3283 cpr2->toggle = NQE_CN_TOGGLE(type);
3284 work_done += __bnxt_poll_work(bp, cpr2,
3285 budget - work_done);
3286 cpr->has_more_work |= cpr2->has_more_work;
3287 } else {
3288 bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
3289 }
3290 raw_cons = NEXT_RAW_CMP(raw_cons);
3291 }
3292 __bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
3293 if (raw_cons != cpr->cp_raw_cons) {
3294 cpr->cp_raw_cons = raw_cons;
3295 BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
3296 }
3297 poll_done:
3298 cpr_rx = &cpr->cp_ring_arr[0];
3299 if (cpr_rx->cp_ring_type == BNXT_NQ_HDL_TYPE_RX &&
3300 (bp->flags & BNXT_FLAG_DIM)) {
3301 struct dim_sample dim_sample = {};
3302
3303 dim_update_sample(cpr->event_ctr,
3304 cpr_rx->rx_packets,
3305 cpr_rx->rx_bytes,
3306 &dim_sample);
3307 net_dim(&cpr->dim, &dim_sample);
3308 }
3309 return work_done;
3310 }
3311
3312 static void bnxt_free_tx_skbs(struct bnxt *bp)
3313 {
3314 int i, max_idx;
3315 struct pci_dev *pdev = bp->pdev;
3316
3317 if (!bp->tx_ring)
3318 return;
3319
3320 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
3321 for (i = 0; i < bp->tx_nr_rings; i++) {
3322 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3323 int j;
3324
3325 if (!txr->tx_buf_ring)
3326 continue;
3327
3328 for (j = 0; j < max_idx;) {
3329 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
3330 struct sk_buff *skb;
3331 int k, last;
3332
3333 if (i < bp->tx_nr_rings_xdp &&
3334 tx_buf->action == XDP_REDIRECT) {
3335 dma_unmap_single(&pdev->dev,
3336 dma_unmap_addr(tx_buf, mapping),
3337 dma_unmap_len(tx_buf, len),
3338 DMA_TO_DEVICE);
3339 xdp_return_frame(tx_buf->xdpf);
3340 tx_buf->action = 0;
3341 tx_buf->xdpf = NULL;
3342 j++;
3343 continue;
3344 }
3345
3346 skb = tx_buf->skb;
3347 if (!skb) {
3348 j++;
3349 continue;
3350 }
3351
3352 tx_buf->skb = NULL;
3353
3354 if (tx_buf->is_push) {
3355 dev_kfree_skb(skb);
3356 j += 2;
3357 continue;
3358 }
3359
3360 dma_unmap_single(&pdev->dev,
3361 dma_unmap_addr(tx_buf, mapping),
3362 skb_headlen(skb),
3363 DMA_TO_DEVICE);
3364
3365 last = tx_buf->nr_frags;
3366 j += 2;
3367 for (k = 0; k < last; k++, j++) {
3368 int ring_idx = j & bp->tx_ring_mask;
3369 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
3370
3371 tx_buf = &txr->tx_buf_ring[ring_idx];
3372 dma_unmap_page(
3373 &pdev->dev,
3374 dma_unmap_addr(tx_buf, mapping),
3375 skb_frag_size(frag), DMA_TO_DEVICE);
3376 }
3377 dev_kfree_skb(skb);
3378 }
3379 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
3380 }
3381 }
3382
3383 static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
3384 {
3385 int i, max_idx;
3386
3387 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
3388
3389 for (i = 0; i < max_idx; i++) {
3390 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
3391 void *data = rx_buf->data;
3392
3393 if (!data)
3394 continue;
3395
3396 rx_buf->data = NULL;
3397 if (BNXT_RX_PAGE_MODE(bp))
3398 page_pool_recycle_direct(rxr->page_pool, data);
3399 else
3400 page_pool_free_va(rxr->head_pool, data, true);
3401 }
3402 }
3403
3404 static void bnxt_free_one_rx_agg_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
3405 {
3406 int i, max_idx;
3407
3408 max_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
3409
3410 for (i = 0; i < max_idx; i++) {
3411 struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
3412 struct page *page = rx_agg_buf->page;
3413
3414 if (!page)
3415 continue;
3416
3417 rx_agg_buf->page = NULL;
3418 __clear_bit(i, rxr->rx_agg_bmap);
3419
3420 page_pool_recycle_direct(rxr->page_pool, page);
3421 }
3422 }
3423
3424 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
3425 {
3426 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3427 struct bnxt_tpa_idx_map *map;
3428 int i;
3429
3430 if (!rxr->rx_tpa)
3431 goto skip_rx_tpa_free;
3432
3433 for (i = 0; i < bp->max_tpa; i++) {
3434 struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
3435 u8 *data = tpa_info->data;
3436
3437 if (!data)
3438 continue;
3439
3440 tpa_info->data = NULL;
3441 page_pool_free_va(rxr->head_pool, data, false);
3442 }
3443
3444 skip_rx_tpa_free:
3445 if (!rxr->rx_buf_ring)
3446 goto skip_rx_buf_free;
3447
3448 bnxt_free_one_rx_ring(bp, rxr);
3449
3450 skip_rx_buf_free:
3451 if (!rxr->rx_agg_ring)
3452 goto skip_rx_agg_free;
3453
3454 bnxt_free_one_rx_agg_ring(bp, rxr);
3455
3456 skip_rx_agg_free:
3457 map = rxr->rx_tpa_idx_map;
3458 if (map)
3459 memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
3460 }
3461
3462 static void bnxt_free_rx_skbs(struct bnxt *bp)
3463 {
3464 int i;
3465
3466 if (!bp->rx_ring)
3467 return;
3468
3469 for (i = 0; i < bp->rx_nr_rings; i++)
3470 bnxt_free_one_rx_ring_skbs(bp, i);
3471 }
3472
3473 static void bnxt_free_skbs(struct bnxt *bp)
3474 {
3475 bnxt_free_tx_skbs(bp);
3476 bnxt_free_rx_skbs(bp);
3477 }
3478
3479 static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len)
3480 {
3481 u8 init_val = ctxm->init_value;
3482 u16 offset = ctxm->init_offset;
3483 u8 *p2 = p;
3484 int i;
3485
3486 if (!init_val)
3487 return;
3488 if (offset == BNXT_CTX_INIT_INVALID_OFFSET) {
3489 memset(p, init_val, len);
3490 return;
3491 }
3492 for (i = 0; i < len; i += ctxm->entry_size)
3493 *(p2 + i + offset) = init_val;
3494 }
3495
3496 static size_t __bnxt_copy_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem,
3497 void *buf, size_t offset, size_t head,
3498 size_t tail)
3499 {
3500 int i, head_page, start_idx, source_offset;
3501 size_t len, rem_len, total_len, max_bytes;
3502
3503 head_page = head / rmem->page_size;
3504 source_offset = head % rmem->page_size;
3505 total_len = (tail - head) & MAX_CTX_BYTES_MASK;
3506 if (!total_len)
3507 total_len = MAX_CTX_BYTES;
3508 start_idx = head_page % MAX_CTX_PAGES;
3509 max_bytes = (rmem->nr_pages - start_idx) * rmem->page_size -
3510 source_offset;
3511 total_len = min(total_len, max_bytes);
3512 rem_len = total_len;
3513
3514 for (i = start_idx; rem_len; i++, source_offset = 0) {
3515 len = min((size_t)(rmem->page_size - source_offset), rem_len);
3516 if (buf)
3517 memcpy(buf + offset, rmem->pg_arr[i] + source_offset,
3518 len);
3519 offset += len;
3520 rem_len -= len;
3521 }
3522 return total_len;
3523 }
3524
3525 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3526 {
3527 struct pci_dev *pdev = bp->pdev;
3528 int i;
3529
3530 if (!rmem->pg_arr)
3531 goto skip_pages;
3532
3533 for (i = 0; i < rmem->nr_pages; i++) {
3534 if (!rmem->pg_arr[i])
3535 continue;
3536
3537 dma_free_coherent(&pdev->dev, rmem->page_size,
3538 rmem->pg_arr[i], rmem->dma_arr[i]);
3539
3540 rmem->pg_arr[i] = NULL;
3541 }
3542 skip_pages:
3543 if (rmem->pg_tbl) {
3544 size_t pg_tbl_size = rmem->nr_pages * 8;
3545
3546 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3547 pg_tbl_size = rmem->page_size;
3548 dma_free_coherent(&pdev->dev, pg_tbl_size,
3549 rmem->pg_tbl, rmem->pg_tbl_map);
3550 rmem->pg_tbl = NULL;
3551 }
3552 if (rmem->vmem_size && *rmem->vmem) {
3553 vfree(*rmem->vmem);
3554 *rmem->vmem = NULL;
3555 }
3556 }
3557
3558 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3559 {
3560 struct pci_dev *pdev = bp->pdev;
3561 u64 valid_bit = 0;
3562 int i;
3563
3564 if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3565 valid_bit = PTU_PTE_VALID;
3566 if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3567 size_t pg_tbl_size = rmem->nr_pages * 8;
3568
3569 if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3570 pg_tbl_size = rmem->page_size;
3571 rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3572 &rmem->pg_tbl_map,
3573 GFP_KERNEL);
3574 if (!rmem->pg_tbl)
3575 return -ENOMEM;
3576 }
3577
3578 for (i = 0; i < rmem->nr_pages; i++) {
3579 u64 extra_bits = valid_bit;
3580
3581 rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3582 rmem->page_size,
3583 &rmem->dma_arr[i],
3584 GFP_KERNEL);
3585 if (!rmem->pg_arr[i])
3586 return -ENOMEM;
3587
3588 if (rmem->ctx_mem)
3589 bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i],
3590 rmem->page_size);
3591 if (rmem->nr_pages > 1 || rmem->depth > 0) {
3592 if (i == rmem->nr_pages - 2 &&
3593 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3594 extra_bits |= PTU_PTE_NEXT_TO_LAST;
3595 else if (i == rmem->nr_pages - 1 &&
3596 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3597 extra_bits |= PTU_PTE_LAST;
3598 rmem->pg_tbl[i] =
3599 cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3600 }
3601 }
3602
3603 if (rmem->vmem_size) {
3604 *rmem->vmem = vzalloc(rmem->vmem_size);
3605 if (!(*rmem->vmem))
3606 return -ENOMEM;
3607 }
3608 return 0;
3609 }
3610
3611 static void bnxt_free_tpa_info(struct bnxt *bp)
3612 {
3613 int i, j;
3614
3615 for (i = 0; i < bp->rx_nr_rings; i++) {
3616 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3617
3618 kfree(rxr->rx_tpa_idx_map);
3619 rxr->rx_tpa_idx_map = NULL;
3620 if (rxr->rx_tpa) {
3621 for (j = 0; j < bp->max_tpa; j++) {
3622 kfree(rxr->rx_tpa[j].agg_arr);
3623 rxr->rx_tpa[j].agg_arr = NULL;
3624 }
3625 }
3626 kfree(rxr->rx_tpa);
3627 rxr->rx_tpa = NULL;
3628 }
3629 }
3630
3631 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3632 {
3633 int i, j;
3634
3635 bp->max_tpa = MAX_TPA;
3636 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
3637 if (!bp->max_tpa_v2)
3638 return 0;
3639 bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3640 }
3641
3642 for (i = 0; i < bp->rx_nr_rings; i++) {
3643 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3644 struct rx_agg_cmp *agg;
3645
3646 rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3647 GFP_KERNEL);
3648 if (!rxr->rx_tpa)
3649 return -ENOMEM;
3650
3651 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
3652 continue;
3653 for (j = 0; j < bp->max_tpa; j++) {
3654 agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3655 if (!agg)
3656 return -ENOMEM;
3657 rxr->rx_tpa[j].agg_arr = agg;
3658 }
3659 rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3660 GFP_KERNEL);
3661 if (!rxr->rx_tpa_idx_map)
3662 return -ENOMEM;
3663 }
3664 return 0;
3665 }
3666
3667 static void bnxt_free_rx_rings(struct bnxt *bp)
3668 {
3669 int i;
3670
3671 if (!bp->rx_ring)
3672 return;
3673
3674 bnxt_free_tpa_info(bp);
3675 for (i = 0; i < bp->rx_nr_rings; i++) {
3676 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3677 struct bnxt_ring_struct *ring;
3678
3679 if (rxr->xdp_prog)
3680 bpf_prog_put(rxr->xdp_prog);
3681
3682 if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3683 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3684
3685 page_pool_destroy(rxr->page_pool);
3686 if (rxr->page_pool != rxr->head_pool)
3687 page_pool_destroy(rxr->head_pool);
3688 rxr->page_pool = rxr->head_pool = NULL;
3689
3690 kfree(rxr->rx_agg_bmap);
3691 rxr->rx_agg_bmap = NULL;
3692
3693 ring = &rxr->rx_ring_struct;
3694 bnxt_free_ring(bp, &ring->ring_mem);
3695
3696 ring = &rxr->rx_agg_ring_struct;
3697 bnxt_free_ring(bp, &ring->ring_mem);
3698 }
3699 }
3700
3701 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3702 struct bnxt_rx_ring_info *rxr,
3703 int numa_node)
3704 {
3705 struct page_pool_params pp = { 0 };
3706 struct page_pool *pool;
3707
3708 pp.pool_size = bp->rx_agg_ring_size;
3709 if (BNXT_RX_PAGE_MODE(bp))
3710 pp.pool_size += bp->rx_ring_size;
3711 pp.nid = numa_node;
3712 pp.napi = &rxr->bnapi->napi;
3713 pp.netdev = bp->dev;
3714 pp.dev = &bp->pdev->dev;
3715 pp.dma_dir = bp->rx_dir;
3716 pp.max_len = PAGE_SIZE;
3717 pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3718
3719 pool = page_pool_create(&pp);
3720 if (IS_ERR(pool))
3721 return PTR_ERR(pool);
3722 rxr->page_pool = pool;
3723
3724 if (bnxt_separate_head_pool()) {
3725 pp.pool_size = max(bp->rx_ring_size, 1024);
3726 pool = page_pool_create(&pp);
3727 if (IS_ERR(pool))
3728 goto err_destroy_pp;
3729 }
3730 rxr->head_pool = pool;
3731
3732 return 0;
3733
3734 err_destroy_pp:
3735 page_pool_destroy(rxr->page_pool);
3736 rxr->page_pool = NULL;
3737 return PTR_ERR(pool);
3738 }
3739
3740 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3741 {
3742 int numa_node = dev_to_node(&bp->pdev->dev);
3743 int i, rc = 0, agg_rings = 0, cpu;
3744
3745 if (!bp->rx_ring)
3746 return -ENOMEM;
3747
3748 if (bp->flags & BNXT_FLAG_AGG_RINGS)
3749 agg_rings = 1;
3750
3751 for (i = 0; i < bp->rx_nr_rings; i++) {
3752 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3753 struct bnxt_ring_struct *ring;
3754 int cpu_node;
3755
3756 ring = &rxr->rx_ring_struct;
3757
3758 cpu = cpumask_local_spread(i, numa_node);
3759 cpu_node = cpu_to_node(cpu);
3760 netdev_dbg(bp->dev, "Allocating page pool for rx_ring[%d] on numa_node: %d\n",
3761 i, cpu_node);
3762 rc = bnxt_alloc_rx_page_pool(bp, rxr, cpu_node);
3763 if (rc)
3764 return rc;
3765
3766 rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3767 if (rc < 0)
3768 return rc;
3769
3770 rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3771 MEM_TYPE_PAGE_POOL,
3772 rxr->page_pool);
3773 if (rc) {
3774 xdp_rxq_info_unreg(&rxr->xdp_rxq);
3775 return rc;
3776 }
3777
3778 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3779 if (rc)
3780 return rc;
3781
3782 ring->grp_idx = i;
3783 if (agg_rings) {
3784 u16 mem_size;
3785
3786 ring = &rxr->rx_agg_ring_struct;
3787 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3788 if (rc)
3789 return rc;
3790
3791 ring->grp_idx = i;
3792 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3793 mem_size = rxr->rx_agg_bmap_size / 8;
3794 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3795 if (!rxr->rx_agg_bmap)
3796 return -ENOMEM;
3797 }
3798 }
3799 if (bp->flags & BNXT_FLAG_TPA)
3800 rc = bnxt_alloc_tpa_info(bp);
3801 return rc;
3802 }
3803
3804 static void bnxt_free_tx_rings(struct bnxt *bp)
3805 {
3806 int i;
3807 struct pci_dev *pdev = bp->pdev;
3808
3809 if (!bp->tx_ring)
3810 return;
3811
3812 for (i = 0; i < bp->tx_nr_rings; i++) {
3813 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3814 struct bnxt_ring_struct *ring;
3815
3816 if (txr->tx_push) {
3817 dma_free_coherent(&pdev->dev, bp->tx_push_size,
3818 txr->tx_push, txr->tx_push_mapping);
3819 txr->tx_push = NULL;
3820 }
3821
3822 ring = &txr->tx_ring_struct;
3823
3824 bnxt_free_ring(bp, &ring->ring_mem);
3825 }
3826 }
3827
3828 #define BNXT_TC_TO_RING_BASE(bp, tc) \
3829 ((tc) * (bp)->tx_nr_rings_per_tc)
3830
3831 #define BNXT_RING_TO_TC_OFF(bp, tx) \
3832 ((tx) % (bp)->tx_nr_rings_per_tc)
3833
3834 #define BNXT_RING_TO_TC(bp, tx) \
3835 ((tx) / (bp)->tx_nr_rings_per_tc)
3836
3837 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3838 {
3839 int i, j, rc;
3840 struct pci_dev *pdev = bp->pdev;
3841
3842 bp->tx_push_size = 0;
3843 if (bp->tx_push_thresh) {
3844 int push_size;
3845
3846 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3847 bp->tx_push_thresh);
3848
3849 if (push_size > 256) {
3850 push_size = 0;
3851 bp->tx_push_thresh = 0;
3852 }
3853
3854 bp->tx_push_size = push_size;
3855 }
3856
3857 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3858 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3859 struct bnxt_ring_struct *ring;
3860 u8 qidx;
3861
3862 ring = &txr->tx_ring_struct;
3863
3864 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3865 if (rc)
3866 return rc;
3867
3868 ring->grp_idx = txr->bnapi->index;
3869 if (bp->tx_push_size) {
3870 dma_addr_t mapping;
3871
3872 /* One pre-allocated DMA buffer to backup
3873 * TX push operation
3874 */
3875 txr->tx_push = dma_alloc_coherent(&pdev->dev,
3876 bp->tx_push_size,
3877 &txr->tx_push_mapping,
3878 GFP_KERNEL);
3879
3880 if (!txr->tx_push)
3881 return -ENOMEM;
3882
3883 mapping = txr->tx_push_mapping +
3884 sizeof(struct tx_push_bd);
3885 txr->data_mapping = cpu_to_le64(mapping);
3886 }
3887 qidx = bp->tc_to_qidx[j];
3888 ring->queue_id = bp->q_info[qidx].queue_id;
3889 spin_lock_init(&txr->xdp_tx_lock);
3890 if (i < bp->tx_nr_rings_xdp)
3891 continue;
3892 if (BNXT_RING_TO_TC_OFF(bp, i) == (bp->tx_nr_rings_per_tc - 1))
3893 j++;
3894 }
3895 return 0;
3896 }
3897
3898 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3899 {
3900 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3901
3902 kfree(cpr->cp_desc_ring);
3903 cpr->cp_desc_ring = NULL;
3904 ring->ring_mem.pg_arr = NULL;
3905 kfree(cpr->cp_desc_mapping);
3906 cpr->cp_desc_mapping = NULL;
3907 ring->ring_mem.dma_arr = NULL;
3908 }
3909
3910 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3911 {
3912 cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3913 if (!cpr->cp_desc_ring)
3914 return -ENOMEM;
3915 cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3916 GFP_KERNEL);
3917 if (!cpr->cp_desc_mapping)
3918 return -ENOMEM;
3919 return 0;
3920 }
3921
3922 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3923 {
3924 int i;
3925
3926 if (!bp->bnapi)
3927 return;
3928 for (i = 0; i < bp->cp_nr_rings; i++) {
3929 struct bnxt_napi *bnapi = bp->bnapi[i];
3930
3931 if (!bnapi)
3932 continue;
3933 bnxt_free_cp_arrays(&bnapi->cp_ring);
3934 }
3935 }
3936
3937 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3938 {
3939 int i, n = bp->cp_nr_pages;
3940
3941 for (i = 0; i < bp->cp_nr_rings; i++) {
3942 struct bnxt_napi *bnapi = bp->bnapi[i];
3943 int rc;
3944
3945 if (!bnapi)
3946 continue;
3947 rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3948 if (rc)
3949 return rc;
3950 }
3951 return 0;
3952 }
3953
3954 static void bnxt_free_cp_rings(struct bnxt *bp)
3955 {
3956 int i;
3957
3958 if (!bp->bnapi)
3959 return;
3960
3961 for (i = 0; i < bp->cp_nr_rings; i++) {
3962 struct bnxt_napi *bnapi = bp->bnapi[i];
3963 struct bnxt_cp_ring_info *cpr;
3964 struct bnxt_ring_struct *ring;
3965 int j;
3966
3967 if (!bnapi)
3968 continue;
3969
3970 cpr = &bnapi->cp_ring;
3971 ring = &cpr->cp_ring_struct;
3972
3973 bnxt_free_ring(bp, &ring->ring_mem);
3974
3975 if (!cpr->cp_ring_arr)
3976 continue;
3977
3978 for (j = 0; j < cpr->cp_ring_count; j++) {
3979 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
3980
3981 ring = &cpr2->cp_ring_struct;
3982 bnxt_free_ring(bp, &ring->ring_mem);
3983 bnxt_free_cp_arrays(cpr2);
3984 }
3985 kfree(cpr->cp_ring_arr);
3986 cpr->cp_ring_arr = NULL;
3987 cpr->cp_ring_count = 0;
3988 }
3989 }
3990
3991 static int bnxt_alloc_cp_sub_ring(struct bnxt *bp,
3992 struct bnxt_cp_ring_info *cpr)
3993 {
3994 struct bnxt_ring_mem_info *rmem;
3995 struct bnxt_ring_struct *ring;
3996 int rc;
3997
3998 rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3999 if (rc) {
4000 bnxt_free_cp_arrays(cpr);
4001 return -ENOMEM;
4002 }
4003 ring = &cpr->cp_ring_struct;
4004 rmem = &ring->ring_mem;
4005 rmem->nr_pages = bp->cp_nr_pages;
4006 rmem->page_size = HW_CMPD_RING_SIZE;
4007 rmem->pg_arr = (void **)cpr->cp_desc_ring;
4008 rmem->dma_arr = cpr->cp_desc_mapping;
4009 rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
4010 rc = bnxt_alloc_ring(bp, rmem);
4011 if (rc) {
4012 bnxt_free_ring(bp, rmem);
4013 bnxt_free_cp_arrays(cpr);
4014 }
4015 return rc;
4016 }
4017
4018 static int bnxt_alloc_cp_rings(struct bnxt *bp)
4019 {
4020 bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
4021 int i, j, rc, ulp_msix;
4022 int tcs = bp->num_tc;
4023
4024 if (!tcs)
4025 tcs = 1;
4026 ulp_msix = bnxt_get_ulp_msix_num(bp);
4027 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
4028 struct bnxt_napi *bnapi = bp->bnapi[i];
4029 struct bnxt_cp_ring_info *cpr, *cpr2;
4030 struct bnxt_ring_struct *ring;
4031 int cp_count = 0, k;
4032 int rx = 0, tx = 0;
4033
4034 if (!bnapi)
4035 continue;
4036
4037 cpr = &bnapi->cp_ring;
4038 cpr->bnapi = bnapi;
4039 ring = &cpr->cp_ring_struct;
4040
4041 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
4042 if (rc)
4043 return rc;
4044
4045 ring->map_idx = ulp_msix + i;
4046
4047 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4048 continue;
4049
4050 if (i < bp->rx_nr_rings) {
4051 cp_count++;
4052 rx = 1;
4053 }
4054 if (i < bp->tx_nr_rings_xdp) {
4055 cp_count++;
4056 tx = 1;
4057 } else if ((sh && i < bp->tx_nr_rings) ||
4058 (!sh && i >= bp->rx_nr_rings)) {
4059 cp_count += tcs;
4060 tx = 1;
4061 }
4062
4063 cpr->cp_ring_arr = kcalloc(cp_count, sizeof(*cpr),
4064 GFP_KERNEL);
4065 if (!cpr->cp_ring_arr)
4066 return -ENOMEM;
4067 cpr->cp_ring_count = cp_count;
4068
4069 for (k = 0; k < cp_count; k++) {
4070 cpr2 = &cpr->cp_ring_arr[k];
4071 rc = bnxt_alloc_cp_sub_ring(bp, cpr2);
4072 if (rc)
4073 return rc;
4074 cpr2->bnapi = bnapi;
4075 cpr2->sw_stats = cpr->sw_stats;
4076 cpr2->cp_idx = k;
4077 if (!k && rx) {
4078 bp->rx_ring[i].rx_cpr = cpr2;
4079 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_RX;
4080 } else {
4081 int n, tc = k - rx;
4082
4083 n = BNXT_TC_TO_RING_BASE(bp, tc) + j;
4084 bp->tx_ring[n].tx_cpr = cpr2;
4085 cpr2->cp_ring_type = BNXT_NQ_HDL_TYPE_TX;
4086 }
4087 }
4088 if (tx)
4089 j++;
4090 }
4091 return 0;
4092 }
4093
4094 static void bnxt_init_rx_ring_struct(struct bnxt *bp,
4095 struct bnxt_rx_ring_info *rxr)
4096 {
4097 struct bnxt_ring_mem_info *rmem;
4098 struct bnxt_ring_struct *ring;
4099
4100 ring = &rxr->rx_ring_struct;
4101 rmem = &ring->ring_mem;
4102 rmem->nr_pages = bp->rx_nr_pages;
4103 rmem->page_size = HW_RXBD_RING_SIZE;
4104 rmem->pg_arr = (void **)rxr->rx_desc_ring;
4105 rmem->dma_arr = rxr->rx_desc_mapping;
4106 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
4107 rmem->vmem = (void **)&rxr->rx_buf_ring;
4108
4109 ring = &rxr->rx_agg_ring_struct;
4110 rmem = &ring->ring_mem;
4111 rmem->nr_pages = bp->rx_agg_nr_pages;
4112 rmem->page_size = HW_RXBD_RING_SIZE;
4113 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
4114 rmem->dma_arr = rxr->rx_agg_desc_mapping;
4115 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
4116 rmem->vmem = (void **)&rxr->rx_agg_ring;
4117 }
4118
4119 static void bnxt_reset_rx_ring_struct(struct bnxt *bp,
4120 struct bnxt_rx_ring_info *rxr)
4121 {
4122 struct bnxt_ring_mem_info *rmem;
4123 struct bnxt_ring_struct *ring;
4124 int i;
4125
4126 rxr->page_pool->p.napi = NULL;
4127 rxr->page_pool = NULL;
4128 memset(&rxr->xdp_rxq, 0, sizeof(struct xdp_rxq_info));
4129
4130 ring = &rxr->rx_ring_struct;
4131 rmem = &ring->ring_mem;
4132 rmem->pg_tbl = NULL;
4133 rmem->pg_tbl_map = 0;
4134 for (i = 0; i < rmem->nr_pages; i++) {
4135 rmem->pg_arr[i] = NULL;
4136 rmem->dma_arr[i] = 0;
4137 }
4138 *rmem->vmem = NULL;
4139
4140 ring = &rxr->rx_agg_ring_struct;
4141 rmem = &ring->ring_mem;
4142 rmem->pg_tbl = NULL;
4143 rmem->pg_tbl_map = 0;
4144 for (i = 0; i < rmem->nr_pages; i++) {
4145 rmem->pg_arr[i] = NULL;
4146 rmem->dma_arr[i] = 0;
4147 }
4148 *rmem->vmem = NULL;
4149 }
4150
4151 static void bnxt_init_ring_struct(struct bnxt *bp)
4152 {
4153 int i, j;
4154
4155 for (i = 0; i < bp->cp_nr_rings; i++) {
4156 struct bnxt_napi *bnapi = bp->bnapi[i];
4157 struct bnxt_ring_mem_info *rmem;
4158 struct bnxt_cp_ring_info *cpr;
4159 struct bnxt_rx_ring_info *rxr;
4160 struct bnxt_tx_ring_info *txr;
4161 struct bnxt_ring_struct *ring;
4162
4163 if (!bnapi)
4164 continue;
4165
4166 cpr = &bnapi->cp_ring;
4167 ring = &cpr->cp_ring_struct;
4168 rmem = &ring->ring_mem;
4169 rmem->nr_pages = bp->cp_nr_pages;
4170 rmem->page_size = HW_CMPD_RING_SIZE;
4171 rmem->pg_arr = (void **)cpr->cp_desc_ring;
4172 rmem->dma_arr = cpr->cp_desc_mapping;
4173 rmem->vmem_size = 0;
4174
4175 rxr = bnapi->rx_ring;
4176 if (!rxr)
4177 goto skip_rx;
4178
4179 ring = &rxr->rx_ring_struct;
4180 rmem = &ring->ring_mem;
4181 rmem->nr_pages = bp->rx_nr_pages;
4182 rmem->page_size = HW_RXBD_RING_SIZE;
4183 rmem->pg_arr = (void **)rxr->rx_desc_ring;
4184 rmem->dma_arr = rxr->rx_desc_mapping;
4185 rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
4186 rmem->vmem = (void **)&rxr->rx_buf_ring;
4187
4188 ring = &rxr->rx_agg_ring_struct;
4189 rmem = &ring->ring_mem;
4190 rmem->nr_pages = bp->rx_agg_nr_pages;
4191 rmem->page_size = HW_RXBD_RING_SIZE;
4192 rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
4193 rmem->dma_arr = rxr->rx_agg_desc_mapping;
4194 rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
4195 rmem->vmem = (void **)&rxr->rx_agg_ring;
4196
4197 skip_rx:
4198 bnxt_for_each_napi_tx(j, bnapi, txr) {
4199 ring = &txr->tx_ring_struct;
4200 rmem = &ring->ring_mem;
4201 rmem->nr_pages = bp->tx_nr_pages;
4202 rmem->page_size = HW_TXBD_RING_SIZE;
4203 rmem->pg_arr = (void **)txr->tx_desc_ring;
4204 rmem->dma_arr = txr->tx_desc_mapping;
4205 rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
4206 rmem->vmem = (void **)&txr->tx_buf_ring;
4207 }
4208 }
4209 }
4210
4211 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
4212 {
4213 int i;
4214 u32 prod;
4215 struct rx_bd **rx_buf_ring;
4216
4217 rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
4218 for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
4219 int j;
4220 struct rx_bd *rxbd;
4221
4222 rxbd = rx_buf_ring[i];
4223 if (!rxbd)
4224 continue;
4225
4226 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
4227 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
4228 rxbd->rx_bd_opaque = prod;
4229 }
4230 }
4231 }
4232
4233 static void bnxt_alloc_one_rx_ring_skb(struct bnxt *bp,
4234 struct bnxt_rx_ring_info *rxr,
4235 int ring_nr)
4236 {
4237 u32 prod;
4238 int i;
4239
4240 prod = rxr->rx_prod;
4241 for (i = 0; i < bp->rx_ring_size; i++) {
4242 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
4243 netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d skbs only\n",
4244 ring_nr, i, bp->rx_ring_size);
4245 break;
4246 }
4247 prod = NEXT_RX(prod);
4248 }
4249 rxr->rx_prod = prod;
4250 }
4251
4252 static void bnxt_alloc_one_rx_ring_page(struct bnxt *bp,
4253 struct bnxt_rx_ring_info *rxr,
4254 int ring_nr)
4255 {
4256 u32 prod;
4257 int i;
4258
4259 prod = rxr->rx_agg_prod;
4260 for (i = 0; i < bp->rx_agg_ring_size; i++) {
4261 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
4262 netdev_warn(bp->dev, "init'ed rx ring %d with %d/%d pages only\n",
4263 ring_nr, i, bp->rx_ring_size);
4264 break;
4265 }
4266 prod = NEXT_RX_AGG(prod);
4267 }
4268 rxr->rx_agg_prod = prod;
4269 }
4270
4271 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
4272 {
4273 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
4274 int i;
4275
4276 bnxt_alloc_one_rx_ring_skb(bp, rxr, ring_nr);
4277
4278 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
4279 return 0;
4280
4281 bnxt_alloc_one_rx_ring_page(bp, rxr, ring_nr);
4282
4283 if (rxr->rx_tpa) {
4284 dma_addr_t mapping;
4285 u8 *data;
4286
4287 for (i = 0; i < bp->max_tpa; i++) {
4288 data = __bnxt_alloc_rx_frag(bp, &mapping, rxr,
4289 GFP_KERNEL);
4290 if (!data)
4291 return -ENOMEM;
4292
4293 rxr->rx_tpa[i].data = data;
4294 rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
4295 rxr->rx_tpa[i].mapping = mapping;
4296 }
4297 }
4298 return 0;
4299 }
4300
4301 static void bnxt_init_one_rx_ring_rxbd(struct bnxt *bp,
4302 struct bnxt_rx_ring_info *rxr)
4303 {
4304 struct bnxt_ring_struct *ring;
4305 u32 type;
4306
4307 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
4308 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
4309
4310 if (NET_IP_ALIGN == 2)
4311 type |= RX_BD_FLAGS_SOP;
4312
4313 ring = &rxr->rx_ring_struct;
4314 bnxt_init_rxbd_pages(ring, type);
4315 ring->fw_ring_id = INVALID_HW_RING_ID;
4316 }
4317
4318 static void bnxt_init_one_rx_agg_ring_rxbd(struct bnxt *bp,
4319 struct bnxt_rx_ring_info *rxr)
4320 {
4321 struct bnxt_ring_struct *ring;
4322 u32 type;
4323
4324 ring = &rxr->rx_agg_ring_struct;
4325 ring->fw_ring_id = INVALID_HW_RING_ID;
4326 if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
4327 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
4328 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
4329
4330 bnxt_init_rxbd_pages(ring, type);
4331 }
4332 }
4333
4334 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
4335 {
4336 struct bnxt_rx_ring_info *rxr;
4337
4338 rxr = &bp->rx_ring[ring_nr];
4339 bnxt_init_one_rx_ring_rxbd(bp, rxr);
4340
4341 netif_queue_set_napi(bp->dev, ring_nr, NETDEV_QUEUE_TYPE_RX,
4342 &rxr->bnapi->napi);
4343
4344 if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
4345 bpf_prog_add(bp->xdp_prog, 1);
4346 rxr->xdp_prog = bp->xdp_prog;
4347 }
4348
4349 bnxt_init_one_rx_agg_ring_rxbd(bp, rxr);
4350
4351 return bnxt_alloc_one_rx_ring(bp, ring_nr);
4352 }
4353
4354 static void bnxt_init_cp_rings(struct bnxt *bp)
4355 {
4356 int i, j;
4357
4358 for (i = 0; i < bp->cp_nr_rings; i++) {
4359 struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
4360 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4361
4362 ring->fw_ring_id = INVALID_HW_RING_ID;
4363 cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4364 cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4365 if (!cpr->cp_ring_arr)
4366 continue;
4367 for (j = 0; j < cpr->cp_ring_count; j++) {
4368 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
4369
4370 ring = &cpr2->cp_ring_struct;
4371 ring->fw_ring_id = INVALID_HW_RING_ID;
4372 cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
4373 cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
4374 }
4375 }
4376 }
4377
4378 static int bnxt_init_rx_rings(struct bnxt *bp)
4379 {
4380 int i, rc = 0;
4381
4382 if (BNXT_RX_PAGE_MODE(bp)) {
4383 bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
4384 bp->rx_dma_offset = XDP_PACKET_HEADROOM;
4385 } else {
4386 bp->rx_offset = BNXT_RX_OFFSET;
4387 bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
4388 }
4389
4390 for (i = 0; i < bp->rx_nr_rings; i++) {
4391 rc = bnxt_init_one_rx_ring(bp, i);
4392 if (rc)
4393 break;
4394 }
4395
4396 return rc;
4397 }
4398
4399 static int bnxt_init_tx_rings(struct bnxt *bp)
4400 {
4401 u16 i;
4402
4403 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
4404 BNXT_MIN_TX_DESC_CNT);
4405
4406 for (i = 0; i < bp->tx_nr_rings; i++) {
4407 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4408 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
4409
4410 ring->fw_ring_id = INVALID_HW_RING_ID;
4411
4412 if (i >= bp->tx_nr_rings_xdp)
4413 netif_queue_set_napi(bp->dev, i - bp->tx_nr_rings_xdp,
4414 NETDEV_QUEUE_TYPE_TX,
4415 &txr->bnapi->napi);
4416 }
4417
4418 return 0;
4419 }
4420
4421 static void bnxt_free_ring_grps(struct bnxt *bp)
4422 {
4423 kfree(bp->grp_info);
4424 bp->grp_info = NULL;
4425 }
4426
4427 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
4428 {
4429 int i;
4430
4431 if (irq_re_init) {
4432 bp->grp_info = kcalloc(bp->cp_nr_rings,
4433 sizeof(struct bnxt_ring_grp_info),
4434 GFP_KERNEL);
4435 if (!bp->grp_info)
4436 return -ENOMEM;
4437 }
4438 for (i = 0; i < bp->cp_nr_rings; i++) {
4439 if (irq_re_init)
4440 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
4441 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
4442 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
4443 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
4444 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
4445 }
4446 return 0;
4447 }
4448
4449 static void bnxt_free_vnics(struct bnxt *bp)
4450 {
4451 kfree(bp->vnic_info);
4452 bp->vnic_info = NULL;
4453 bp->nr_vnics = 0;
4454 }
4455
4456 static int bnxt_alloc_vnics(struct bnxt *bp)
4457 {
4458 int num_vnics = 1;
4459
4460 #ifdef CONFIG_RFS_ACCEL
4461 if (bp->flags & BNXT_FLAG_RFS) {
4462 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
4463 num_vnics++;
4464 else if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4465 num_vnics += bp->rx_nr_rings;
4466 }
4467 #endif
4468
4469 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4470 num_vnics++;
4471
4472 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
4473 GFP_KERNEL);
4474 if (!bp->vnic_info)
4475 return -ENOMEM;
4476
4477 bp->nr_vnics = num_vnics;
4478 return 0;
4479 }
4480
4481 static void bnxt_init_vnics(struct bnxt *bp)
4482 {
4483 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT];
4484 int i;
4485
4486 for (i = 0; i < bp->nr_vnics; i++) {
4487 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4488 int j;
4489
4490 vnic->fw_vnic_id = INVALID_HW_RING_ID;
4491 vnic->vnic_id = i;
4492 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
4493 vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
4494
4495 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
4496
4497 if (bp->vnic_info[i].rss_hash_key) {
4498 if (i == BNXT_VNIC_DEFAULT) {
4499 u8 *key = (void *)vnic->rss_hash_key;
4500 int k;
4501
4502 if (!bp->rss_hash_key_valid &&
4503 !bp->rss_hash_key_updated) {
4504 get_random_bytes(bp->rss_hash_key,
4505 HW_HASH_KEY_SIZE);
4506 bp->rss_hash_key_updated = true;
4507 }
4508
4509 memcpy(vnic->rss_hash_key, bp->rss_hash_key,
4510 HW_HASH_KEY_SIZE);
4511
4512 if (!bp->rss_hash_key_updated)
4513 continue;
4514
4515 bp->rss_hash_key_updated = false;
4516 bp->rss_hash_key_valid = true;
4517
4518 bp->toeplitz_prefix = 0;
4519 for (k = 0; k < 8; k++) {
4520 bp->toeplitz_prefix <<= 8;
4521 bp->toeplitz_prefix |= key[k];
4522 }
4523 } else {
4524 memcpy(vnic->rss_hash_key, vnic0->rss_hash_key,
4525 HW_HASH_KEY_SIZE);
4526 }
4527 }
4528 }
4529 }
4530
4531 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
4532 {
4533 int pages;
4534
4535 pages = ring_size / desc_per_pg;
4536
4537 if (!pages)
4538 return 1;
4539
4540 pages++;
4541
4542 while (pages & (pages - 1))
4543 pages++;
4544
4545 return pages;
4546 }
4547
4548 void bnxt_set_tpa_flags(struct bnxt *bp)
4549 {
4550 bp->flags &= ~BNXT_FLAG_TPA;
4551 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
4552 return;
4553 if (bp->dev->features & NETIF_F_LRO)
4554 bp->flags |= BNXT_FLAG_LRO;
4555 else if (bp->dev->features & NETIF_F_GRO_HW)
4556 bp->flags |= BNXT_FLAG_GRO;
4557 }
4558
4559 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
4560 * be set on entry.
4561 */
4562 void bnxt_set_ring_params(struct bnxt *bp)
4563 {
4564 u32 ring_size, rx_size, rx_space, max_rx_cmpl;
4565 u32 agg_factor = 0, agg_ring_size = 0;
4566
4567 /* 8 for CRC and VLAN */
4568 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
4569
4570 rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
4571 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4572
4573 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
4574 ring_size = bp->rx_ring_size;
4575 bp->rx_agg_ring_size = 0;
4576 bp->rx_agg_nr_pages = 0;
4577
4578 if (bp->flags & BNXT_FLAG_TPA)
4579 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
4580
4581 bp->flags &= ~BNXT_FLAG_JUMBO;
4582 if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
4583 u32 jumbo_factor;
4584
4585 bp->flags |= BNXT_FLAG_JUMBO;
4586 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
4587 if (jumbo_factor > agg_factor)
4588 agg_factor = jumbo_factor;
4589 }
4590 if (agg_factor) {
4591 if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
4592 ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
4593 netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
4594 bp->rx_ring_size, ring_size);
4595 bp->rx_ring_size = ring_size;
4596 }
4597 agg_ring_size = ring_size * agg_factor;
4598
4599 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
4600 RX_DESC_CNT);
4601 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
4602 u32 tmp = agg_ring_size;
4603
4604 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
4605 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
4606 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
4607 tmp, agg_ring_size);
4608 }
4609 bp->rx_agg_ring_size = agg_ring_size;
4610 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
4611
4612 if (BNXT_RX_PAGE_MODE(bp)) {
4613 rx_space = PAGE_SIZE;
4614 rx_size = PAGE_SIZE -
4615 ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
4616 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4617 } else {
4618 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
4619 rx_space = rx_size + NET_SKB_PAD +
4620 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
4621 }
4622 }
4623
4624 bp->rx_buf_use_size = rx_size;
4625 bp->rx_buf_size = rx_space;
4626
4627 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
4628 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
4629
4630 ring_size = bp->tx_ring_size;
4631 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
4632 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
4633
4634 max_rx_cmpl = bp->rx_ring_size;
4635 /* MAX TPA needs to be added because TPA_START completions are
4636 * immediately recycled, so the TPA completions are not bound by
4637 * the RX ring size.
4638 */
4639 if (bp->flags & BNXT_FLAG_TPA)
4640 max_rx_cmpl += bp->max_tpa;
4641 /* RX and TPA completions are 32-byte, all others are 16-byte */
4642 ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
4643 bp->cp_ring_size = ring_size;
4644
4645 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
4646 if (bp->cp_nr_pages > MAX_CP_PAGES) {
4647 bp->cp_nr_pages = MAX_CP_PAGES;
4648 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
4649 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
4650 ring_size, bp->cp_ring_size);
4651 }
4652 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
4653 bp->cp_ring_mask = bp->cp_bit - 1;
4654 }
4655
4656 /* Changing allocation mode of RX rings.
4657 * TODO: Update when extending xdp_rxq_info to support allocation modes.
4658 */
4659 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
4660 {
4661 struct net_device *dev = bp->dev;
4662
4663 if (page_mode) {
4664 bp->flags &= ~(BNXT_FLAG_AGG_RINGS | BNXT_FLAG_NO_AGG_RINGS);
4665 bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
4666
4667 if (bp->xdp_prog->aux->xdp_has_frags)
4668 dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
4669 else
4670 dev->max_mtu =
4671 min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
4672 if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
4673 bp->flags |= BNXT_FLAG_JUMBO;
4674 bp->rx_skb_func = bnxt_rx_multi_page_skb;
4675 } else {
4676 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4677 bp->rx_skb_func = bnxt_rx_page_skb;
4678 }
4679 bp->rx_dir = DMA_BIDIRECTIONAL;
4680 /* Disable LRO or GRO_HW */
4681 netdev_update_features(dev);
4682 } else {
4683 dev->max_mtu = bp->max_mtu;
4684 bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4685 bp->rx_dir = DMA_FROM_DEVICE;
4686 bp->rx_skb_func = bnxt_rx_skb;
4687 }
4688 return 0;
4689 }
4690
4691 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4692 {
4693 int i;
4694 struct bnxt_vnic_info *vnic;
4695 struct pci_dev *pdev = bp->pdev;
4696
4697 if (!bp->vnic_info)
4698 return;
4699
4700 for (i = 0; i < bp->nr_vnics; i++) {
4701 vnic = &bp->vnic_info[i];
4702
4703 kfree(vnic->fw_grp_ids);
4704 vnic->fw_grp_ids = NULL;
4705
4706 kfree(vnic->uc_list);
4707 vnic->uc_list = NULL;
4708
4709 if (vnic->mc_list) {
4710 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4711 vnic->mc_list, vnic->mc_list_mapping);
4712 vnic->mc_list = NULL;
4713 }
4714
4715 if (vnic->rss_table) {
4716 dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4717 vnic->rss_table,
4718 vnic->rss_table_dma_addr);
4719 vnic->rss_table = NULL;
4720 }
4721
4722 vnic->rss_hash_key = NULL;
4723 vnic->flags = 0;
4724 }
4725 }
4726
4727 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4728 {
4729 int i, rc = 0, size;
4730 struct bnxt_vnic_info *vnic;
4731 struct pci_dev *pdev = bp->pdev;
4732 int max_rings;
4733
4734 for (i = 0; i < bp->nr_vnics; i++) {
4735 vnic = &bp->vnic_info[i];
4736
4737 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4738 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4739
4740 if (mem_size > 0) {
4741 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4742 if (!vnic->uc_list) {
4743 rc = -ENOMEM;
4744 goto out;
4745 }
4746 }
4747 }
4748
4749 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4750 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4751 vnic->mc_list =
4752 dma_alloc_coherent(&pdev->dev,
4753 vnic->mc_list_size,
4754 &vnic->mc_list_mapping,
4755 GFP_KERNEL);
4756 if (!vnic->mc_list) {
4757 rc = -ENOMEM;
4758 goto out;
4759 }
4760 }
4761
4762 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4763 goto vnic_skip_grps;
4764
4765 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4766 max_rings = bp->rx_nr_rings;
4767 else
4768 max_rings = 1;
4769
4770 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4771 if (!vnic->fw_grp_ids) {
4772 rc = -ENOMEM;
4773 goto out;
4774 }
4775 vnic_skip_grps:
4776 if ((bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) &&
4777 !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4778 continue;
4779
4780 /* Allocate rss table and hash key */
4781 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4782 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4783 size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4784
4785 vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4786 vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4787 vnic->rss_table_size,
4788 &vnic->rss_table_dma_addr,
4789 GFP_KERNEL);
4790 if (!vnic->rss_table) {
4791 rc = -ENOMEM;
4792 goto out;
4793 }
4794
4795 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4796 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4797 }
4798 return 0;
4799
4800 out:
4801 return rc;
4802 }
4803
4804 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4805 {
4806 struct bnxt_hwrm_wait_token *token;
4807
4808 dma_pool_destroy(bp->hwrm_dma_pool);
4809 bp->hwrm_dma_pool = NULL;
4810
4811 rcu_read_lock();
4812 hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4813 WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4814 rcu_read_unlock();
4815 }
4816
4817 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4818 {
4819 bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4820 BNXT_HWRM_DMA_SIZE,
4821 BNXT_HWRM_DMA_ALIGN, 0);
4822 if (!bp->hwrm_dma_pool)
4823 return -ENOMEM;
4824
4825 INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4826
4827 return 0;
4828 }
4829
4830 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4831 {
4832 kfree(stats->hw_masks);
4833 stats->hw_masks = NULL;
4834 kfree(stats->sw_stats);
4835 stats->sw_stats = NULL;
4836 if (stats->hw_stats) {
4837 dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4838 stats->hw_stats_map);
4839 stats->hw_stats = NULL;
4840 }
4841 }
4842
4843 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4844 bool alloc_masks)
4845 {
4846 stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4847 &stats->hw_stats_map, GFP_KERNEL);
4848 if (!stats->hw_stats)
4849 return -ENOMEM;
4850
4851 stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4852 if (!stats->sw_stats)
4853 goto stats_mem_err;
4854
4855 if (alloc_masks) {
4856 stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4857 if (!stats->hw_masks)
4858 goto stats_mem_err;
4859 }
4860 return 0;
4861
4862 stats_mem_err:
4863 bnxt_free_stats_mem(bp, stats);
4864 return -ENOMEM;
4865 }
4866
4867 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4868 {
4869 int i;
4870
4871 for (i = 0; i < count; i++)
4872 mask_arr[i] = mask;
4873 }
4874
4875 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4876 {
4877 int i;
4878
4879 for (i = 0; i < count; i++)
4880 mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4881 }
4882
4883 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4884 struct bnxt_stats_mem *stats)
4885 {
4886 struct hwrm_func_qstats_ext_output *resp;
4887 struct hwrm_func_qstats_ext_input *req;
4888 __le64 *hw_masks;
4889 int rc;
4890
4891 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4892 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
4893 return -EOPNOTSUPP;
4894
4895 rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4896 if (rc)
4897 return rc;
4898
4899 req->fid = cpu_to_le16(0xffff);
4900 req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4901
4902 resp = hwrm_req_hold(bp, req);
4903 rc = hwrm_req_send(bp, req);
4904 if (!rc) {
4905 hw_masks = &resp->rx_ucast_pkts;
4906 bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4907 }
4908 hwrm_req_drop(bp, req);
4909 return rc;
4910 }
4911
4912 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4913 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4914
4915 static void bnxt_init_stats(struct bnxt *bp)
4916 {
4917 struct bnxt_napi *bnapi = bp->bnapi[0];
4918 struct bnxt_cp_ring_info *cpr;
4919 struct bnxt_stats_mem *stats;
4920 __le64 *rx_stats, *tx_stats;
4921 int rc, rx_count, tx_count;
4922 u64 *rx_masks, *tx_masks;
4923 u64 mask;
4924 u8 flags;
4925
4926 cpr = &bnapi->cp_ring;
4927 stats = &cpr->stats;
4928 rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4929 if (rc) {
4930 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
4931 mask = (1ULL << 48) - 1;
4932 else
4933 mask = -1ULL;
4934 bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4935 }
4936 if (bp->flags & BNXT_FLAG_PORT_STATS) {
4937 stats = &bp->port_stats;
4938 rx_stats = stats->hw_stats;
4939 rx_masks = stats->hw_masks;
4940 rx_count = sizeof(struct rx_port_stats) / 8;
4941 tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4942 tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4943 tx_count = sizeof(struct tx_port_stats) / 8;
4944
4945 flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4946 rc = bnxt_hwrm_port_qstats(bp, flags);
4947 if (rc) {
4948 mask = (1ULL << 40) - 1;
4949
4950 bnxt_fill_masks(rx_masks, mask, rx_count);
4951 bnxt_fill_masks(tx_masks, mask, tx_count);
4952 } else {
4953 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4954 bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4955 bnxt_hwrm_port_qstats(bp, 0);
4956 }
4957 }
4958 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4959 stats = &bp->rx_port_stats_ext;
4960 rx_stats = stats->hw_stats;
4961 rx_masks = stats->hw_masks;
4962 rx_count = sizeof(struct rx_port_stats_ext) / 8;
4963 stats = &bp->tx_port_stats_ext;
4964 tx_stats = stats->hw_stats;
4965 tx_masks = stats->hw_masks;
4966 tx_count = sizeof(struct tx_port_stats_ext) / 8;
4967
4968 flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4969 rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4970 if (rc) {
4971 mask = (1ULL << 40) - 1;
4972
4973 bnxt_fill_masks(rx_masks, mask, rx_count);
4974 if (tx_stats)
4975 bnxt_fill_masks(tx_masks, mask, tx_count);
4976 } else {
4977 bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4978 if (tx_stats)
4979 bnxt_copy_hw_masks(tx_masks, tx_stats,
4980 tx_count);
4981 bnxt_hwrm_port_qstats_ext(bp, 0);
4982 }
4983 }
4984 }
4985
4986 static void bnxt_free_port_stats(struct bnxt *bp)
4987 {
4988 bp->flags &= ~BNXT_FLAG_PORT_STATS;
4989 bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4990
4991 bnxt_free_stats_mem(bp, &bp->port_stats);
4992 bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4993 bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4994 }
4995
4996 static void bnxt_free_ring_stats(struct bnxt *bp)
4997 {
4998 int i;
4999
5000 if (!bp->bnapi)
5001 return;
5002
5003 for (i = 0; i < bp->cp_nr_rings; i++) {
5004 struct bnxt_napi *bnapi = bp->bnapi[i];
5005 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5006
5007 bnxt_free_stats_mem(bp, &cpr->stats);
5008
5009 kfree(cpr->sw_stats);
5010 cpr->sw_stats = NULL;
5011 }
5012 }
5013
5014 static int bnxt_alloc_stats(struct bnxt *bp)
5015 {
5016 u32 size, i;
5017 int rc;
5018
5019 size = bp->hw_ring_stats_size;
5020
5021 for (i = 0; i < bp->cp_nr_rings; i++) {
5022 struct bnxt_napi *bnapi = bp->bnapi[i];
5023 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5024
5025 cpr->sw_stats = kzalloc(sizeof(*cpr->sw_stats), GFP_KERNEL);
5026 if (!cpr->sw_stats)
5027 return -ENOMEM;
5028
5029 cpr->stats.len = size;
5030 rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
5031 if (rc)
5032 return rc;
5033
5034 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
5035 }
5036
5037 if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
5038 return 0;
5039
5040 if (bp->port_stats.hw_stats)
5041 goto alloc_ext_stats;
5042
5043 bp->port_stats.len = BNXT_PORT_STATS_SIZE;
5044 rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
5045 if (rc)
5046 return rc;
5047
5048 bp->flags |= BNXT_FLAG_PORT_STATS;
5049
5050 alloc_ext_stats:
5051 /* Display extended statistics only if FW supports it */
5052 if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
5053 if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
5054 return 0;
5055
5056 if (bp->rx_port_stats_ext.hw_stats)
5057 goto alloc_tx_ext_stats;
5058
5059 bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
5060 rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
5061 /* Extended stats are optional */
5062 if (rc)
5063 return 0;
5064
5065 alloc_tx_ext_stats:
5066 if (bp->tx_port_stats_ext.hw_stats)
5067 return 0;
5068
5069 if (bp->hwrm_spec_code >= 0x10902 ||
5070 (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
5071 bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
5072 rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
5073 /* Extended stats are optional */
5074 if (rc)
5075 return 0;
5076 }
5077 bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
5078 return 0;
5079 }
5080
5081 static void bnxt_clear_ring_indices(struct bnxt *bp)
5082 {
5083 int i, j;
5084
5085 if (!bp->bnapi)
5086 return;
5087
5088 for (i = 0; i < bp->cp_nr_rings; i++) {
5089 struct bnxt_napi *bnapi = bp->bnapi[i];
5090 struct bnxt_cp_ring_info *cpr;
5091 struct bnxt_rx_ring_info *rxr;
5092 struct bnxt_tx_ring_info *txr;
5093
5094 if (!bnapi)
5095 continue;
5096
5097 cpr = &bnapi->cp_ring;
5098 cpr->cp_raw_cons = 0;
5099
5100 bnxt_for_each_napi_tx(j, bnapi, txr) {
5101 txr->tx_prod = 0;
5102 txr->tx_cons = 0;
5103 txr->tx_hw_cons = 0;
5104 }
5105
5106 rxr = bnapi->rx_ring;
5107 if (rxr) {
5108 rxr->rx_prod = 0;
5109 rxr->rx_agg_prod = 0;
5110 rxr->rx_sw_agg_prod = 0;
5111 rxr->rx_next_cons = 0;
5112 }
5113 bnapi->events = 0;
5114 }
5115 }
5116
5117 void bnxt_insert_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
5118 {
5119 u8 type = fltr->type, flags = fltr->flags;
5120
5121 INIT_LIST_HEAD(&fltr->list);
5122 if ((type == BNXT_FLTR_TYPE_L2 && flags & BNXT_ACT_RING_DST) ||
5123 (type == BNXT_FLTR_TYPE_NTUPLE && flags & BNXT_ACT_NO_AGING))
5124 list_add_tail(&fltr->list, &bp->usr_fltr_list);
5125 }
5126
5127 void bnxt_del_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
5128 {
5129 if (!list_empty(&fltr->list))
5130 list_del_init(&fltr->list);
5131 }
5132
5133 static void bnxt_clear_usr_fltrs(struct bnxt *bp, bool all)
5134 {
5135 struct bnxt_filter_base *usr_fltr, *tmp;
5136
5137 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) {
5138 if (!all && usr_fltr->type == BNXT_FLTR_TYPE_L2)
5139 continue;
5140 bnxt_del_one_usr_fltr(bp, usr_fltr);
5141 }
5142 }
5143
5144 static void bnxt_del_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
5145 {
5146 hlist_del(&fltr->hash);
5147 bnxt_del_one_usr_fltr(bp, fltr);
5148 if (fltr->flags) {
5149 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
5150 bp->ntp_fltr_count--;
5151 }
5152 kfree(fltr);
5153 }
5154
5155 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool all)
5156 {
5157 int i;
5158
5159 /* Under rtnl_lock and all our NAPIs have been disabled. It's
5160 * safe to delete the hash table.
5161 */
5162 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
5163 struct hlist_head *head;
5164 struct hlist_node *tmp;
5165 struct bnxt_ntuple_filter *fltr;
5166
5167 head = &bp->ntp_fltr_hash_tbl[i];
5168 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
5169 bnxt_del_l2_filter(bp, fltr->l2_fltr);
5170 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) ||
5171 !list_empty(&fltr->base.list)))
5172 continue;
5173 bnxt_del_fltr(bp, &fltr->base);
5174 }
5175 }
5176 if (!all)
5177 return;
5178
5179 bitmap_free(bp->ntp_fltr_bmap);
5180 bp->ntp_fltr_bmap = NULL;
5181 bp->ntp_fltr_count = 0;
5182 }
5183
5184 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
5185 {
5186 int i, rc = 0;
5187
5188 if (!(bp->flags & BNXT_FLAG_RFS) || bp->ntp_fltr_bmap)
5189 return 0;
5190
5191 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
5192 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
5193
5194 bp->ntp_fltr_count = 0;
5195 bp->ntp_fltr_bmap = bitmap_zalloc(bp->max_fltr, GFP_KERNEL);
5196
5197 if (!bp->ntp_fltr_bmap)
5198 rc = -ENOMEM;
5199
5200 return rc;
5201 }
5202
5203 static void bnxt_free_l2_filters(struct bnxt *bp, bool all)
5204 {
5205 int i;
5206
5207 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++) {
5208 struct hlist_head *head;
5209 struct hlist_node *tmp;
5210 struct bnxt_l2_filter *fltr;
5211
5212 head = &bp->l2_fltr_hash_tbl[i];
5213 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
5214 if (!all && ((fltr->base.flags & BNXT_ACT_FUNC_DST) ||
5215 !list_empty(&fltr->base.list)))
5216 continue;
5217 bnxt_del_fltr(bp, &fltr->base);
5218 }
5219 }
5220 }
5221
5222 static void bnxt_init_l2_fltr_tbl(struct bnxt *bp)
5223 {
5224 int i;
5225
5226 for (i = 0; i < BNXT_L2_FLTR_HASH_SIZE; i++)
5227 INIT_HLIST_HEAD(&bp->l2_fltr_hash_tbl[i]);
5228 get_random_bytes(&bp->hash_seed, sizeof(bp->hash_seed));
5229 }
5230
5231 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
5232 {
5233 bnxt_free_vnic_attributes(bp);
5234 bnxt_free_tx_rings(bp);
5235 bnxt_free_rx_rings(bp);
5236 bnxt_free_cp_rings(bp);
5237 bnxt_free_all_cp_arrays(bp);
5238 bnxt_free_ntp_fltrs(bp, false);
5239 bnxt_free_l2_filters(bp, false);
5240 if (irq_re_init) {
5241 bnxt_free_ring_stats(bp);
5242 if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
5243 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
5244 bnxt_free_port_stats(bp);
5245 bnxt_free_ring_grps(bp);
5246 bnxt_free_vnics(bp);
5247 kfree(bp->tx_ring_map);
5248 bp->tx_ring_map = NULL;
5249 kfree(bp->tx_ring);
5250 bp->tx_ring = NULL;
5251 kfree(bp->rx_ring);
5252 bp->rx_ring = NULL;
5253 kfree(bp->bnapi);
5254 bp->bnapi = NULL;
5255 } else {
5256 bnxt_clear_ring_indices(bp);
5257 }
5258 }
5259
5260 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
5261 {
5262 int i, j, rc, size, arr_size;
5263 void *bnapi;
5264
5265 if (irq_re_init) {
5266 /* Allocate bnapi mem pointer array and mem block for
5267 * all queues
5268 */
5269 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
5270 bp->cp_nr_rings);
5271 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
5272 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
5273 if (!bnapi)
5274 return -ENOMEM;
5275
5276 bp->bnapi = bnapi;
5277 bnapi += arr_size;
5278 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
5279 bp->bnapi[i] = bnapi;
5280 bp->bnapi[i]->index = i;
5281 bp->bnapi[i]->bp = bp;
5282 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5283 struct bnxt_cp_ring_info *cpr =
5284 &bp->bnapi[i]->cp_ring;
5285
5286 cpr->cp_ring_struct.ring_mem.flags =
5287 BNXT_RMEM_RING_PTE_FLAG;
5288 }
5289 }
5290
5291 bp->rx_ring = kcalloc(bp->rx_nr_rings,
5292 sizeof(struct bnxt_rx_ring_info),
5293 GFP_KERNEL);
5294 if (!bp->rx_ring)
5295 return -ENOMEM;
5296
5297 for (i = 0; i < bp->rx_nr_rings; i++) {
5298 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5299
5300 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
5301 rxr->rx_ring_struct.ring_mem.flags =
5302 BNXT_RMEM_RING_PTE_FLAG;
5303 rxr->rx_agg_ring_struct.ring_mem.flags =
5304 BNXT_RMEM_RING_PTE_FLAG;
5305 } else {
5306 rxr->rx_cpr = &bp->bnapi[i]->cp_ring;
5307 }
5308 rxr->bnapi = bp->bnapi[i];
5309 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
5310 }
5311
5312 bp->tx_ring = kcalloc(bp->tx_nr_rings,
5313 sizeof(struct bnxt_tx_ring_info),
5314 GFP_KERNEL);
5315 if (!bp->tx_ring)
5316 return -ENOMEM;
5317
5318 bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
5319 GFP_KERNEL);
5320
5321 if (!bp->tx_ring_map)
5322 return -ENOMEM;
5323
5324 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5325 j = 0;
5326 else
5327 j = bp->rx_nr_rings;
5328
5329 for (i = 0; i < bp->tx_nr_rings; i++) {
5330 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5331 struct bnxt_napi *bnapi2;
5332
5333 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
5334 txr->tx_ring_struct.ring_mem.flags =
5335 BNXT_RMEM_RING_PTE_FLAG;
5336 bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
5337 if (i >= bp->tx_nr_rings_xdp) {
5338 int k = j + BNXT_RING_TO_TC_OFF(bp, i);
5339
5340 bnapi2 = bp->bnapi[k];
5341 txr->txq_index = i - bp->tx_nr_rings_xdp;
5342 txr->tx_napi_idx =
5343 BNXT_RING_TO_TC(bp, txr->txq_index);
5344 bnapi2->tx_ring[txr->tx_napi_idx] = txr;
5345 bnapi2->tx_int = bnxt_tx_int;
5346 } else {
5347 bnapi2 = bp->bnapi[j];
5348 bnapi2->flags |= BNXT_NAPI_FLAG_XDP;
5349 bnapi2->tx_ring[0] = txr;
5350 bnapi2->tx_int = bnxt_tx_int_xdp;
5351 j++;
5352 }
5353 txr->bnapi = bnapi2;
5354 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
5355 txr->tx_cpr = &bnapi2->cp_ring;
5356 }
5357
5358 rc = bnxt_alloc_stats(bp);
5359 if (rc)
5360 goto alloc_mem_err;
5361 bnxt_init_stats(bp);
5362
5363 rc = bnxt_alloc_ntp_fltrs(bp);
5364 if (rc)
5365 goto alloc_mem_err;
5366
5367 rc = bnxt_alloc_vnics(bp);
5368 if (rc)
5369 goto alloc_mem_err;
5370 }
5371
5372 rc = bnxt_alloc_all_cp_arrays(bp);
5373 if (rc)
5374 goto alloc_mem_err;
5375
5376 bnxt_init_ring_struct(bp);
5377
5378 rc = bnxt_alloc_rx_rings(bp);
5379 if (rc)
5380 goto alloc_mem_err;
5381
5382 rc = bnxt_alloc_tx_rings(bp);
5383 if (rc)
5384 goto alloc_mem_err;
5385
5386 rc = bnxt_alloc_cp_rings(bp);
5387 if (rc)
5388 goto alloc_mem_err;
5389
5390 bp->vnic_info[BNXT_VNIC_DEFAULT].flags |= BNXT_VNIC_RSS_FLAG |
5391 BNXT_VNIC_MCAST_FLAG |
5392 BNXT_VNIC_UCAST_FLAG;
5393 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp) && (bp->flags & BNXT_FLAG_RFS))
5394 bp->vnic_info[BNXT_VNIC_NTUPLE].flags |=
5395 BNXT_VNIC_RSS_FLAG | BNXT_VNIC_NTUPLE_FLAG;
5396
5397 rc = bnxt_alloc_vnic_attributes(bp);
5398 if (rc)
5399 goto alloc_mem_err;
5400 return 0;
5401
5402 alloc_mem_err:
5403 bnxt_free_mem(bp, true);
5404 return rc;
5405 }
5406
5407 static void bnxt_disable_int(struct bnxt *bp)
5408 {
5409 int i;
5410
5411 if (!bp->bnapi)
5412 return;
5413
5414 for (i = 0; i < bp->cp_nr_rings; i++) {
5415 struct bnxt_napi *bnapi = bp->bnapi[i];
5416 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5417 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5418
5419 if (ring->fw_ring_id != INVALID_HW_RING_ID)
5420 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5421 }
5422 }
5423
5424 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
5425 {
5426 struct bnxt_napi *bnapi = bp->bnapi[n];
5427 struct bnxt_cp_ring_info *cpr;
5428
5429 cpr = &bnapi->cp_ring;
5430 return cpr->cp_ring_struct.map_idx;
5431 }
5432
5433 static void bnxt_disable_int_sync(struct bnxt *bp)
5434 {
5435 int i;
5436
5437 if (!bp->irq_tbl)
5438 return;
5439
5440 atomic_inc(&bp->intr_sem);
5441
5442 bnxt_disable_int(bp);
5443 for (i = 0; i < bp->cp_nr_rings; i++) {
5444 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
5445
5446 synchronize_irq(bp->irq_tbl[map_idx].vector);
5447 }
5448 }
5449
5450 static void bnxt_enable_int(struct bnxt *bp)
5451 {
5452 int i;
5453
5454 atomic_set(&bp->intr_sem, 0);
5455 for (i = 0; i < bp->cp_nr_rings; i++) {
5456 struct bnxt_napi *bnapi = bp->bnapi[i];
5457 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5458
5459 bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
5460 }
5461 }
5462
5463 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
5464 bool async_only)
5465 {
5466 DECLARE_BITMAP(async_events_bmap, 256);
5467 u32 *events = (u32 *)async_events_bmap;
5468 struct hwrm_func_drv_rgtr_output *resp;
5469 struct hwrm_func_drv_rgtr_input *req;
5470 u32 flags;
5471 int rc, i;
5472
5473 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
5474 if (rc)
5475 return rc;
5476
5477 req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
5478 FUNC_DRV_RGTR_REQ_ENABLES_VER |
5479 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5480
5481 req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
5482 flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
5483 if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
5484 flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
5485 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
5486 flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
5487 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
5488 req->flags = cpu_to_le32(flags);
5489 req->ver_maj_8b = DRV_VER_MAJ;
5490 req->ver_min_8b = DRV_VER_MIN;
5491 req->ver_upd_8b = DRV_VER_UPD;
5492 req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
5493 req->ver_min = cpu_to_le16(DRV_VER_MIN);
5494 req->ver_upd = cpu_to_le16(DRV_VER_UPD);
5495
5496 if (BNXT_PF(bp)) {
5497 u32 data[8];
5498 int i;
5499
5500 memset(data, 0, sizeof(data));
5501 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
5502 u16 cmd = bnxt_vf_req_snif[i];
5503 unsigned int bit, idx;
5504
5505 idx = cmd / 32;
5506 bit = cmd % 32;
5507 data[idx] |= 1 << bit;
5508 }
5509
5510 for (i = 0; i < 8; i++)
5511 req->vf_req_fwd[i] = cpu_to_le32(data[i]);
5512
5513 req->enables |=
5514 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
5515 }
5516
5517 if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
5518 req->flags |= cpu_to_le32(
5519 FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
5520
5521 memset(async_events_bmap, 0, sizeof(async_events_bmap));
5522 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
5523 u16 event_id = bnxt_async_events_arr[i];
5524
5525 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
5526 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5527 continue;
5528 if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
5529 !bp->ptp_cfg)
5530 continue;
5531 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
5532 }
5533 if (bmap && bmap_size) {
5534 for (i = 0; i < bmap_size; i++) {
5535 if (test_bit(i, bmap))
5536 __set_bit(i, async_events_bmap);
5537 }
5538 }
5539 for (i = 0; i < 8; i++)
5540 req->async_event_fwd[i] |= cpu_to_le32(events[i]);
5541
5542 if (async_only)
5543 req->enables =
5544 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
5545
5546 resp = hwrm_req_hold(bp, req);
5547 rc = hwrm_req_send(bp, req);
5548 if (!rc) {
5549 set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
5550 if (resp->flags &
5551 cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
5552 bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
5553 }
5554 hwrm_req_drop(bp, req);
5555 return rc;
5556 }
5557
5558 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
5559 {
5560 struct hwrm_func_drv_unrgtr_input *req;
5561 int rc;
5562
5563 if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
5564 return 0;
5565
5566 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
5567 if (rc)
5568 return rc;
5569 return hwrm_req_send(bp, req);
5570 }
5571
5572 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa);
5573
5574 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
5575 {
5576 struct hwrm_tunnel_dst_port_free_input *req;
5577 int rc;
5578
5579 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
5580 bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
5581 return 0;
5582 if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
5583 bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
5584 return 0;
5585
5586 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
5587 if (rc)
5588 return rc;
5589
5590 req->tunnel_type = tunnel_type;
5591
5592 switch (tunnel_type) {
5593 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
5594 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
5595 bp->vxlan_port = 0;
5596 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
5597 break;
5598 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
5599 req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
5600 bp->nge_port = 0;
5601 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
5602 break;
5603 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE:
5604 req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_gpe_fw_dst_port_id);
5605 bp->vxlan_gpe_port = 0;
5606 bp->vxlan_gpe_fw_dst_port_id = INVALID_HW_RING_ID;
5607 break;
5608 default:
5609 break;
5610 }
5611
5612 rc = hwrm_req_send(bp, req);
5613 if (rc)
5614 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
5615 rc);
5616 if (bp->flags & BNXT_FLAG_TPA)
5617 bnxt_set_tpa(bp, true);
5618 return rc;
5619 }
5620
5621 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
5622 u8 tunnel_type)
5623 {
5624 struct hwrm_tunnel_dst_port_alloc_output *resp;
5625 struct hwrm_tunnel_dst_port_alloc_input *req;
5626 int rc;
5627
5628 rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
5629 if (rc)
5630 return rc;
5631
5632 req->tunnel_type = tunnel_type;
5633 req->tunnel_dst_port_val = port;
5634
5635 resp = hwrm_req_hold(bp, req);
5636 rc = hwrm_req_send(bp, req);
5637 if (rc) {
5638 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
5639 rc);
5640 goto err_out;
5641 }
5642
5643 switch (tunnel_type) {
5644 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
5645 bp->vxlan_port = port;
5646 bp->vxlan_fw_dst_port_id =
5647 le16_to_cpu(resp->tunnel_dst_port_id);
5648 break;
5649 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
5650 bp->nge_port = port;
5651 bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
5652 break;
5653 case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE:
5654 bp->vxlan_gpe_port = port;
5655 bp->vxlan_gpe_fw_dst_port_id =
5656 le16_to_cpu(resp->tunnel_dst_port_id);
5657 break;
5658 default:
5659 break;
5660 }
5661 if (bp->flags & BNXT_FLAG_TPA)
5662 bnxt_set_tpa(bp, true);
5663
5664 err_out:
5665 hwrm_req_drop(bp, req);
5666 return rc;
5667 }
5668
5669 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
5670 {
5671 struct hwrm_cfa_l2_set_rx_mask_input *req;
5672 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5673 int rc;
5674
5675 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
5676 if (rc)
5677 return rc;
5678
5679 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5680 if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
5681 req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
5682 req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
5683 }
5684 req->mask = cpu_to_le32(vnic->rx_mask);
5685 return hwrm_req_send_silent(bp, req);
5686 }
5687
5688 void bnxt_del_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5689 {
5690 if (!atomic_dec_and_test(&fltr->refcnt))
5691 return;
5692 spin_lock_bh(&bp->ntp_fltr_lock);
5693 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
5694 spin_unlock_bh(&bp->ntp_fltr_lock);
5695 return;
5696 }
5697 hlist_del_rcu(&fltr->base.hash);
5698 bnxt_del_one_usr_fltr(bp, &fltr->base);
5699 if (fltr->base.flags) {
5700 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
5701 bp->ntp_fltr_count--;
5702 }
5703 spin_unlock_bh(&bp->ntp_fltr_lock);
5704 kfree_rcu(fltr, base.rcu);
5705 }
5706
5707 static struct bnxt_l2_filter *__bnxt_lookup_l2_filter(struct bnxt *bp,
5708 struct bnxt_l2_key *key,
5709 u32 idx)
5710 {
5711 struct hlist_head *head = &bp->l2_fltr_hash_tbl[idx];
5712 struct bnxt_l2_filter *fltr;
5713
5714 hlist_for_each_entry_rcu(fltr, head, base.hash) {
5715 struct bnxt_l2_key *l2_key = &fltr->l2_key;
5716
5717 if (ether_addr_equal(l2_key->dst_mac_addr, key->dst_mac_addr) &&
5718 l2_key->vlan == key->vlan)
5719 return fltr;
5720 }
5721 return NULL;
5722 }
5723
5724 static struct bnxt_l2_filter *bnxt_lookup_l2_filter(struct bnxt *bp,
5725 struct bnxt_l2_key *key,
5726 u32 idx)
5727 {
5728 struct bnxt_l2_filter *fltr = NULL;
5729
5730 rcu_read_lock();
5731 fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5732 if (fltr)
5733 atomic_inc(&fltr->refcnt);
5734 rcu_read_unlock();
5735 return fltr;
5736 }
5737
5738 #define BNXT_IPV4_4TUPLE(bp, fkeys) \
5739 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \
5740 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4) || \
5741 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \
5742 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4))
5743
5744 #define BNXT_IPV6_4TUPLE(bp, fkeys) \
5745 (((fkeys)->basic.ip_proto == IPPROTO_TCP && \
5746 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6) || \
5747 ((fkeys)->basic.ip_proto == IPPROTO_UDP && \
5748 (bp)->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6))
5749
5750 static u32 bnxt_get_rss_flow_tuple_len(struct bnxt *bp, struct flow_keys *fkeys)
5751 {
5752 if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5753 if (BNXT_IPV4_4TUPLE(bp, fkeys))
5754 return sizeof(fkeys->addrs.v4addrs) +
5755 sizeof(fkeys->ports);
5756
5757 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4)
5758 return sizeof(fkeys->addrs.v4addrs);
5759 }
5760
5761 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
5762 if (BNXT_IPV6_4TUPLE(bp, fkeys))
5763 return sizeof(fkeys->addrs.v6addrs) +
5764 sizeof(fkeys->ports);
5765
5766 if (bp->rss_hash_cfg & VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6)
5767 return sizeof(fkeys->addrs.v6addrs);
5768 }
5769
5770 return 0;
5771 }
5772
5773 static u32 bnxt_toeplitz(struct bnxt *bp, struct flow_keys *fkeys,
5774 const unsigned char *key)
5775 {
5776 u64 prefix = bp->toeplitz_prefix, hash = 0;
5777 struct bnxt_ipv4_tuple tuple4;
5778 struct bnxt_ipv6_tuple tuple6;
5779 int i, j, len = 0;
5780 u8 *four_tuple;
5781
5782 len = bnxt_get_rss_flow_tuple_len(bp, fkeys);
5783 if (!len)
5784 return 0;
5785
5786 if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
5787 tuple4.v4addrs = fkeys->addrs.v4addrs;
5788 tuple4.ports = fkeys->ports;
5789 four_tuple = (unsigned char *)&tuple4;
5790 } else {
5791 tuple6.v6addrs = fkeys->addrs.v6addrs;
5792 tuple6.ports = fkeys->ports;
5793 four_tuple = (unsigned char *)&tuple6;
5794 }
5795
5796 for (i = 0, j = 8; i < len; i++, j++) {
5797 u8 byte = four_tuple[i];
5798 int bit;
5799
5800 for (bit = 0; bit < 8; bit++, prefix <<= 1, byte <<= 1) {
5801 if (byte & 0x80)
5802 hash ^= prefix;
5803 }
5804 prefix |= (j < HW_HASH_KEY_SIZE) ? key[j] : 0;
5805 }
5806
5807 /* The valid part of the hash is in the upper 32 bits. */
5808 return (hash >> 32) & BNXT_NTP_FLTR_HASH_MASK;
5809 }
5810
5811 #ifdef CONFIG_RFS_ACCEL
5812 static struct bnxt_l2_filter *
5813 bnxt_lookup_l2_filter_from_key(struct bnxt *bp, struct bnxt_l2_key *key)
5814 {
5815 struct bnxt_l2_filter *fltr;
5816 u32 idx;
5817
5818 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5819 BNXT_L2_FLTR_HASH_MASK;
5820 fltr = bnxt_lookup_l2_filter(bp, key, idx);
5821 return fltr;
5822 }
5823 #endif
5824
5825 static int bnxt_init_l2_filter(struct bnxt *bp, struct bnxt_l2_filter *fltr,
5826 struct bnxt_l2_key *key, u32 idx)
5827 {
5828 struct hlist_head *head;
5829
5830 ether_addr_copy(fltr->l2_key.dst_mac_addr, key->dst_mac_addr);
5831 fltr->l2_key.vlan = key->vlan;
5832 fltr->base.type = BNXT_FLTR_TYPE_L2;
5833 if (fltr->base.flags) {
5834 int bit_id;
5835
5836 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
5837 bp->max_fltr, 0);
5838 if (bit_id < 0)
5839 return -ENOMEM;
5840 fltr->base.sw_id = (u16)bit_id;
5841 bp->ntp_fltr_count++;
5842 }
5843 head = &bp->l2_fltr_hash_tbl[idx];
5844 hlist_add_head_rcu(&fltr->base.hash, head);
5845 bnxt_insert_usr_fltr(bp, &fltr->base);
5846 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
5847 atomic_set(&fltr->refcnt, 1);
5848 return 0;
5849 }
5850
5851 static struct bnxt_l2_filter *bnxt_alloc_l2_filter(struct bnxt *bp,
5852 struct bnxt_l2_key *key,
5853 gfp_t gfp)
5854 {
5855 struct bnxt_l2_filter *fltr;
5856 u32 idx;
5857 int rc;
5858
5859 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5860 BNXT_L2_FLTR_HASH_MASK;
5861 fltr = bnxt_lookup_l2_filter(bp, key, idx);
5862 if (fltr)
5863 return fltr;
5864
5865 fltr = kzalloc(sizeof(*fltr), gfp);
5866 if (!fltr)
5867 return ERR_PTR(-ENOMEM);
5868 spin_lock_bh(&bp->ntp_fltr_lock);
5869 rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5870 spin_unlock_bh(&bp->ntp_fltr_lock);
5871 if (rc) {
5872 bnxt_del_l2_filter(bp, fltr);
5873 fltr = ERR_PTR(rc);
5874 }
5875 return fltr;
5876 }
5877
5878 struct bnxt_l2_filter *bnxt_alloc_new_l2_filter(struct bnxt *bp,
5879 struct bnxt_l2_key *key,
5880 u16 flags)
5881 {
5882 struct bnxt_l2_filter *fltr;
5883 u32 idx;
5884 int rc;
5885
5886 idx = jhash2(&key->filter_key, BNXT_L2_KEY_SIZE, bp->hash_seed) &
5887 BNXT_L2_FLTR_HASH_MASK;
5888 spin_lock_bh(&bp->ntp_fltr_lock);
5889 fltr = __bnxt_lookup_l2_filter(bp, key, idx);
5890 if (fltr) {
5891 fltr = ERR_PTR(-EEXIST);
5892 goto l2_filter_exit;
5893 }
5894 fltr = kzalloc(sizeof(*fltr), GFP_ATOMIC);
5895 if (!fltr) {
5896 fltr = ERR_PTR(-ENOMEM);
5897 goto l2_filter_exit;
5898 }
5899 fltr->base.flags = flags;
5900 rc = bnxt_init_l2_filter(bp, fltr, key, idx);
5901 if (rc) {
5902 spin_unlock_bh(&bp->ntp_fltr_lock);
5903 bnxt_del_l2_filter(bp, fltr);
5904 return ERR_PTR(rc);
5905 }
5906
5907 l2_filter_exit:
5908 spin_unlock_bh(&bp->ntp_fltr_lock);
5909 return fltr;
5910 }
5911
5912 static u16 bnxt_vf_target_id(struct bnxt_pf_info *pf, u16 vf_idx)
5913 {
5914 #ifdef CONFIG_BNXT_SRIOV
5915 struct bnxt_vf_info *vf = &pf->vf[vf_idx];
5916
5917 return vf->fw_fid;
5918 #else
5919 return INVALID_HW_RING_ID;
5920 #endif
5921 }
5922
5923 int bnxt_hwrm_l2_filter_free(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5924 {
5925 struct hwrm_cfa_l2_filter_free_input *req;
5926 u16 target_id = 0xffff;
5927 int rc;
5928
5929 if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5930 struct bnxt_pf_info *pf = &bp->pf;
5931
5932 if (fltr->base.vf_idx >= pf->active_vfs)
5933 return -EINVAL;
5934
5935 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5936 if (target_id == INVALID_HW_RING_ID)
5937 return -EINVAL;
5938 }
5939
5940 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5941 if (rc)
5942 return rc;
5943
5944 req->target_id = cpu_to_le16(target_id);
5945 req->l2_filter_id = fltr->base.filter_id;
5946 return hwrm_req_send(bp, req);
5947 }
5948
5949 int bnxt_hwrm_l2_filter_alloc(struct bnxt *bp, struct bnxt_l2_filter *fltr)
5950 {
5951 struct hwrm_cfa_l2_filter_alloc_output *resp;
5952 struct hwrm_cfa_l2_filter_alloc_input *req;
5953 u16 target_id = 0xffff;
5954 int rc;
5955
5956 if (fltr->base.flags & BNXT_ACT_FUNC_DST) {
5957 struct bnxt_pf_info *pf = &bp->pf;
5958
5959 if (fltr->base.vf_idx >= pf->active_vfs)
5960 return -EINVAL;
5961
5962 target_id = bnxt_vf_target_id(pf, fltr->base.vf_idx);
5963 }
5964 rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5965 if (rc)
5966 return rc;
5967
5968 req->target_id = cpu_to_le16(target_id);
5969 req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5970
5971 if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5972 req->flags |=
5973 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5974 req->dst_id = cpu_to_le16(fltr->base.fw_vnic_id);
5975 req->enables =
5976 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5977 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5978 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5979 ether_addr_copy(req->l2_addr, fltr->l2_key.dst_mac_addr);
5980 eth_broadcast_addr(req->l2_addr_mask);
5981
5982 if (fltr->l2_key.vlan) {
5983 req->enables |=
5984 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN |
5985 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_IVLAN_MASK |
5986 CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS);
5987 req->num_vlans = 1;
5988 req->l2_ivlan = cpu_to_le16(fltr->l2_key.vlan);
5989 req->l2_ivlan_mask = cpu_to_le16(0xfff);
5990 }
5991
5992 resp = hwrm_req_hold(bp, req);
5993 rc = hwrm_req_send(bp, req);
5994 if (!rc) {
5995 fltr->base.filter_id = resp->l2_filter_id;
5996 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
5997 }
5998 hwrm_req_drop(bp, req);
5999 return rc;
6000 }
6001
6002 int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
6003 struct bnxt_ntuple_filter *fltr)
6004 {
6005 struct hwrm_cfa_ntuple_filter_free_input *req;
6006 int rc;
6007
6008 set_bit(BNXT_FLTR_FW_DELETED, &fltr->base.state);
6009 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
6010 if (rc)
6011 return rc;
6012
6013 req->ntuple_filter_id = fltr->base.filter_id;
6014 return hwrm_req_send(bp, req);
6015 }
6016
6017 #define BNXT_NTP_FLTR_FLAGS \
6018 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
6019 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
6020 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
6021 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
6022 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
6023 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
6024 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
6025 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
6026 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
6027 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
6028 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
6029 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
6030 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
6031
6032 #define BNXT_NTP_TUNNEL_FLTR_FLAG \
6033 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
6034
6035 void bnxt_fill_ipv6_mask(__be32 mask[4])
6036 {
6037 int i;
6038
6039 for (i = 0; i < 4; i++)
6040 mask[i] = cpu_to_be32(~0);
6041 }
6042
6043 static void
6044 bnxt_cfg_rfs_ring_tbl_idx(struct bnxt *bp,
6045 struct hwrm_cfa_ntuple_filter_alloc_input *req,
6046 struct bnxt_ntuple_filter *fltr)
6047 {
6048 u16 rxq = fltr->base.rxq;
6049
6050 if (fltr->base.flags & BNXT_ACT_RSS_CTX) {
6051 struct ethtool_rxfh_context *ctx;
6052 struct bnxt_rss_ctx *rss_ctx;
6053 struct bnxt_vnic_info *vnic;
6054
6055 ctx = xa_load(&bp->dev->ethtool->rss_ctx,
6056 fltr->base.fw_vnic_id);
6057 if (ctx) {
6058 rss_ctx = ethtool_rxfh_context_priv(ctx);
6059 vnic = &rss_ctx->vnic;
6060
6061 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
6062 }
6063 return;
6064 }
6065 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) {
6066 struct bnxt_vnic_info *vnic;
6067 u32 enables;
6068
6069 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE];
6070 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
6071 enables = CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX;
6072 req->enables |= cpu_to_le32(enables);
6073 req->rfs_ring_tbl_idx = cpu_to_le16(rxq);
6074 } else {
6075 u32 flags;
6076
6077 flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
6078 req->flags |= cpu_to_le32(flags);
6079 req->dst_id = cpu_to_le16(rxq);
6080 }
6081 }
6082
6083 int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
6084 struct bnxt_ntuple_filter *fltr)
6085 {
6086 struct hwrm_cfa_ntuple_filter_alloc_output *resp;
6087 struct hwrm_cfa_ntuple_filter_alloc_input *req;
6088 struct bnxt_flow_masks *masks = &fltr->fmasks;
6089 struct flow_keys *keys = &fltr->fkeys;
6090 struct bnxt_l2_filter *l2_fltr;
6091 struct bnxt_vnic_info *vnic;
6092 int rc;
6093
6094 rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
6095 if (rc)
6096 return rc;
6097
6098 l2_fltr = fltr->l2_fltr;
6099 req->l2_filter_id = l2_fltr->base.filter_id;
6100
6101 if (fltr->base.flags & BNXT_ACT_DROP) {
6102 req->flags =
6103 cpu_to_le32(CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP);
6104 } else if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
6105 bnxt_cfg_rfs_ring_tbl_idx(bp, req, fltr);
6106 } else {
6107 vnic = &bp->vnic_info[fltr->base.rxq + 1];
6108 req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
6109 }
6110 req->enables |= cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
6111
6112 req->ethertype = htons(ETH_P_IP);
6113 req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
6114 req->ip_protocol = keys->basic.ip_proto;
6115
6116 if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
6117 req->ethertype = htons(ETH_P_IPV6);
6118 req->ip_addr_type =
6119 CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
6120 *(struct in6_addr *)&req->src_ipaddr[0] = keys->addrs.v6addrs.src;
6121 *(struct in6_addr *)&req->src_ipaddr_mask[0] = masks->addrs.v6addrs.src;
6122 *(struct in6_addr *)&req->dst_ipaddr[0] = keys->addrs.v6addrs.dst;
6123 *(struct in6_addr *)&req->dst_ipaddr_mask[0] = masks->addrs.v6addrs.dst;
6124 } else {
6125 req->src_ipaddr[0] = keys->addrs.v4addrs.src;
6126 req->src_ipaddr_mask[0] = masks->addrs.v4addrs.src;
6127 req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
6128 req->dst_ipaddr_mask[0] = masks->addrs.v4addrs.dst;
6129 }
6130 if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
6131 req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
6132 req->tunnel_type =
6133 CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
6134 }
6135
6136 req->src_port = keys->ports.src;
6137 req->src_port_mask = masks->ports.src;
6138 req->dst_port = keys->ports.dst;
6139 req->dst_port_mask = masks->ports.dst;
6140
6141 resp = hwrm_req_hold(bp, req);
6142 rc = hwrm_req_send(bp, req);
6143 if (!rc)
6144 fltr->base.filter_id = resp->ntuple_filter_id;
6145 hwrm_req_drop(bp, req);
6146 return rc;
6147 }
6148
6149 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
6150 const u8 *mac_addr)
6151 {
6152 struct bnxt_l2_filter *fltr;
6153 struct bnxt_l2_key key;
6154 int rc;
6155
6156 ether_addr_copy(key.dst_mac_addr, mac_addr);
6157 key.vlan = 0;
6158 fltr = bnxt_alloc_l2_filter(bp, &key, GFP_KERNEL);
6159 if (IS_ERR(fltr))
6160 return PTR_ERR(fltr);
6161
6162 fltr->base.fw_vnic_id = bp->vnic_info[vnic_id].fw_vnic_id;
6163 rc = bnxt_hwrm_l2_filter_alloc(bp, fltr);
6164 if (rc)
6165 bnxt_del_l2_filter(bp, fltr);
6166 else
6167 bp->vnic_info[vnic_id].l2_filters[idx] = fltr;
6168 return rc;
6169 }
6170
6171 static void bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
6172 {
6173 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
6174
6175 /* Any associated ntuple filters will also be cleared by firmware. */
6176 for (i = 0; i < num_of_vnics; i++) {
6177 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6178
6179 for (j = 0; j < vnic->uc_filter_count; j++) {
6180 struct bnxt_l2_filter *fltr = vnic->l2_filters[j];
6181
6182 bnxt_hwrm_l2_filter_free(bp, fltr);
6183 bnxt_del_l2_filter(bp, fltr);
6184 }
6185 vnic->uc_filter_count = 0;
6186 }
6187 }
6188
6189 #define BNXT_DFLT_TUNL_TPA_BMAP \
6190 (VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE | \
6191 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 | \
6192 VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6)
6193
6194 static void bnxt_hwrm_vnic_update_tunl_tpa(struct bnxt *bp,
6195 struct hwrm_vnic_tpa_cfg_input *req)
6196 {
6197 u32 tunl_tpa_bmap = BNXT_DFLT_TUNL_TPA_BMAP;
6198
6199 if (!(bp->fw_cap & BNXT_FW_CAP_VNIC_TUNNEL_TPA))
6200 return;
6201
6202 if (bp->vxlan_port)
6203 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN;
6204 if (bp->vxlan_gpe_port)
6205 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE;
6206 if (bp->nge_port)
6207 tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE;
6208
6209 req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN);
6210 req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap);
6211 }
6212
6213 int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6214 u32 tpa_flags)
6215 {
6216 u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
6217 struct hwrm_vnic_tpa_cfg_input *req;
6218 int rc;
6219
6220 if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
6221 return 0;
6222
6223 rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
6224 if (rc)
6225 return rc;
6226
6227 if (tpa_flags) {
6228 u16 mss = bp->dev->mtu - 40;
6229 u32 nsegs, n, segs = 0, flags;
6230
6231 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
6232 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
6233 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
6234 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
6235 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
6236 if (tpa_flags & BNXT_FLAG_GRO)
6237 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
6238
6239 req->flags = cpu_to_le32(flags);
6240
6241 req->enables =
6242 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
6243 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
6244 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
6245
6246 /* Number of segs are log2 units, and first packet is not
6247 * included as part of this units.
6248 */
6249 if (mss <= BNXT_RX_PAGE_SIZE) {
6250 n = BNXT_RX_PAGE_SIZE / mss;
6251 nsegs = (MAX_SKB_FRAGS - 1) * n;
6252 } else {
6253 n = mss / BNXT_RX_PAGE_SIZE;
6254 if (mss & (BNXT_RX_PAGE_SIZE - 1))
6255 n++;
6256 nsegs = (MAX_SKB_FRAGS - n) / n;
6257 }
6258
6259 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6260 segs = MAX_TPA_SEGS_P5;
6261 max_aggs = bp->max_tpa;
6262 } else {
6263 segs = ilog2(nsegs);
6264 }
6265 req->max_agg_segs = cpu_to_le16(segs);
6266 req->max_aggs = cpu_to_le16(max_aggs);
6267
6268 req->min_agg_len = cpu_to_le32(512);
6269 bnxt_hwrm_vnic_update_tunl_tpa(bp, req);
6270 }
6271 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6272
6273 return hwrm_req_send(bp, req);
6274 }
6275
6276 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
6277 {
6278 struct bnxt_ring_grp_info *grp_info;
6279
6280 grp_info = &bp->grp_info[ring->grp_idx];
6281 return grp_info->cp_fw_ring_id;
6282 }
6283
6284 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
6285 {
6286 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6287 return rxr->rx_cpr->cp_ring_struct.fw_ring_id;
6288 else
6289 return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
6290 }
6291
6292 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
6293 {
6294 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6295 return txr->tx_cpr->cp_ring_struct.fw_ring_id;
6296 else
6297 return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
6298 }
6299
6300 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
6301 {
6302 int entries;
6303
6304 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6305 entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
6306 else
6307 entries = HW_HASH_INDEX_SIZE;
6308
6309 bp->rss_indir_tbl_entries = entries;
6310 bp->rss_indir_tbl =
6311 kmalloc_array(entries, sizeof(*bp->rss_indir_tbl), GFP_KERNEL);
6312 if (!bp->rss_indir_tbl)
6313 return -ENOMEM;
6314
6315 return 0;
6316 }
6317
6318 void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp,
6319 struct ethtool_rxfh_context *rss_ctx)
6320 {
6321 u16 max_rings, max_entries, pad, i;
6322 u32 *rss_indir_tbl;
6323
6324 if (!bp->rx_nr_rings)
6325 return;
6326
6327 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6328 max_rings = bp->rx_nr_rings - 1;
6329 else
6330 max_rings = bp->rx_nr_rings;
6331
6332 max_entries = bnxt_get_rxfh_indir_size(bp->dev);
6333 if (rss_ctx)
6334 rss_indir_tbl = ethtool_rxfh_context_indir(rss_ctx);
6335 else
6336 rss_indir_tbl = &bp->rss_indir_tbl[0];
6337
6338 for (i = 0; i < max_entries; i++)
6339 rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
6340
6341 pad = bp->rss_indir_tbl_entries - max_entries;
6342 if (pad)
6343 memset(&rss_indir_tbl[i], 0, pad * sizeof(*rss_indir_tbl));
6344 }
6345
6346 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
6347 {
6348 u32 i, tbl_size, max_ring = 0;
6349
6350 if (!bp->rss_indir_tbl)
6351 return 0;
6352
6353 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
6354 for (i = 0; i < tbl_size; i++)
6355 max_ring = max(max_ring, bp->rss_indir_tbl[i]);
6356 return max_ring;
6357 }
6358
6359 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
6360 {
6361 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6362 if (!rx_rings)
6363 return 0;
6364 return bnxt_calc_nr_ring_pages(rx_rings - 1,
6365 BNXT_RSS_TABLE_ENTRIES_P5);
6366 }
6367 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6368 return 2;
6369 return 1;
6370 }
6371
6372 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6373 {
6374 bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
6375 u16 i, j;
6376
6377 /* Fill the RSS indirection table with ring group ids */
6378 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
6379 if (!no_rss)
6380 j = bp->rss_indir_tbl[i];
6381 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
6382 }
6383 }
6384
6385 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
6386 struct bnxt_vnic_info *vnic)
6387 {
6388 __le16 *ring_tbl = vnic->rss_table;
6389 struct bnxt_rx_ring_info *rxr;
6390 u16 tbl_size, i;
6391
6392 tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
6393
6394 for (i = 0; i < tbl_size; i++) {
6395 u16 ring_id, j;
6396
6397 if (vnic->flags & BNXT_VNIC_NTUPLE_FLAG)
6398 j = ethtool_rxfh_indir_default(i, bp->rx_nr_rings);
6399 else if (vnic->flags & BNXT_VNIC_RSSCTX_FLAG)
6400 j = ethtool_rxfh_context_indir(vnic->rss_ctx)[i];
6401 else
6402 j = bp->rss_indir_tbl[i];
6403 rxr = &bp->rx_ring[j];
6404
6405 ring_id = rxr->rx_ring_struct.fw_ring_id;
6406 *ring_tbl++ = cpu_to_le16(ring_id);
6407 ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6408 *ring_tbl++ = cpu_to_le16(ring_id);
6409 }
6410 }
6411
6412 static void
6413 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
6414 struct bnxt_vnic_info *vnic)
6415 {
6416 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6417 bnxt_fill_hw_rss_tbl_p5(bp, vnic);
6418 if (bp->flags & BNXT_FLAG_CHIP_P7)
6419 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT;
6420 } else {
6421 bnxt_fill_hw_rss_tbl(bp, vnic);
6422 }
6423
6424 if (bp->rss_hash_delta) {
6425 req->hash_type = cpu_to_le32(bp->rss_hash_delta);
6426 if (bp->rss_hash_cfg & bp->rss_hash_delta)
6427 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
6428 else
6429 req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
6430 } else {
6431 req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
6432 }
6433 req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
6434 req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
6435 req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
6436 }
6437
6438 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6439 bool set_rss)
6440 {
6441 struct hwrm_vnic_rss_cfg_input *req;
6442 int rc;
6443
6444 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ||
6445 vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
6446 return 0;
6447
6448 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6449 if (rc)
6450 return rc;
6451
6452 if (set_rss)
6453 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6454 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6455 return hwrm_req_send(bp, req);
6456 }
6457
6458 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp,
6459 struct bnxt_vnic_info *vnic, bool set_rss)
6460 {
6461 struct hwrm_vnic_rss_cfg_input *req;
6462 dma_addr_t ring_tbl_map;
6463 u32 i, nr_ctxs;
6464 int rc;
6465
6466 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
6467 if (rc)
6468 return rc;
6469
6470 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6471 if (!set_rss)
6472 return hwrm_req_send(bp, req);
6473
6474 __bnxt_hwrm_vnic_set_rss(bp, req, vnic);
6475 ring_tbl_map = vnic->rss_table_dma_addr;
6476 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
6477
6478 hwrm_req_hold(bp, req);
6479 for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
6480 req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
6481 req->ring_table_pair_index = i;
6482 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
6483 rc = hwrm_req_send(bp, req);
6484 if (rc)
6485 goto exit;
6486 }
6487
6488 exit:
6489 hwrm_req_drop(bp, req);
6490 return rc;
6491 }
6492
6493 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
6494 {
6495 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
6496 struct hwrm_vnic_rss_qcfg_output *resp;
6497 struct hwrm_vnic_rss_qcfg_input *req;
6498
6499 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
6500 return;
6501
6502 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6503 /* all contexts configured to same hash_type, zero always exists */
6504 req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6505 resp = hwrm_req_hold(bp, req);
6506 if (!hwrm_req_send(bp, req)) {
6507 bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
6508 bp->rss_hash_delta = 0;
6509 }
6510 hwrm_req_drop(bp, req);
6511 }
6512
6513 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6514 {
6515 struct hwrm_vnic_plcmodes_cfg_input *req;
6516 int rc;
6517
6518 rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
6519 if (rc)
6520 return rc;
6521
6522 req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
6523 req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
6524
6525 if (BNXT_RX_PAGE_MODE(bp)) {
6526 req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
6527 } else {
6528 req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
6529 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
6530 req->enables |=
6531 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
6532 req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
6533 req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
6534 }
6535 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6536 return hwrm_req_send(bp, req);
6537 }
6538
6539 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp,
6540 struct bnxt_vnic_info *vnic,
6541 u16 ctx_idx)
6542 {
6543 struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
6544
6545 if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
6546 return;
6547
6548 req->rss_cos_lb_ctx_id =
6549 cpu_to_le16(vnic->fw_rss_cos_lb_ctx[ctx_idx]);
6550
6551 hwrm_req_send(bp, req);
6552 vnic->fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
6553 }
6554
6555 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
6556 {
6557 int i, j;
6558
6559 for (i = 0; i < bp->nr_vnics; i++) {
6560 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
6561
6562 for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
6563 if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
6564 bnxt_hwrm_vnic_ctx_free_one(bp, vnic, j);
6565 }
6566 }
6567 bp->rsscos_nr_ctxs = 0;
6568 }
6569
6570 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp,
6571 struct bnxt_vnic_info *vnic, u16 ctx_idx)
6572 {
6573 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
6574 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
6575 int rc;
6576
6577 rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
6578 if (rc)
6579 return rc;
6580
6581 resp = hwrm_req_hold(bp, req);
6582 rc = hwrm_req_send(bp, req);
6583 if (!rc)
6584 vnic->fw_rss_cos_lb_ctx[ctx_idx] =
6585 le16_to_cpu(resp->rss_cos_lb_ctx_id);
6586 hwrm_req_drop(bp, req);
6587
6588 return rc;
6589 }
6590
6591 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
6592 {
6593 if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
6594 return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
6595 return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
6596 }
6597
6598 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
6599 {
6600 struct bnxt_vnic_info *vnic0 = &bp->vnic_info[BNXT_VNIC_DEFAULT];
6601 struct hwrm_vnic_cfg_input *req;
6602 unsigned int ring = 0, grp_idx;
6603 u16 def_vlan = 0;
6604 int rc;
6605
6606 rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
6607 if (rc)
6608 return rc;
6609
6610 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6611 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
6612
6613 req->default_rx_ring_id =
6614 cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
6615 req->default_cmpl_ring_id =
6616 cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
6617 req->enables =
6618 cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
6619 VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
6620 goto vnic_mru;
6621 }
6622 req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
6623 /* Only RSS support for now TBD: COS & LB */
6624 if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
6625 req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
6626 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6627 VNIC_CFG_REQ_ENABLES_MRU);
6628 } else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
6629 req->rss_rule = cpu_to_le16(vnic0->fw_rss_cos_lb_ctx[0]);
6630 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
6631 VNIC_CFG_REQ_ENABLES_MRU);
6632 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
6633 } else {
6634 req->rss_rule = cpu_to_le16(0xffff);
6635 }
6636
6637 if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
6638 (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
6639 req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
6640 req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
6641 } else {
6642 req->cos_rule = cpu_to_le16(0xffff);
6643 }
6644
6645 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
6646 ring = 0;
6647 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
6648 ring = vnic->vnic_id - 1;
6649 else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
6650 ring = bp->rx_nr_rings - 1;
6651
6652 grp_idx = bp->rx_ring[ring].bnapi->index;
6653 req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
6654 req->lb_rule = cpu_to_le16(0xffff);
6655 vnic_mru:
6656 vnic->mru = bp->dev->mtu + ETH_HLEN + VLAN_HLEN;
6657 req->mru = cpu_to_le16(vnic->mru);
6658
6659 req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
6660 #ifdef CONFIG_BNXT_SRIOV
6661 if (BNXT_VF(bp))
6662 def_vlan = bp->vf.vlan;
6663 #endif
6664 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
6665 req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
6666 if (vnic->vnic_id == BNXT_VNIC_DEFAULT && bnxt_ulp_registered(bp->edev))
6667 req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
6668
6669 return hwrm_req_send(bp, req);
6670 }
6671
6672 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp,
6673 struct bnxt_vnic_info *vnic)
6674 {
6675 if (vnic->fw_vnic_id != INVALID_HW_RING_ID) {
6676 struct hwrm_vnic_free_input *req;
6677
6678 if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
6679 return;
6680
6681 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
6682
6683 hwrm_req_send(bp, req);
6684 vnic->fw_vnic_id = INVALID_HW_RING_ID;
6685 }
6686 }
6687
6688 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
6689 {
6690 u16 i;
6691
6692 for (i = 0; i < bp->nr_vnics; i++)
6693 bnxt_hwrm_vnic_free_one(bp, &bp->vnic_info[i]);
6694 }
6695
6696 int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic,
6697 unsigned int start_rx_ring_idx,
6698 unsigned int nr_rings)
6699 {
6700 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
6701 struct hwrm_vnic_alloc_output *resp;
6702 struct hwrm_vnic_alloc_input *req;
6703 int rc;
6704
6705 rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
6706 if (rc)
6707 return rc;
6708
6709 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6710 goto vnic_no_ring_grps;
6711
6712 /* map ring groups to this vnic */
6713 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
6714 grp_idx = bp->rx_ring[i].bnapi->index;
6715 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
6716 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
6717 j, nr_rings);
6718 break;
6719 }
6720 vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
6721 }
6722
6723 vnic_no_ring_grps:
6724 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
6725 vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
6726 if (vnic->vnic_id == BNXT_VNIC_DEFAULT)
6727 req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
6728
6729 resp = hwrm_req_hold(bp, req);
6730 rc = hwrm_req_send(bp, req);
6731 if (!rc)
6732 vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
6733 hwrm_req_drop(bp, req);
6734 return rc;
6735 }
6736
6737 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
6738 {
6739 struct hwrm_vnic_qcaps_output *resp;
6740 struct hwrm_vnic_qcaps_input *req;
6741 int rc;
6742
6743 bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
6744 bp->flags &= ~BNXT_FLAG_ROCE_MIRROR_CAP;
6745 bp->rss_cap &= ~BNXT_RSS_CAP_NEW_RSS_CAP;
6746 if (bp->hwrm_spec_code < 0x10600)
6747 return 0;
6748
6749 rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
6750 if (rc)
6751 return rc;
6752
6753 resp = hwrm_req_hold(bp, req);
6754 rc = hwrm_req_send(bp, req);
6755 if (!rc) {
6756 u32 flags = le32_to_cpu(resp->flags);
6757
6758 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
6759 (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
6760 bp->rss_cap |= BNXT_RSS_CAP_NEW_RSS_CAP;
6761 if (flags &
6762 VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
6763 bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
6764
6765 /* Older P5 fw before EXT_HW_STATS support did not set
6766 * VLAN_STRIP_CAP properly.
6767 */
6768 if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
6769 (BNXT_CHIP_P5(bp) &&
6770 !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
6771 bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
6772 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
6773 bp->rss_cap |= BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA;
6774 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_PROF_TCAM_MODE_ENABLED)
6775 bp->rss_cap |= BNXT_RSS_CAP_RSS_TCAM;
6776 bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
6777 if (bp->max_tpa_v2) {
6778 if (BNXT_CHIP_P5(bp))
6779 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
6780 else
6781 bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P7;
6782 }
6783 if (flags & VNIC_QCAPS_RESP_FLAGS_HW_TUNNEL_TPA_CAP)
6784 bp->fw_cap |= BNXT_FW_CAP_VNIC_TUNNEL_TPA;
6785 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV4_CAP)
6786 bp->rss_cap |= BNXT_RSS_CAP_AH_V4_RSS_CAP;
6787 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_AH_SPI_IPV6_CAP)
6788 bp->rss_cap |= BNXT_RSS_CAP_AH_V6_RSS_CAP;
6789 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV4_CAP)
6790 bp->rss_cap |= BNXT_RSS_CAP_ESP_V4_RSS_CAP;
6791 if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_IPSEC_ESP_SPI_IPV6_CAP)
6792 bp->rss_cap |= BNXT_RSS_CAP_ESP_V6_RSS_CAP;
6793 if (flags & VNIC_QCAPS_RESP_FLAGS_RE_FLUSH_CAP)
6794 bp->fw_cap |= BNXT_FW_CAP_VNIC_RE_FLUSH;
6795 }
6796 hwrm_req_drop(bp, req);
6797 return rc;
6798 }
6799
6800 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
6801 {
6802 struct hwrm_ring_grp_alloc_output *resp;
6803 struct hwrm_ring_grp_alloc_input *req;
6804 int rc;
6805 u16 i;
6806
6807 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
6808 return 0;
6809
6810 rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
6811 if (rc)
6812 return rc;
6813
6814 resp = hwrm_req_hold(bp, req);
6815 for (i = 0; i < bp->rx_nr_rings; i++) {
6816 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
6817
6818 req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
6819 req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
6820 req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
6821 req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
6822
6823 rc = hwrm_req_send(bp, req);
6824
6825 if (rc)
6826 break;
6827
6828 bp->grp_info[grp_idx].fw_grp_id =
6829 le32_to_cpu(resp->ring_group_id);
6830 }
6831 hwrm_req_drop(bp, req);
6832 return rc;
6833 }
6834
6835 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
6836 {
6837 struct hwrm_ring_grp_free_input *req;
6838 u16 i;
6839
6840 if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
6841 return;
6842
6843 if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
6844 return;
6845
6846 hwrm_req_hold(bp, req);
6847 for (i = 0; i < bp->cp_nr_rings; i++) {
6848 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
6849 continue;
6850 req->ring_group_id =
6851 cpu_to_le32(bp->grp_info[i].fw_grp_id);
6852
6853 hwrm_req_send(bp, req);
6854 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
6855 }
6856 hwrm_req_drop(bp, req);
6857 }
6858
6859 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
6860 struct bnxt_ring_struct *ring,
6861 u32 ring_type, u32 map_index)
6862 {
6863 struct hwrm_ring_alloc_output *resp;
6864 struct hwrm_ring_alloc_input *req;
6865 struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
6866 struct bnxt_ring_grp_info *grp_info;
6867 int rc, err = 0;
6868 u16 ring_id;
6869
6870 rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
6871 if (rc)
6872 goto exit;
6873
6874 req->enables = 0;
6875 if (rmem->nr_pages > 1) {
6876 req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
6877 /* Page size is in log2 units */
6878 req->page_size = BNXT_PAGE_SHIFT;
6879 req->page_tbl_depth = 1;
6880 } else {
6881 req->page_tbl_addr = cpu_to_le64(rmem->dma_arr[0]);
6882 }
6883 req->fbo = 0;
6884 /* Association of ring index with doorbell index and MSIX number */
6885 req->logical_id = cpu_to_le16(map_index);
6886
6887 switch (ring_type) {
6888 case HWRM_RING_ALLOC_TX: {
6889 struct bnxt_tx_ring_info *txr;
6890 u16 flags = 0;
6891
6892 txr = container_of(ring, struct bnxt_tx_ring_info,
6893 tx_ring_struct);
6894 req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
6895 /* Association of transmit ring with completion ring */
6896 grp_info = &bp->grp_info[ring->grp_idx];
6897 req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
6898 req->length = cpu_to_le32(bp->tx_ring_mask + 1);
6899 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6900 req->queue_id = cpu_to_le16(ring->queue_id);
6901 if (bp->flags & BNXT_FLAG_TX_COAL_CMPL)
6902 req->cmpl_coal_cnt =
6903 RING_ALLOC_REQ_CMPL_COAL_CNT_COAL_64;
6904 if ((bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP) && bp->ptp_cfg)
6905 flags |= RING_ALLOC_REQ_FLAGS_TX_PKT_TS_CMPL_ENABLE;
6906 req->flags = cpu_to_le16(flags);
6907 break;
6908 }
6909 case HWRM_RING_ALLOC_RX:
6910 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6911 req->length = cpu_to_le32(bp->rx_ring_mask + 1);
6912 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6913 u16 flags = 0;
6914
6915 /* Association of rx ring with stats context */
6916 grp_info = &bp->grp_info[ring->grp_idx];
6917 req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
6918 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6919 req->enables |= cpu_to_le32(
6920 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6921 if (NET_IP_ALIGN == 2)
6922 flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
6923 req->flags = cpu_to_le16(flags);
6924 }
6925 break;
6926 case HWRM_RING_ALLOC_AGG:
6927 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6928 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
6929 /* Association of agg ring with rx ring */
6930 grp_info = &bp->grp_info[ring->grp_idx];
6931 req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
6932 req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
6933 req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
6934 req->enables |= cpu_to_le32(
6935 RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
6936 RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
6937 } else {
6938 req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
6939 }
6940 req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
6941 break;
6942 case HWRM_RING_ALLOC_CMPL:
6943 req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
6944 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6945 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
6946 /* Association of cp ring with nq */
6947 grp_info = &bp->grp_info[map_index];
6948 req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
6949 req->cq_handle = cpu_to_le64(ring->handle);
6950 req->enables |= cpu_to_le32(
6951 RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
6952 } else {
6953 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6954 }
6955 break;
6956 case HWRM_RING_ALLOC_NQ:
6957 req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
6958 req->length = cpu_to_le32(bp->cp_ring_mask + 1);
6959 req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
6960 break;
6961 default:
6962 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
6963 ring_type);
6964 return -1;
6965 }
6966
6967 resp = hwrm_req_hold(bp, req);
6968 rc = hwrm_req_send(bp, req);
6969 err = le16_to_cpu(resp->error_code);
6970 ring_id = le16_to_cpu(resp->ring_id);
6971 hwrm_req_drop(bp, req);
6972
6973 exit:
6974 if (rc || err) {
6975 netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
6976 ring_type, rc, err);
6977 return -EIO;
6978 }
6979 ring->fw_ring_id = ring_id;
6980 return rc;
6981 }
6982
6983 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
6984 {
6985 int rc;
6986
6987 if (BNXT_PF(bp)) {
6988 struct hwrm_func_cfg_input *req;
6989
6990 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
6991 if (rc)
6992 return rc;
6993
6994 req->fid = cpu_to_le16(0xffff);
6995 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
6996 req->async_event_cr = cpu_to_le16(idx);
6997 return hwrm_req_send(bp, req);
6998 } else {
6999 struct hwrm_func_vf_cfg_input *req;
7000
7001 rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
7002 if (rc)
7003 return rc;
7004
7005 req->enables =
7006 cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
7007 req->async_event_cr = cpu_to_le16(idx);
7008 return hwrm_req_send(bp, req);
7009 }
7010 }
7011
7012 static void bnxt_set_db_mask(struct bnxt *bp, struct bnxt_db_info *db,
7013 u32 ring_type)
7014 {
7015 switch (ring_type) {
7016 case HWRM_RING_ALLOC_TX:
7017 db->db_ring_mask = bp->tx_ring_mask;
7018 break;
7019 case HWRM_RING_ALLOC_RX:
7020 db->db_ring_mask = bp->rx_ring_mask;
7021 break;
7022 case HWRM_RING_ALLOC_AGG:
7023 db->db_ring_mask = bp->rx_agg_ring_mask;
7024 break;
7025 case HWRM_RING_ALLOC_CMPL:
7026 case HWRM_RING_ALLOC_NQ:
7027 db->db_ring_mask = bp->cp_ring_mask;
7028 break;
7029 }
7030 if (bp->flags & BNXT_FLAG_CHIP_P7) {
7031 db->db_epoch_mask = db->db_ring_mask + 1;
7032 db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask);
7033 }
7034 }
7035
7036 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
7037 u32 map_idx, u32 xid)
7038 {
7039 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7040 switch (ring_type) {
7041 case HWRM_RING_ALLOC_TX:
7042 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
7043 break;
7044 case HWRM_RING_ALLOC_RX:
7045 case HWRM_RING_ALLOC_AGG:
7046 db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
7047 break;
7048 case HWRM_RING_ALLOC_CMPL:
7049 db->db_key64 = DBR_PATH_L2;
7050 break;
7051 case HWRM_RING_ALLOC_NQ:
7052 db->db_key64 = DBR_PATH_L2;
7053 break;
7054 }
7055 db->db_key64 |= (u64)xid << DBR_XID_SFT;
7056
7057 if (bp->flags & BNXT_FLAG_CHIP_P7)
7058 db->db_key64 |= DBR_VALID;
7059
7060 db->doorbell = bp->bar1 + bp->db_offset;
7061 } else {
7062 db->doorbell = bp->bar1 + map_idx * 0x80;
7063 switch (ring_type) {
7064 case HWRM_RING_ALLOC_TX:
7065 db->db_key32 = DB_KEY_TX;
7066 break;
7067 case HWRM_RING_ALLOC_RX:
7068 case HWRM_RING_ALLOC_AGG:
7069 db->db_key32 = DB_KEY_RX;
7070 break;
7071 case HWRM_RING_ALLOC_CMPL:
7072 db->db_key32 = DB_KEY_CP;
7073 break;
7074 }
7075 }
7076 bnxt_set_db_mask(bp, db, ring_type);
7077 }
7078
7079 static int bnxt_hwrm_rx_ring_alloc(struct bnxt *bp,
7080 struct bnxt_rx_ring_info *rxr)
7081 {
7082 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
7083 struct bnxt_napi *bnapi = rxr->bnapi;
7084 u32 type = HWRM_RING_ALLOC_RX;
7085 u32 map_idx = bnapi->index;
7086 int rc;
7087
7088 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7089 if (rc)
7090 return rc;
7091
7092 bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
7093 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
7094
7095 return 0;
7096 }
7097
7098 static int bnxt_hwrm_rx_agg_ring_alloc(struct bnxt *bp,
7099 struct bnxt_rx_ring_info *rxr)
7100 {
7101 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
7102 u32 type = HWRM_RING_ALLOC_AGG;
7103 u32 grp_idx = ring->grp_idx;
7104 u32 map_idx;
7105 int rc;
7106
7107 map_idx = grp_idx + bp->rx_nr_rings;
7108 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7109 if (rc)
7110 return rc;
7111
7112 bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
7113 ring->fw_ring_id);
7114 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
7115 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
7116 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
7117
7118 return 0;
7119 }
7120
7121 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
7122 {
7123 bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
7124 int i, rc = 0;
7125 u32 type;
7126
7127 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7128 type = HWRM_RING_ALLOC_NQ;
7129 else
7130 type = HWRM_RING_ALLOC_CMPL;
7131 for (i = 0; i < bp->cp_nr_rings; i++) {
7132 struct bnxt_napi *bnapi = bp->bnapi[i];
7133 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7134 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
7135 u32 map_idx = ring->map_idx;
7136 unsigned int vector;
7137
7138 vector = bp->irq_tbl[map_idx].vector;
7139 disable_irq_nosync(vector);
7140 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7141 if (rc) {
7142 enable_irq(vector);
7143 goto err_out;
7144 }
7145 bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
7146 bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
7147 enable_irq(vector);
7148 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
7149
7150 if (!i) {
7151 rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
7152 if (rc)
7153 netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
7154 }
7155 }
7156
7157 type = HWRM_RING_ALLOC_TX;
7158 for (i = 0; i < bp->tx_nr_rings; i++) {
7159 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
7160 struct bnxt_ring_struct *ring;
7161 u32 map_idx;
7162
7163 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7164 struct bnxt_cp_ring_info *cpr2 = txr->tx_cpr;
7165 struct bnxt_napi *bnapi = txr->bnapi;
7166 u32 type2 = HWRM_RING_ALLOC_CMPL;
7167
7168 ring = &cpr2->cp_ring_struct;
7169 ring->handle = BNXT_SET_NQ_HDL(cpr2);
7170 map_idx = bnapi->index;
7171 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
7172 if (rc)
7173 goto err_out;
7174 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
7175 ring->fw_ring_id);
7176 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
7177 }
7178 ring = &txr->tx_ring_struct;
7179 map_idx = i;
7180 rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
7181 if (rc)
7182 goto err_out;
7183 bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
7184 }
7185
7186 for (i = 0; i < bp->rx_nr_rings; i++) {
7187 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
7188
7189 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr);
7190 if (rc)
7191 goto err_out;
7192 /* If we have agg rings, post agg buffers first. */
7193 if (!agg_rings)
7194 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
7195 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7196 struct bnxt_cp_ring_info *cpr2 = rxr->rx_cpr;
7197 struct bnxt_napi *bnapi = rxr->bnapi;
7198 u32 type2 = HWRM_RING_ALLOC_CMPL;
7199 struct bnxt_ring_struct *ring;
7200 u32 map_idx = bnapi->index;
7201
7202 ring = &cpr2->cp_ring_struct;
7203 ring->handle = BNXT_SET_NQ_HDL(cpr2);
7204 rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
7205 if (rc)
7206 goto err_out;
7207 bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
7208 ring->fw_ring_id);
7209 bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
7210 }
7211 }
7212
7213 if (agg_rings) {
7214 for (i = 0; i < bp->rx_nr_rings; i++) {
7215 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, &bp->rx_ring[i]);
7216 if (rc)
7217 goto err_out;
7218 }
7219 }
7220 err_out:
7221 return rc;
7222 }
7223
7224 static int hwrm_ring_free_send_msg(struct bnxt *bp,
7225 struct bnxt_ring_struct *ring,
7226 u32 ring_type, int cmpl_ring_id)
7227 {
7228 struct hwrm_ring_free_output *resp;
7229 struct hwrm_ring_free_input *req;
7230 u16 error_code = 0;
7231 int rc;
7232
7233 if (BNXT_NO_FW_ACCESS(bp))
7234 return 0;
7235
7236 rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
7237 if (rc)
7238 goto exit;
7239
7240 req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
7241 req->ring_type = ring_type;
7242 req->ring_id = cpu_to_le16(ring->fw_ring_id);
7243
7244 resp = hwrm_req_hold(bp, req);
7245 rc = hwrm_req_send(bp, req);
7246 error_code = le16_to_cpu(resp->error_code);
7247 hwrm_req_drop(bp, req);
7248 exit:
7249 if (rc || error_code) {
7250 netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
7251 ring_type, rc, error_code);
7252 return -EIO;
7253 }
7254 return 0;
7255 }
7256
7257 static void bnxt_hwrm_rx_ring_free(struct bnxt *bp,
7258 struct bnxt_rx_ring_info *rxr,
7259 bool close_path)
7260 {
7261 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
7262 u32 grp_idx = rxr->bnapi->index;
7263 u32 cmpl_ring_id;
7264
7265 if (ring->fw_ring_id == INVALID_HW_RING_ID)
7266 return;
7267
7268 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
7269 hwrm_ring_free_send_msg(bp, ring,
7270 RING_FREE_REQ_RING_TYPE_RX,
7271 close_path ? cmpl_ring_id :
7272 INVALID_HW_RING_ID);
7273 ring->fw_ring_id = INVALID_HW_RING_ID;
7274 bp->grp_info[grp_idx].rx_fw_ring_id = INVALID_HW_RING_ID;
7275 }
7276
7277 static void bnxt_hwrm_rx_agg_ring_free(struct bnxt *bp,
7278 struct bnxt_rx_ring_info *rxr,
7279 bool close_path)
7280 {
7281 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
7282 u32 grp_idx = rxr->bnapi->index;
7283 u32 type, cmpl_ring_id;
7284
7285 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7286 type = RING_FREE_REQ_RING_TYPE_RX_AGG;
7287 else
7288 type = RING_FREE_REQ_RING_TYPE_RX;
7289
7290 if (ring->fw_ring_id == INVALID_HW_RING_ID)
7291 return;
7292
7293 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
7294 hwrm_ring_free_send_msg(bp, ring, type,
7295 close_path ? cmpl_ring_id :
7296 INVALID_HW_RING_ID);
7297 ring->fw_ring_id = INVALID_HW_RING_ID;
7298 bp->grp_info[grp_idx].agg_fw_ring_id = INVALID_HW_RING_ID;
7299 }
7300
7301 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
7302 {
7303 u32 type;
7304 int i;
7305
7306 if (!bp->bnapi)
7307 return;
7308
7309 for (i = 0; i < bp->tx_nr_rings; i++) {
7310 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
7311 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
7312
7313 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7314 u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
7315
7316 hwrm_ring_free_send_msg(bp, ring,
7317 RING_FREE_REQ_RING_TYPE_TX,
7318 close_path ? cmpl_ring_id :
7319 INVALID_HW_RING_ID);
7320 ring->fw_ring_id = INVALID_HW_RING_ID;
7321 }
7322 }
7323
7324 for (i = 0; i < bp->rx_nr_rings; i++) {
7325 bnxt_hwrm_rx_ring_free(bp, &bp->rx_ring[i], close_path);
7326 bnxt_hwrm_rx_agg_ring_free(bp, &bp->rx_ring[i], close_path);
7327 }
7328
7329 /* The completion rings are about to be freed. After that the
7330 * IRQ doorbell will not work anymore. So we need to disable
7331 * IRQ here.
7332 */
7333 bnxt_disable_int_sync(bp);
7334
7335 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7336 type = RING_FREE_REQ_RING_TYPE_NQ;
7337 else
7338 type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
7339 for (i = 0; i < bp->cp_nr_rings; i++) {
7340 struct bnxt_napi *bnapi = bp->bnapi[i];
7341 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
7342 struct bnxt_ring_struct *ring;
7343 int j;
7344
7345 for (j = 0; j < cpr->cp_ring_count && cpr->cp_ring_arr; j++) {
7346 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
7347
7348 ring = &cpr2->cp_ring_struct;
7349 if (ring->fw_ring_id == INVALID_HW_RING_ID)
7350 continue;
7351 hwrm_ring_free_send_msg(bp, ring,
7352 RING_FREE_REQ_RING_TYPE_L2_CMPL,
7353 INVALID_HW_RING_ID);
7354 ring->fw_ring_id = INVALID_HW_RING_ID;
7355 }
7356 ring = &cpr->cp_ring_struct;
7357 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
7358 hwrm_ring_free_send_msg(bp, ring, type,
7359 INVALID_HW_RING_ID);
7360 ring->fw_ring_id = INVALID_HW_RING_ID;
7361 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
7362 }
7363 }
7364 }
7365
7366 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7367 bool shared);
7368 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
7369 bool shared);
7370
7371 static int bnxt_hwrm_get_rings(struct bnxt *bp)
7372 {
7373 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7374 struct hwrm_func_qcfg_output *resp;
7375 struct hwrm_func_qcfg_input *req;
7376 int rc;
7377
7378 if (bp->hwrm_spec_code < 0x10601)
7379 return 0;
7380
7381 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7382 if (rc)
7383 return rc;
7384
7385 req->fid = cpu_to_le16(0xffff);
7386 resp = hwrm_req_hold(bp, req);
7387 rc = hwrm_req_send(bp, req);
7388 if (rc) {
7389 hwrm_req_drop(bp, req);
7390 return rc;
7391 }
7392
7393 hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
7394 if (BNXT_NEW_RM(bp)) {
7395 u16 cp, stats;
7396
7397 hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
7398 hw_resc->resv_hw_ring_grps =
7399 le32_to_cpu(resp->alloc_hw_ring_grps);
7400 hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
7401 hw_resc->resv_rsscos_ctxs = le16_to_cpu(resp->alloc_rsscos_ctx);
7402 cp = le16_to_cpu(resp->alloc_cmpl_rings);
7403 stats = le16_to_cpu(resp->alloc_stat_ctx);
7404 hw_resc->resv_irqs = cp;
7405 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7406 int rx = hw_resc->resv_rx_rings;
7407 int tx = hw_resc->resv_tx_rings;
7408
7409 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7410 rx >>= 1;
7411 if (cp < (rx + tx)) {
7412 rc = __bnxt_trim_rings(bp, &rx, &tx, cp, false);
7413 if (rc)
7414 goto get_rings_exit;
7415 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7416 rx <<= 1;
7417 hw_resc->resv_rx_rings = rx;
7418 hw_resc->resv_tx_rings = tx;
7419 }
7420 hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
7421 hw_resc->resv_hw_ring_grps = rx;
7422 }
7423 hw_resc->resv_cp_rings = cp;
7424 hw_resc->resv_stat_ctxs = stats;
7425 }
7426 get_rings_exit:
7427 hwrm_req_drop(bp, req);
7428 return rc;
7429 }
7430
7431 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
7432 {
7433 struct hwrm_func_qcfg_output *resp;
7434 struct hwrm_func_qcfg_input *req;
7435 int rc;
7436
7437 if (bp->hwrm_spec_code < 0x10601)
7438 return 0;
7439
7440 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
7441 if (rc)
7442 return rc;
7443
7444 req->fid = cpu_to_le16(fid);
7445 resp = hwrm_req_hold(bp, req);
7446 rc = hwrm_req_send(bp, req);
7447 if (!rc)
7448 *tx_rings = le16_to_cpu(resp->alloc_tx_rings);
7449
7450 hwrm_req_drop(bp, req);
7451 return rc;
7452 }
7453
7454 static bool bnxt_rfs_supported(struct bnxt *bp);
7455
7456 static struct hwrm_func_cfg_input *
7457 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7458 {
7459 struct hwrm_func_cfg_input *req;
7460 u32 enables = 0;
7461
7462 if (bnxt_hwrm_func_cfg_short_req_init(bp, &req))
7463 return NULL;
7464
7465 req->fid = cpu_to_le16(0xffff);
7466 enables |= hwr->tx ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7467 req->num_tx_rings = cpu_to_le16(hwr->tx);
7468 if (BNXT_NEW_RM(bp)) {
7469 enables |= hwr->rx ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
7470 enables |= hwr->stat ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7471 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7472 enables |= hwr->cp ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
7473 enables |= hwr->cp_p5 ?
7474 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7475 } else {
7476 enables |= hwr->cp ?
7477 FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7478 enables |= hwr->grp ?
7479 FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7480 }
7481 enables |= hwr->vnic ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
7482 enables |= hwr->rss_ctx ? FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS :
7483 0;
7484 req->num_rx_rings = cpu_to_le16(hwr->rx);
7485 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx);
7486 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7487 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5);
7488 req->num_msix = cpu_to_le16(hwr->cp);
7489 } else {
7490 req->num_cmpl_rings = cpu_to_le16(hwr->cp);
7491 req->num_hw_ring_grps = cpu_to_le16(hwr->grp);
7492 }
7493 req->num_stat_ctxs = cpu_to_le16(hwr->stat);
7494 req->num_vnics = cpu_to_le16(hwr->vnic);
7495 }
7496 req->enables = cpu_to_le32(enables);
7497 return req;
7498 }
7499
7500 static struct hwrm_func_vf_cfg_input *
7501 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7502 {
7503 struct hwrm_func_vf_cfg_input *req;
7504 u32 enables = 0;
7505
7506 if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
7507 return NULL;
7508
7509 enables |= hwr->tx ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
7510 enables |= hwr->rx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
7511 FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7512 enables |= hwr->stat ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
7513 enables |= hwr->rss_ctx ? FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
7514 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7515 enables |= hwr->cp_p5 ?
7516 FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7517 } else {
7518 enables |= hwr->cp ? FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
7519 enables |= hwr->grp ?
7520 FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
7521 }
7522 enables |= hwr->vnic ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
7523 enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
7524
7525 req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
7526 req->num_tx_rings = cpu_to_le16(hwr->tx);
7527 req->num_rx_rings = cpu_to_le16(hwr->rx);
7528 req->num_rsscos_ctxs = cpu_to_le16(hwr->rss_ctx);
7529 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7530 req->num_cmpl_rings = cpu_to_le16(hwr->cp_p5);
7531 } else {
7532 req->num_cmpl_rings = cpu_to_le16(hwr->cp);
7533 req->num_hw_ring_grps = cpu_to_le16(hwr->grp);
7534 }
7535 req->num_stat_ctxs = cpu_to_le16(hwr->stat);
7536 req->num_vnics = cpu_to_le16(hwr->vnic);
7537
7538 req->enables = cpu_to_le32(enables);
7539 return req;
7540 }
7541
7542 static int
7543 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7544 {
7545 struct hwrm_func_cfg_input *req;
7546 int rc;
7547
7548 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr);
7549 if (!req)
7550 return -ENOMEM;
7551
7552 if (!req->enables) {
7553 hwrm_req_drop(bp, req);
7554 return 0;
7555 }
7556
7557 rc = hwrm_req_send(bp, req);
7558 if (rc)
7559 return rc;
7560
7561 if (bp->hwrm_spec_code < 0x10601)
7562 bp->hw_resc.resv_tx_rings = hwr->tx;
7563
7564 return bnxt_hwrm_get_rings(bp);
7565 }
7566
7567 static int
7568 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7569 {
7570 struct hwrm_func_vf_cfg_input *req;
7571 int rc;
7572
7573 if (!BNXT_NEW_RM(bp)) {
7574 bp->hw_resc.resv_tx_rings = hwr->tx;
7575 return 0;
7576 }
7577
7578 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr);
7579 if (!req)
7580 return -ENOMEM;
7581
7582 rc = hwrm_req_send(bp, req);
7583 if (rc)
7584 return rc;
7585
7586 return bnxt_hwrm_get_rings(bp);
7587 }
7588
7589 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7590 {
7591 if (BNXT_PF(bp))
7592 return bnxt_hwrm_reserve_pf_rings(bp, hwr);
7593 else
7594 return bnxt_hwrm_reserve_vf_rings(bp, hwr);
7595 }
7596
7597 int bnxt_nq_rings_in_use(struct bnxt *bp)
7598 {
7599 return bp->cp_nr_rings + bnxt_get_ulp_msix_num(bp);
7600 }
7601
7602 static int bnxt_cp_rings_in_use(struct bnxt *bp)
7603 {
7604 int cp;
7605
7606 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7607 return bnxt_nq_rings_in_use(bp);
7608
7609 cp = bp->tx_nr_rings + bp->rx_nr_rings;
7610 return cp;
7611 }
7612
7613 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
7614 {
7615 return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp);
7616 }
7617
7618 static int bnxt_get_total_rss_ctxs(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7619 {
7620 if (!hwr->grp)
7621 return 0;
7622 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
7623 int rss_ctx = bnxt_get_nr_rss_ctxs(bp, hwr->grp);
7624
7625 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
7626 rss_ctx *= hwr->vnic;
7627 return rss_ctx;
7628 }
7629 if (BNXT_VF(bp))
7630 return BNXT_VF_MAX_RSS_CTX;
7631 if (!(bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP) && bnxt_rfs_supported(bp))
7632 return hwr->grp + 1;
7633 return 1;
7634 }
7635
7636 /* Check if a default RSS map needs to be setup. This function is only
7637 * used on older firmware that does not require reserving RX rings.
7638 */
7639 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
7640 {
7641 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7642
7643 /* The RSS map is valid for RX rings set to resv_rx_rings */
7644 if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
7645 hw_resc->resv_rx_rings = bp->rx_nr_rings;
7646 if (!netif_is_rxfh_configured(bp->dev))
7647 bnxt_set_dflt_rss_indir_tbl(bp, NULL);
7648 }
7649 }
7650
7651 static int bnxt_get_total_vnics(struct bnxt *bp, int rx_rings)
7652 {
7653 if (bp->flags & BNXT_FLAG_RFS) {
7654 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
7655 return 2 + bp->num_rss_ctx;
7656 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7657 return rx_rings + 1;
7658 }
7659 return 1;
7660 }
7661
7662 static bool bnxt_need_reserve_rings(struct bnxt *bp)
7663 {
7664 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7665 int cp = bnxt_cp_rings_in_use(bp);
7666 int nq = bnxt_nq_rings_in_use(bp);
7667 int rx = bp->rx_nr_rings, stat;
7668 int vnic, grp = rx;
7669
7670 /* Old firmware does not need RX ring reservations but we still
7671 * need to setup a default RSS map when needed. With new firmware
7672 * we go through RX ring reservations first and then set up the
7673 * RSS map for the successfully reserved RX rings when needed.
7674 */
7675 if (!BNXT_NEW_RM(bp))
7676 bnxt_check_rss_tbl_no_rmgr(bp);
7677
7678 if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
7679 bp->hwrm_spec_code >= 0x10601)
7680 return true;
7681
7682 if (!BNXT_NEW_RM(bp))
7683 return false;
7684
7685 vnic = bnxt_get_total_vnics(bp, rx);
7686
7687 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7688 rx <<= 1;
7689 stat = bnxt_get_func_stat_ctxs(bp);
7690 if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
7691 hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
7692 (hw_resc->resv_hw_ring_grps != grp &&
7693 !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)))
7694 return true;
7695 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && BNXT_PF(bp) &&
7696 hw_resc->resv_irqs != nq)
7697 return true;
7698 return false;
7699 }
7700
7701 static void bnxt_copy_reserved_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7702 {
7703 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7704
7705 hwr->tx = hw_resc->resv_tx_rings;
7706 if (BNXT_NEW_RM(bp)) {
7707 hwr->rx = hw_resc->resv_rx_rings;
7708 hwr->cp = hw_resc->resv_irqs;
7709 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7710 hwr->cp_p5 = hw_resc->resv_cp_rings;
7711 hwr->grp = hw_resc->resv_hw_ring_grps;
7712 hwr->vnic = hw_resc->resv_vnics;
7713 hwr->stat = hw_resc->resv_stat_ctxs;
7714 hwr->rss_ctx = hw_resc->resv_rsscos_ctxs;
7715 }
7716 }
7717
7718 static bool bnxt_rings_ok(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7719 {
7720 return hwr->tx && hwr->rx && hwr->cp && hwr->grp && hwr->vnic &&
7721 hwr->stat && (hwr->cp_p5 || !(bp->flags & BNXT_FLAG_CHIP_P5_PLUS));
7722 }
7723
7724 static int bnxt_get_avail_msix(struct bnxt *bp, int num);
7725
7726 static int __bnxt_reserve_rings(struct bnxt *bp)
7727 {
7728 struct bnxt_hw_rings hwr = {0};
7729 int rx_rings, old_rx_rings, rc;
7730 int cp = bp->cp_nr_rings;
7731 int ulp_msix = 0;
7732 bool sh = false;
7733 int tx_cp;
7734
7735 if (!bnxt_need_reserve_rings(bp))
7736 return 0;
7737
7738 if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) {
7739 ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want);
7740 if (!ulp_msix)
7741 bnxt_set_ulp_stat_ctxs(bp, 0);
7742
7743 if (ulp_msix > bp->ulp_num_msix_want)
7744 ulp_msix = bp->ulp_num_msix_want;
7745 hwr.cp = cp + ulp_msix;
7746 } else {
7747 hwr.cp = bnxt_nq_rings_in_use(bp);
7748 }
7749
7750 hwr.tx = bp->tx_nr_rings;
7751 hwr.rx = bp->rx_nr_rings;
7752 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
7753 sh = true;
7754 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7755 hwr.cp_p5 = hwr.rx + hwr.tx;
7756
7757 hwr.vnic = bnxt_get_total_vnics(bp, hwr.rx);
7758
7759 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7760 hwr.rx <<= 1;
7761 hwr.grp = bp->rx_nr_rings;
7762 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
7763 hwr.stat = bnxt_get_func_stat_ctxs(bp);
7764 old_rx_rings = bp->hw_resc.resv_rx_rings;
7765
7766 rc = bnxt_hwrm_reserve_rings(bp, &hwr);
7767 if (rc)
7768 return rc;
7769
7770 bnxt_copy_reserved_rings(bp, &hwr);
7771
7772 rx_rings = hwr.rx;
7773 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7774 if (hwr.rx >= 2) {
7775 rx_rings = hwr.rx >> 1;
7776 } else {
7777 if (netif_running(bp->dev))
7778 return -ENOMEM;
7779
7780 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
7781 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
7782 bp->dev->hw_features &= ~NETIF_F_LRO;
7783 bp->dev->features &= ~NETIF_F_LRO;
7784 bnxt_set_ring_params(bp);
7785 }
7786 }
7787 rx_rings = min_t(int, rx_rings, hwr.grp);
7788 hwr.cp = min_t(int, hwr.cp, bp->cp_nr_rings);
7789 if (hwr.stat > bnxt_get_ulp_stat_ctxs(bp))
7790 hwr.stat -= bnxt_get_ulp_stat_ctxs(bp);
7791 hwr.cp = min_t(int, hwr.cp, hwr.stat);
7792 rc = bnxt_trim_rings(bp, &rx_rings, &hwr.tx, hwr.cp, sh);
7793 if (bp->flags & BNXT_FLAG_AGG_RINGS)
7794 hwr.rx = rx_rings << 1;
7795 tx_cp = bnxt_num_tx_to_cp(bp, hwr.tx);
7796 hwr.cp = sh ? max_t(int, tx_cp, rx_rings) : tx_cp + rx_rings;
7797 bp->tx_nr_rings = hwr.tx;
7798
7799 /* If we cannot reserve all the RX rings, reset the RSS map only
7800 * if absolutely necessary
7801 */
7802 if (rx_rings != bp->rx_nr_rings) {
7803 netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
7804 rx_rings, bp->rx_nr_rings);
7805 if (netif_is_rxfh_configured(bp->dev) &&
7806 (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
7807 bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
7808 bnxt_get_max_rss_ring(bp) >= rx_rings)) {
7809 netdev_warn(bp->dev, "RSS table entries reverting to default\n");
7810 bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
7811 }
7812 }
7813 bp->rx_nr_rings = rx_rings;
7814 bp->cp_nr_rings = hwr.cp;
7815
7816 if (!bnxt_rings_ok(bp, &hwr))
7817 return -ENOMEM;
7818
7819 if (old_rx_rings != bp->hw_resc.resv_rx_rings &&
7820 !netif_is_rxfh_configured(bp->dev))
7821 bnxt_set_dflt_rss_indir_tbl(bp, NULL);
7822
7823 if (!bnxt_ulp_registered(bp->edev) && BNXT_NEW_RM(bp)) {
7824 int resv_msix, resv_ctx, ulp_ctxs;
7825 struct bnxt_hw_resc *hw_resc;
7826
7827 hw_resc = &bp->hw_resc;
7828 resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings;
7829 ulp_msix = min_t(int, resv_msix, ulp_msix);
7830 bnxt_set_ulp_msix_num(bp, ulp_msix);
7831 resv_ctx = hw_resc->resv_stat_ctxs - bp->cp_nr_rings;
7832 ulp_ctxs = min(resv_ctx, bnxt_get_ulp_stat_ctxs(bp));
7833 bnxt_set_ulp_stat_ctxs(bp, ulp_ctxs);
7834 }
7835
7836 return rc;
7837 }
7838
7839 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7840 {
7841 struct hwrm_func_vf_cfg_input *req;
7842 u32 flags;
7843
7844 if (!BNXT_NEW_RM(bp))
7845 return 0;
7846
7847 req = __bnxt_hwrm_reserve_vf_rings(bp, hwr);
7848 flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
7849 FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7850 FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7851 FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7852 FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
7853 FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
7854 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
7855 flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7856
7857 req->flags = cpu_to_le32(flags);
7858 return hwrm_req_send_silent(bp, req);
7859 }
7860
7861 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7862 {
7863 struct hwrm_func_cfg_input *req;
7864 u32 flags;
7865
7866 req = __bnxt_hwrm_reserve_pf_rings(bp, hwr);
7867 flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
7868 if (BNXT_NEW_RM(bp)) {
7869 flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
7870 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
7871 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
7872 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
7873 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
7874 flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
7875 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
7876 else
7877 flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
7878 }
7879
7880 req->flags = cpu_to_le32(flags);
7881 return hwrm_req_send_silent(bp, req);
7882 }
7883
7884 static int bnxt_hwrm_check_rings(struct bnxt *bp, struct bnxt_hw_rings *hwr)
7885 {
7886 if (bp->hwrm_spec_code < 0x10801)
7887 return 0;
7888
7889 if (BNXT_PF(bp))
7890 return bnxt_hwrm_check_pf_rings(bp, hwr);
7891
7892 return bnxt_hwrm_check_vf_rings(bp, hwr);
7893 }
7894
7895 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
7896 {
7897 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7898 struct hwrm_ring_aggint_qcaps_output *resp;
7899 struct hwrm_ring_aggint_qcaps_input *req;
7900 int rc;
7901
7902 coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
7903 coal_cap->num_cmpl_dma_aggr_max = 63;
7904 coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
7905 coal_cap->cmpl_aggr_dma_tmr_max = 65535;
7906 coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
7907 coal_cap->int_lat_tmr_min_max = 65535;
7908 coal_cap->int_lat_tmr_max_max = 65535;
7909 coal_cap->num_cmpl_aggr_int_max = 65535;
7910 coal_cap->timer_units = 80;
7911
7912 if (bp->hwrm_spec_code < 0x10902)
7913 return;
7914
7915 if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
7916 return;
7917
7918 resp = hwrm_req_hold(bp, req);
7919 rc = hwrm_req_send_silent(bp, req);
7920 if (!rc) {
7921 coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
7922 coal_cap->nq_params = le32_to_cpu(resp->nq_params);
7923 coal_cap->num_cmpl_dma_aggr_max =
7924 le16_to_cpu(resp->num_cmpl_dma_aggr_max);
7925 coal_cap->num_cmpl_dma_aggr_during_int_max =
7926 le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
7927 coal_cap->cmpl_aggr_dma_tmr_max =
7928 le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
7929 coal_cap->cmpl_aggr_dma_tmr_during_int_max =
7930 le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
7931 coal_cap->int_lat_tmr_min_max =
7932 le16_to_cpu(resp->int_lat_tmr_min_max);
7933 coal_cap->int_lat_tmr_max_max =
7934 le16_to_cpu(resp->int_lat_tmr_max_max);
7935 coal_cap->num_cmpl_aggr_int_max =
7936 le16_to_cpu(resp->num_cmpl_aggr_int_max);
7937 coal_cap->timer_units = le16_to_cpu(resp->timer_units);
7938 }
7939 hwrm_req_drop(bp, req);
7940 }
7941
7942 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
7943 {
7944 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7945
7946 return usec * 1000 / coal_cap->timer_units;
7947 }
7948
7949 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
7950 struct bnxt_coal *hw_coal,
7951 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
7952 {
7953 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
7954 u16 val, tmr, max, flags = hw_coal->flags;
7955 u32 cmpl_params = coal_cap->cmpl_params;
7956
7957 max = hw_coal->bufs_per_record * 128;
7958 if (hw_coal->budget)
7959 max = hw_coal->bufs_per_record * hw_coal->budget;
7960 max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
7961
7962 val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
7963 req->num_cmpl_aggr_int = cpu_to_le16(val);
7964
7965 val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
7966 req->num_cmpl_dma_aggr = cpu_to_le16(val);
7967
7968 val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
7969 coal_cap->num_cmpl_dma_aggr_during_int_max);
7970 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
7971
7972 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
7973 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
7974 req->int_lat_tmr_max = cpu_to_le16(tmr);
7975
7976 /* min timer set to 1/2 of interrupt timer */
7977 if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
7978 val = tmr / 2;
7979 val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
7980 req->int_lat_tmr_min = cpu_to_le16(val);
7981 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
7982 }
7983
7984 /* buf timer set to 1/4 of interrupt timer */
7985 val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
7986 req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
7987
7988 if (cmpl_params &
7989 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
7990 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
7991 val = clamp_t(u16, tmr, 1,
7992 coal_cap->cmpl_aggr_dma_tmr_during_int_max);
7993 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
7994 req->enables |=
7995 cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
7996 }
7997
7998 if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
7999 hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
8000 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
8001 req->flags = cpu_to_le16(flags);
8002 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
8003 }
8004
8005 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
8006 struct bnxt_coal *hw_coal)
8007 {
8008 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
8009 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8010 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
8011 u32 nq_params = coal_cap->nq_params;
8012 u16 tmr;
8013 int rc;
8014
8015 if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
8016 return 0;
8017
8018 rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
8019 if (rc)
8020 return rc;
8021
8022 req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
8023 req->flags =
8024 cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
8025
8026 tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
8027 tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
8028 req->int_lat_tmr_min = cpu_to_le16(tmr);
8029 req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
8030 return hwrm_req_send(bp, req);
8031 }
8032
8033 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
8034 {
8035 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
8036 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8037 struct bnxt_coal coal;
8038 int rc;
8039
8040 /* Tick values in micro seconds.
8041 * 1 coal_buf x bufs_per_record = 1 completion record.
8042 */
8043 memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
8044
8045 coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
8046 coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
8047
8048 if (!bnapi->rx_ring)
8049 return -ENODEV;
8050
8051 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
8052 if (rc)
8053 return rc;
8054
8055 bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
8056
8057 req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
8058
8059 return hwrm_req_send(bp, req_rx);
8060 }
8061
8062 static int
8063 bnxt_hwrm_set_rx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
8064 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
8065 {
8066 u16 ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
8067
8068 req->ring_id = cpu_to_le16(ring_id);
8069 return hwrm_req_send(bp, req);
8070 }
8071
8072 static int
8073 bnxt_hwrm_set_tx_coal(struct bnxt *bp, struct bnxt_napi *bnapi,
8074 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
8075 {
8076 struct bnxt_tx_ring_info *txr;
8077 int i, rc;
8078
8079 bnxt_for_each_napi_tx(i, bnapi, txr) {
8080 u16 ring_id;
8081
8082 ring_id = bnxt_cp_ring_for_tx(bp, txr);
8083 req->ring_id = cpu_to_le16(ring_id);
8084 rc = hwrm_req_send(bp, req);
8085 if (rc)
8086 return rc;
8087 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
8088 return 0;
8089 }
8090 return 0;
8091 }
8092
8093 int bnxt_hwrm_set_coal(struct bnxt *bp)
8094 {
8095 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx;
8096 int i, rc;
8097
8098 rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
8099 if (rc)
8100 return rc;
8101
8102 rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
8103 if (rc) {
8104 hwrm_req_drop(bp, req_rx);
8105 return rc;
8106 }
8107
8108 bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
8109 bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
8110
8111 hwrm_req_hold(bp, req_rx);
8112 hwrm_req_hold(bp, req_tx);
8113 for (i = 0; i < bp->cp_nr_rings; i++) {
8114 struct bnxt_napi *bnapi = bp->bnapi[i];
8115 struct bnxt_coal *hw_coal;
8116
8117 if (!bnapi->rx_ring)
8118 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
8119 else
8120 rc = bnxt_hwrm_set_rx_coal(bp, bnapi, req_rx);
8121 if (rc)
8122 break;
8123
8124 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
8125 continue;
8126
8127 if (bnapi->rx_ring && bnapi->tx_ring[0]) {
8128 rc = bnxt_hwrm_set_tx_coal(bp, bnapi, req_tx);
8129 if (rc)
8130 break;
8131 }
8132 if (bnapi->rx_ring)
8133 hw_coal = &bp->rx_coal;
8134 else
8135 hw_coal = &bp->tx_coal;
8136 __bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
8137 }
8138 hwrm_req_drop(bp, req_rx);
8139 hwrm_req_drop(bp, req_tx);
8140 return rc;
8141 }
8142
8143 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
8144 {
8145 struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
8146 struct hwrm_stat_ctx_free_input *req;
8147 int i;
8148
8149 if (!bp->bnapi)
8150 return;
8151
8152 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8153 return;
8154
8155 if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
8156 return;
8157 if (BNXT_FW_MAJ(bp) <= 20) {
8158 if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
8159 hwrm_req_drop(bp, req);
8160 return;
8161 }
8162 hwrm_req_hold(bp, req0);
8163 }
8164 hwrm_req_hold(bp, req);
8165 for (i = 0; i < bp->cp_nr_rings; i++) {
8166 struct bnxt_napi *bnapi = bp->bnapi[i];
8167 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8168
8169 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
8170 req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
8171 if (req0) {
8172 req0->stat_ctx_id = req->stat_ctx_id;
8173 hwrm_req_send(bp, req0);
8174 }
8175 hwrm_req_send(bp, req);
8176
8177 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
8178 }
8179 }
8180 hwrm_req_drop(bp, req);
8181 if (req0)
8182 hwrm_req_drop(bp, req0);
8183 }
8184
8185 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
8186 {
8187 struct hwrm_stat_ctx_alloc_output *resp;
8188 struct hwrm_stat_ctx_alloc_input *req;
8189 int rc, i;
8190
8191 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8192 return 0;
8193
8194 rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
8195 if (rc)
8196 return rc;
8197
8198 req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
8199 req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
8200
8201 resp = hwrm_req_hold(bp, req);
8202 for (i = 0; i < bp->cp_nr_rings; i++) {
8203 struct bnxt_napi *bnapi = bp->bnapi[i];
8204 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
8205
8206 req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
8207
8208 rc = hwrm_req_send(bp, req);
8209 if (rc)
8210 break;
8211
8212 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
8213
8214 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
8215 }
8216 hwrm_req_drop(bp, req);
8217 return rc;
8218 }
8219
8220 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
8221 {
8222 struct hwrm_func_qcfg_output *resp;
8223 struct hwrm_func_qcfg_input *req;
8224 u16 flags;
8225 int rc;
8226
8227 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
8228 if (rc)
8229 return rc;
8230
8231 req->fid = cpu_to_le16(0xffff);
8232 resp = hwrm_req_hold(bp, req);
8233 rc = hwrm_req_send(bp, req);
8234 if (rc)
8235 goto func_qcfg_exit;
8236
8237 #ifdef CONFIG_BNXT_SRIOV
8238 if (BNXT_VF(bp)) {
8239 struct bnxt_vf_info *vf = &bp->vf;
8240
8241 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
8242 } else {
8243 bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
8244 }
8245 #endif
8246 flags = le16_to_cpu(resp->flags);
8247 if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
8248 FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
8249 bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
8250 if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
8251 bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
8252 }
8253 if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
8254 bp->flags |= BNXT_FLAG_MULTI_HOST;
8255
8256 if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
8257 bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
8258
8259 if (flags & FUNC_QCFG_RESP_FLAGS_ENABLE_RDMA_SRIOV)
8260 bp->fw_cap |= BNXT_FW_CAP_ENABLE_RDMA_SRIOV;
8261
8262 switch (resp->port_partition_type) {
8263 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
8264 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
8265 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
8266 bp->port_partition_type = resp->port_partition_type;
8267 break;
8268 }
8269 if (bp->hwrm_spec_code < 0x10707 ||
8270 resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
8271 bp->br_mode = BRIDGE_MODE_VEB;
8272 else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
8273 bp->br_mode = BRIDGE_MODE_VEPA;
8274 else
8275 bp->br_mode = BRIDGE_MODE_UNDEF;
8276
8277 bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
8278 if (!bp->max_mtu)
8279 bp->max_mtu = BNXT_MAX_MTU;
8280
8281 if (bp->db_size)
8282 goto func_qcfg_exit;
8283
8284 bp->db_offset = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024;
8285 if (BNXT_CHIP_P5(bp)) {
8286 if (BNXT_PF(bp))
8287 bp->db_offset = DB_PF_OFFSET_P5;
8288 else
8289 bp->db_offset = DB_VF_OFFSET_P5;
8290 }
8291 bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
8292 1024);
8293 if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
8294 bp->db_size <= bp->db_offset)
8295 bp->db_size = pci_resource_len(bp->pdev, 2);
8296
8297 func_qcfg_exit:
8298 hwrm_req_drop(bp, req);
8299 return rc;
8300 }
8301
8302 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm,
8303 u8 init_val, u8 init_offset,
8304 bool init_mask_set)
8305 {
8306 ctxm->init_value = init_val;
8307 ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET;
8308 if (init_mask_set)
8309 ctxm->init_offset = init_offset * 4;
8310 else
8311 ctxm->init_value = 0;
8312 }
8313
8314 static int bnxt_alloc_all_ctx_pg_info(struct bnxt *bp, int ctx_max)
8315 {
8316 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8317 u16 type;
8318
8319 for (type = 0; type < ctx_max; type++) {
8320 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8321 int n = 1;
8322
8323 if (!ctxm->max_entries)
8324 continue;
8325
8326 if (ctxm->instance_bmap)
8327 n = hweight32(ctxm->instance_bmap);
8328 ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_KERNEL);
8329 if (!ctxm->pg_info)
8330 return -ENOMEM;
8331 }
8332 return 0;
8333 }
8334
8335 static void bnxt_free_one_ctx_mem(struct bnxt *bp,
8336 struct bnxt_ctx_mem_type *ctxm, bool force);
8337
8338 #define BNXT_CTX_INIT_VALID(flags) \
8339 (!!((flags) & \
8340 FUNC_BACKING_STORE_QCAPS_V2_RESP_FLAGS_ENABLE_CTX_KIND_INIT))
8341
8342 static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt *bp)
8343 {
8344 struct hwrm_func_backing_store_qcaps_v2_output *resp;
8345 struct hwrm_func_backing_store_qcaps_v2_input *req;
8346 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8347 u16 type;
8348 int rc;
8349
8350 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS_V2);
8351 if (rc)
8352 return rc;
8353
8354 if (!ctx) {
8355 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
8356 if (!ctx)
8357 return -ENOMEM;
8358 bp->ctx = ctx;
8359 }
8360
8361 resp = hwrm_req_hold(bp, req);
8362
8363 for (type = 0; type < BNXT_CTX_V2_MAX; ) {
8364 struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type];
8365 u8 init_val, init_off, i;
8366 u32 max_entries;
8367 u16 entry_size;
8368 __le32 *p;
8369 u32 flags;
8370
8371 req->type = cpu_to_le16(type);
8372 rc = hwrm_req_send(bp, req);
8373 if (rc)
8374 goto ctx_done;
8375 flags = le32_to_cpu(resp->flags);
8376 type = le16_to_cpu(resp->next_valid_type);
8377 if (!(flags & BNXT_CTX_MEM_TYPE_VALID)) {
8378 bnxt_free_one_ctx_mem(bp, ctxm, true);
8379 continue;
8380 }
8381 entry_size = le16_to_cpu(resp->entry_size);
8382 max_entries = le32_to_cpu(resp->max_num_entries);
8383 if (ctxm->mem_valid) {
8384 if (!(flags & BNXT_CTX_MEM_PERSIST) ||
8385 ctxm->entry_size != entry_size ||
8386 ctxm->max_entries != max_entries)
8387 bnxt_free_one_ctx_mem(bp, ctxm, true);
8388 else
8389 continue;
8390 }
8391 ctxm->type = le16_to_cpu(resp->type);
8392 ctxm->entry_size = entry_size;
8393 ctxm->flags = flags;
8394 ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map);
8395 ctxm->entry_multiple = resp->entry_multiple;
8396 ctxm->max_entries = max_entries;
8397 ctxm->min_entries = le32_to_cpu(resp->min_num_entries);
8398 init_val = resp->ctx_init_value;
8399 init_off = resp->ctx_init_offset;
8400 bnxt_init_ctx_initializer(ctxm, init_val, init_off,
8401 BNXT_CTX_INIT_VALID(flags));
8402 ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt,
8403 BNXT_MAX_SPLIT_ENTRY);
8404 for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt;
8405 i++, p++)
8406 ctxm->split[i] = le32_to_cpu(*p);
8407 }
8408 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_V2_MAX);
8409
8410 ctx_done:
8411 hwrm_req_drop(bp, req);
8412 return rc;
8413 }
8414
8415 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
8416 {
8417 struct hwrm_func_backing_store_qcaps_output *resp;
8418 struct hwrm_func_backing_store_qcaps_input *req;
8419 int rc;
8420
8421 if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) ||
8422 (bp->ctx && bp->ctx->flags & BNXT_CTX_FLAG_INITED))
8423 return 0;
8424
8425 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
8426 return bnxt_hwrm_func_backing_store_qcaps_v2(bp);
8427
8428 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
8429 if (rc)
8430 return rc;
8431
8432 resp = hwrm_req_hold(bp, req);
8433 rc = hwrm_req_send_silent(bp, req);
8434 if (!rc) {
8435 struct bnxt_ctx_mem_type *ctxm;
8436 struct bnxt_ctx_mem_info *ctx;
8437 u8 init_val, init_idx = 0;
8438 u16 init_mask;
8439
8440 ctx = bp->ctx;
8441 if (!ctx) {
8442 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
8443 if (!ctx) {
8444 rc = -ENOMEM;
8445 goto ctx_err;
8446 }
8447 bp->ctx = ctx;
8448 }
8449 init_val = resp->ctx_kind_initializer;
8450 init_mask = le16_to_cpu(resp->ctx_init_mask);
8451
8452 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8453 ctxm->max_entries = le32_to_cpu(resp->qp_max_entries);
8454 ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
8455 ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
8456 ctxm->qp_fast_qpmd_entries = le16_to_cpu(resp->fast_qpmd_qp_num_entries);
8457 ctxm->entry_size = le16_to_cpu(resp->qp_entry_size);
8458 bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset,
8459 (init_mask & (1 << init_idx++)) != 0);
8460
8461 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8462 ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
8463 ctxm->max_entries = le32_to_cpu(resp->srq_max_entries);
8464 ctxm->entry_size = le16_to_cpu(resp->srq_entry_size);
8465 bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset,
8466 (init_mask & (1 << init_idx++)) != 0);
8467
8468 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8469 ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
8470 ctxm->max_entries = le32_to_cpu(resp->cq_max_entries);
8471 ctxm->entry_size = le16_to_cpu(resp->cq_entry_size);
8472 bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset,
8473 (init_mask & (1 << init_idx++)) != 0);
8474
8475 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8476 ctxm->vnic_entries = le16_to_cpu(resp->vnic_max_vnic_entries);
8477 ctxm->max_entries = ctxm->vnic_entries +
8478 le16_to_cpu(resp->vnic_max_ring_table_entries);
8479 ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size);
8480 bnxt_init_ctx_initializer(ctxm, init_val,
8481 resp->vnic_init_offset,
8482 (init_mask & (1 << init_idx++)) != 0);
8483
8484 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8485 ctxm->max_entries = le32_to_cpu(resp->stat_max_entries);
8486 ctxm->entry_size = le16_to_cpu(resp->stat_entry_size);
8487 bnxt_init_ctx_initializer(ctxm, init_val,
8488 resp->stat_init_offset,
8489 (init_mask & (1 << init_idx++)) != 0);
8490
8491 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8492 ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size);
8493 ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring);
8494 ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring);
8495 ctxm->entry_multiple = resp->tqm_entries_multiple;
8496 if (!ctxm->entry_multiple)
8497 ctxm->entry_multiple = 1;
8498
8499 memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm));
8500
8501 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8502 ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries);
8503 ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size);
8504 ctxm->mrav_num_entries_units =
8505 le16_to_cpu(resp->mrav_num_entries_units);
8506 bnxt_init_ctx_initializer(ctxm, init_val,
8507 resp->mrav_init_offset,
8508 (init_mask & (1 << init_idx++)) != 0);
8509
8510 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8511 ctxm->entry_size = le16_to_cpu(resp->tim_entry_size);
8512 ctxm->max_entries = le32_to_cpu(resp->tim_max_entries);
8513
8514 ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
8515 if (!ctx->tqm_fp_rings_count)
8516 ctx->tqm_fp_rings_count = bp->max_q;
8517 else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
8518 ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
8519
8520 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
8521 memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm));
8522 ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1;
8523
8524 rc = bnxt_alloc_all_ctx_pg_info(bp, BNXT_CTX_MAX);
8525 } else {
8526 rc = 0;
8527 }
8528 ctx_err:
8529 hwrm_req_drop(bp, req);
8530 return rc;
8531 }
8532
8533 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
8534 __le64 *pg_dir)
8535 {
8536 if (!rmem->nr_pages)
8537 return;
8538
8539 BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
8540 if (rmem->depth >= 1) {
8541 if (rmem->depth == 2)
8542 *pg_attr |= 2;
8543 else
8544 *pg_attr |= 1;
8545 *pg_dir = cpu_to_le64(rmem->pg_tbl_map);
8546 } else {
8547 *pg_dir = cpu_to_le64(rmem->dma_arr[0]);
8548 }
8549 }
8550
8551 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES \
8552 (FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP | \
8553 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ | \
8554 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ | \
8555 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC | \
8556 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
8557
8558 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
8559 {
8560 struct hwrm_func_backing_store_cfg_input *req;
8561 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8562 struct bnxt_ctx_pg_info *ctx_pg;
8563 struct bnxt_ctx_mem_type *ctxm;
8564 void **__req = (void **)&req;
8565 u32 req_len = sizeof(*req);
8566 __le32 *num_entries;
8567 __le64 *pg_dir;
8568 u32 flags = 0;
8569 u8 *pg_attr;
8570 u32 ena;
8571 int rc;
8572 int i;
8573
8574 if (!ctx)
8575 return 0;
8576
8577 if (req_len > bp->hwrm_max_ext_req_len)
8578 req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
8579 rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
8580 if (rc)
8581 return rc;
8582
8583 req->enables = cpu_to_le32(enables);
8584 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
8585 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
8586 ctx_pg = ctxm->pg_info;
8587 req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
8588 req->qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries);
8589 req->qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries);
8590 req->qp_entry_size = cpu_to_le16(ctxm->entry_size);
8591 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8592 &req->qpc_pg_size_qpc_lvl,
8593 &req->qpc_page_dir);
8594
8595 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD)
8596 req->qp_num_fast_qpmd_entries = cpu_to_le16(ctxm->qp_fast_qpmd_entries);
8597 }
8598 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
8599 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
8600 ctx_pg = ctxm->pg_info;
8601 req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
8602 req->srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries);
8603 req->srq_entry_size = cpu_to_le16(ctxm->entry_size);
8604 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8605 &req->srq_pg_size_srq_lvl,
8606 &req->srq_page_dir);
8607 }
8608 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
8609 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
8610 ctx_pg = ctxm->pg_info;
8611 req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
8612 req->cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries);
8613 req->cq_entry_size = cpu_to_le16(ctxm->entry_size);
8614 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8615 &req->cq_pg_size_cq_lvl,
8616 &req->cq_page_dir);
8617 }
8618 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
8619 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
8620 ctx_pg = ctxm->pg_info;
8621 req->vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries);
8622 req->vnic_num_ring_table_entries =
8623 cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries);
8624 req->vnic_entry_size = cpu_to_le16(ctxm->entry_size);
8625 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8626 &req->vnic_pg_size_vnic_lvl,
8627 &req->vnic_page_dir);
8628 }
8629 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
8630 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
8631 ctx_pg = ctxm->pg_info;
8632 req->stat_num_entries = cpu_to_le32(ctxm->max_entries);
8633 req->stat_entry_size = cpu_to_le16(ctxm->entry_size);
8634 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8635 &req->stat_pg_size_stat_lvl,
8636 &req->stat_page_dir);
8637 }
8638 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
8639 u32 units;
8640
8641 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
8642 ctx_pg = ctxm->pg_info;
8643 req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
8644 units = ctxm->mrav_num_entries_units;
8645 if (units) {
8646 u32 num_mr, num_ah = ctxm->mrav_av_entries;
8647 u32 entries;
8648
8649 num_mr = ctx_pg->entries - num_ah;
8650 entries = ((num_mr / units) << 16) | (num_ah / units);
8651 req->mrav_num_entries = cpu_to_le32(entries);
8652 flags |= FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
8653 }
8654 req->mrav_entry_size = cpu_to_le16(ctxm->entry_size);
8655 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8656 &req->mrav_pg_size_mrav_lvl,
8657 &req->mrav_page_dir);
8658 }
8659 if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
8660 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
8661 ctx_pg = ctxm->pg_info;
8662 req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
8663 req->tim_entry_size = cpu_to_le16(ctxm->entry_size);
8664 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8665 &req->tim_pg_size_tim_lvl,
8666 &req->tim_page_dir);
8667 }
8668 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
8669 for (i = 0, num_entries = &req->tqm_sp_num_entries,
8670 pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
8671 pg_dir = &req->tqm_sp_page_dir,
8672 ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP,
8673 ctx_pg = ctxm->pg_info;
8674 i < BNXT_MAX_TQM_RINGS;
8675 ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i],
8676 i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
8677 if (!(enables & ena))
8678 continue;
8679
8680 req->tqm_entry_size = cpu_to_le16(ctxm->entry_size);
8681 *num_entries = cpu_to_le32(ctx_pg->entries);
8682 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
8683 }
8684 req->flags = cpu_to_le32(flags);
8685 return hwrm_req_send(bp, req);
8686 }
8687
8688 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
8689 struct bnxt_ctx_pg_info *ctx_pg)
8690 {
8691 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8692
8693 rmem->page_size = BNXT_PAGE_SIZE;
8694 rmem->pg_arr = ctx_pg->ctx_pg_arr;
8695 rmem->dma_arr = ctx_pg->ctx_dma_arr;
8696 rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
8697 if (rmem->depth >= 1)
8698 rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
8699 return bnxt_alloc_ring(bp, rmem);
8700 }
8701
8702 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
8703 struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
8704 u8 depth, struct bnxt_ctx_mem_type *ctxm)
8705 {
8706 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8707 int rc;
8708
8709 if (!mem_size)
8710 return -EINVAL;
8711
8712 ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8713 if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
8714 ctx_pg->nr_pages = 0;
8715 return -EINVAL;
8716 }
8717 if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
8718 int nr_tbls, i;
8719
8720 rmem->depth = 2;
8721 ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
8722 GFP_KERNEL);
8723 if (!ctx_pg->ctx_pg_tbl)
8724 return -ENOMEM;
8725 nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
8726 rmem->nr_pages = nr_tbls;
8727 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8728 if (rc)
8729 return rc;
8730 for (i = 0; i < nr_tbls; i++) {
8731 struct bnxt_ctx_pg_info *pg_tbl;
8732
8733 pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
8734 if (!pg_tbl)
8735 return -ENOMEM;
8736 ctx_pg->ctx_pg_tbl[i] = pg_tbl;
8737 rmem = &pg_tbl->ring_mem;
8738 rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
8739 rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
8740 rmem->depth = 1;
8741 rmem->nr_pages = MAX_CTX_PAGES;
8742 rmem->ctx_mem = ctxm;
8743 if (i == (nr_tbls - 1)) {
8744 int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
8745
8746 if (rem)
8747 rmem->nr_pages = rem;
8748 }
8749 rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
8750 if (rc)
8751 break;
8752 }
8753 } else {
8754 rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
8755 if (rmem->nr_pages > 1 || depth)
8756 rmem->depth = 1;
8757 rmem->ctx_mem = ctxm;
8758 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
8759 }
8760 return rc;
8761 }
8762
8763 static size_t bnxt_copy_ctx_pg_tbls(struct bnxt *bp,
8764 struct bnxt_ctx_pg_info *ctx_pg,
8765 void *buf, size_t offset, size_t head,
8766 size_t tail)
8767 {
8768 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8769 size_t nr_pages = ctx_pg->nr_pages;
8770 int page_size = rmem->page_size;
8771 size_t len = 0, total_len = 0;
8772 u16 depth = rmem->depth;
8773
8774 tail %= nr_pages * page_size;
8775 do {
8776 if (depth > 1) {
8777 int i = head / (page_size * MAX_CTX_PAGES);
8778 struct bnxt_ctx_pg_info *pg_tbl;
8779
8780 pg_tbl = ctx_pg->ctx_pg_tbl[i];
8781 rmem = &pg_tbl->ring_mem;
8782 }
8783 len = __bnxt_copy_ring(bp, rmem, buf, offset, head, tail);
8784 head += len;
8785 offset += len;
8786 total_len += len;
8787 if (head >= nr_pages * page_size)
8788 head = 0;
8789 } while (head != tail);
8790 return total_len;
8791 }
8792
8793 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
8794 struct bnxt_ctx_pg_info *ctx_pg)
8795 {
8796 struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
8797
8798 if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
8799 ctx_pg->ctx_pg_tbl) {
8800 int i, nr_tbls = rmem->nr_pages;
8801
8802 for (i = 0; i < nr_tbls; i++) {
8803 struct bnxt_ctx_pg_info *pg_tbl;
8804 struct bnxt_ring_mem_info *rmem2;
8805
8806 pg_tbl = ctx_pg->ctx_pg_tbl[i];
8807 if (!pg_tbl)
8808 continue;
8809 rmem2 = &pg_tbl->ring_mem;
8810 bnxt_free_ring(bp, rmem2);
8811 ctx_pg->ctx_pg_arr[i] = NULL;
8812 kfree(pg_tbl);
8813 ctx_pg->ctx_pg_tbl[i] = NULL;
8814 }
8815 kfree(ctx_pg->ctx_pg_tbl);
8816 ctx_pg->ctx_pg_tbl = NULL;
8817 }
8818 bnxt_free_ring(bp, rmem);
8819 ctx_pg->nr_pages = 0;
8820 }
8821
8822 static int bnxt_setup_ctxm_pg_tbls(struct bnxt *bp,
8823 struct bnxt_ctx_mem_type *ctxm, u32 entries,
8824 u8 pg_lvl)
8825 {
8826 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8827 int i, rc = 0, n = 1;
8828 u32 mem_size;
8829
8830 if (!ctxm->entry_size || !ctx_pg)
8831 return -EINVAL;
8832 if (ctxm->instance_bmap)
8833 n = hweight32(ctxm->instance_bmap);
8834 if (ctxm->entry_multiple)
8835 entries = roundup(entries, ctxm->entry_multiple);
8836 entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries);
8837 mem_size = entries * ctxm->entry_size;
8838 for (i = 0; i < n && !rc; i++) {
8839 ctx_pg[i].entries = entries;
8840 rc = bnxt_alloc_ctx_pg_tbls(bp, &ctx_pg[i], mem_size, pg_lvl,
8841 ctxm->init_value ? ctxm : NULL);
8842 }
8843 if (!rc)
8844 ctxm->mem_valid = 1;
8845 return rc;
8846 }
8847
8848 static int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt *bp,
8849 struct bnxt_ctx_mem_type *ctxm,
8850 bool last)
8851 {
8852 struct hwrm_func_backing_store_cfg_v2_input *req;
8853 u32 instance_bmap = ctxm->instance_bmap;
8854 int i, j, rc = 0, n = 1;
8855 __le32 *p;
8856
8857 if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info)
8858 return 0;
8859
8860 if (instance_bmap)
8861 n = hweight32(ctxm->instance_bmap);
8862 else
8863 instance_bmap = 1;
8864
8865 rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_CFG_V2);
8866 if (rc)
8867 return rc;
8868 hwrm_req_hold(bp, req);
8869 req->type = cpu_to_le16(ctxm->type);
8870 req->entry_size = cpu_to_le16(ctxm->entry_size);
8871 if ((ctxm->flags & BNXT_CTX_MEM_PERSIST) &&
8872 bnxt_bs_trace_avail(bp, ctxm->type)) {
8873 struct bnxt_bs_trace_info *bs_trace;
8874 u32 enables;
8875
8876 enables = FUNC_BACKING_STORE_CFG_V2_REQ_ENABLES_NEXT_BS_OFFSET;
8877 req->enables = cpu_to_le32(enables);
8878 bs_trace = &bp->bs_trace[bnxt_bstore_to_trace[ctxm->type]];
8879 req->next_bs_offset = cpu_to_le32(bs_trace->last_offset);
8880 }
8881 req->subtype_valid_cnt = ctxm->split_entry_cnt;
8882 for (i = 0, p = &req->split_entry_0; i < ctxm->split_entry_cnt; i++)
8883 p[i] = cpu_to_le32(ctxm->split[i]);
8884 for (i = 0, j = 0; j < n && !rc; i++) {
8885 struct bnxt_ctx_pg_info *ctx_pg;
8886
8887 if (!(instance_bmap & (1 << i)))
8888 continue;
8889 req->instance = cpu_to_le16(i);
8890 ctx_pg = &ctxm->pg_info[j++];
8891 if (!ctx_pg->entries)
8892 continue;
8893 req->num_entries = cpu_to_le32(ctx_pg->entries);
8894 bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
8895 &req->page_size_pbl_level,
8896 &req->page_dir);
8897 if (last && j == n)
8898 req->flags =
8899 cpu_to_le32(FUNC_BACKING_STORE_CFG_V2_REQ_FLAGS_BS_CFG_ALL_DONE);
8900 rc = hwrm_req_send(bp, req);
8901 }
8902 hwrm_req_drop(bp, req);
8903 return rc;
8904 }
8905
8906 static int bnxt_backing_store_cfg_v2(struct bnxt *bp, u32 ena)
8907 {
8908 struct bnxt_ctx_mem_info *ctx = bp->ctx;
8909 struct bnxt_ctx_mem_type *ctxm;
8910 u16 last_type = BNXT_CTX_INV;
8911 int rc = 0;
8912 u16 type;
8913
8914 for (type = BNXT_CTX_SRT; type <= BNXT_CTX_RIGP1; type++) {
8915 ctxm = &ctx->ctx_arr[type];
8916 if (!bnxt_bs_trace_avail(bp, type))
8917 continue;
8918 if (!ctxm->mem_valid) {
8919 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm,
8920 ctxm->max_entries, 1);
8921 if (rc) {
8922 netdev_warn(bp->dev, "Unable to setup ctx page for type:0x%x.\n",
8923 type);
8924 continue;
8925 }
8926 bnxt_bs_trace_init(bp, ctxm);
8927 last_type = type;
8928 }
8929 }
8930
8931 if (last_type == BNXT_CTX_INV) {
8932 if (!ena)
8933 return 0;
8934 else if (ena & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM)
8935 last_type = BNXT_CTX_MAX - 1;
8936 else
8937 last_type = BNXT_CTX_L2_MAX - 1;
8938 }
8939 ctx->ctx_arr[last_type].last = 1;
8940
8941 for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) {
8942 ctxm = &ctx->ctx_arr[type];
8943
8944 if (!ctxm->mem_valid)
8945 continue;
8946 rc = bnxt_hwrm_func_backing_store_cfg_v2(bp, ctxm, ctxm->last);
8947 if (rc)
8948 return rc;
8949 }
8950 return 0;
8951 }
8952
8953 /**
8954 * __bnxt_copy_ctx_mem - copy host context memory
8955 * @bp: The driver context
8956 * @ctxm: The pointer to the context memory type
8957 * @buf: The destination buffer or NULL to just obtain the length
8958 * @offset: The buffer offset to copy the data to
8959 * @head: The head offset of context memory to copy from
8960 * @tail: The tail offset (last byte + 1) of context memory to end the copy
8961 *
8962 * This function is called for debugging purposes to dump the host context
8963 * used by the chip.
8964 *
8965 * Return: Length of memory copied
8966 */
8967 static size_t __bnxt_copy_ctx_mem(struct bnxt *bp,
8968 struct bnxt_ctx_mem_type *ctxm, void *buf,
8969 size_t offset, size_t head, size_t tail)
8970 {
8971 struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info;
8972 size_t len = 0, total_len = 0;
8973 int i, n = 1;
8974
8975 if (!ctx_pg)
8976 return 0;
8977
8978 if (ctxm->instance_bmap)
8979 n = hweight32(ctxm->instance_bmap);
8980 for (i = 0; i < n; i++) {
8981 len = bnxt_copy_ctx_pg_tbls(bp, &ctx_pg[i], buf, offset, head,
8982 tail);
8983 offset += len;
8984 total_len += len;
8985 }
8986 return total_len;
8987 }
8988
8989 size_t bnxt_copy_ctx_mem(struct bnxt *bp, struct bnxt_ctx_mem_type *ctxm,
8990 void *buf, size_t offset)
8991 {
8992 size_t tail = ctxm->max_entries * ctxm->entry_size;
8993
8994 return __bnxt_copy_ctx_mem(bp, ctxm, buf, offset, 0, tail);
8995 }
8996
8997 static void bnxt_free_one_ctx_mem(struct bnxt *bp,
8998 struct bnxt_ctx_mem_type *ctxm, bool force)
8999 {
9000 struct bnxt_ctx_pg_info *ctx_pg;
9001 int i, n = 1;
9002
9003 ctxm->last = 0;
9004
9005 if (ctxm->mem_valid && !force && (ctxm->flags & BNXT_CTX_MEM_PERSIST))
9006 return;
9007
9008 ctx_pg = ctxm->pg_info;
9009 if (ctx_pg) {
9010 if (ctxm->instance_bmap)
9011 n = hweight32(ctxm->instance_bmap);
9012 for (i = 0; i < n; i++)
9013 bnxt_free_ctx_pg_tbls(bp, &ctx_pg[i]);
9014
9015 kfree(ctx_pg);
9016 ctxm->pg_info = NULL;
9017 ctxm->mem_valid = 0;
9018 }
9019 memset(ctxm, 0, sizeof(*ctxm));
9020 }
9021
9022 void bnxt_free_ctx_mem(struct bnxt *bp, bool force)
9023 {
9024 struct bnxt_ctx_mem_info *ctx = bp->ctx;
9025 u16 type;
9026
9027 if (!ctx)
9028 return;
9029
9030 for (type = 0; type < BNXT_CTX_V2_MAX; type++)
9031 bnxt_free_one_ctx_mem(bp, &ctx->ctx_arr[type], force);
9032
9033 ctx->flags &= ~BNXT_CTX_FLAG_INITED;
9034 if (force) {
9035 kfree(ctx);
9036 bp->ctx = NULL;
9037 }
9038 }
9039
9040 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
9041 {
9042 struct bnxt_ctx_mem_type *ctxm;
9043 struct bnxt_ctx_mem_info *ctx;
9044 u32 l2_qps, qp1_qps, max_qps;
9045 u32 ena, entries_sp, entries;
9046 u32 srqs, max_srqs, min;
9047 u32 num_mr, num_ah;
9048 u32 extra_srqs = 0;
9049 u32 extra_qps = 0;
9050 u32 fast_qpmd_qps;
9051 u8 pg_lvl = 1;
9052 int i, rc;
9053
9054 rc = bnxt_hwrm_func_backing_store_qcaps(bp);
9055 if (rc) {
9056 netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
9057 rc);
9058 return rc;
9059 }
9060 ctx = bp->ctx;
9061 if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
9062 return 0;
9063
9064 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
9065 l2_qps = ctxm->qp_l2_entries;
9066 qp1_qps = ctxm->qp_qp1_entries;
9067 fast_qpmd_qps = ctxm->qp_fast_qpmd_entries;
9068 max_qps = ctxm->max_entries;
9069 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
9070 srqs = ctxm->srq_l2_entries;
9071 max_srqs = ctxm->max_entries;
9072 ena = 0;
9073 if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
9074 pg_lvl = 2;
9075 extra_qps = min_t(u32, 65536, max_qps - l2_qps - qp1_qps);
9076 /* allocate extra qps if fw supports RoCE fast qp destroy feature */
9077 extra_qps += fast_qpmd_qps;
9078 extra_srqs = min_t(u32, 8192, max_srqs - srqs);
9079 if (fast_qpmd_qps)
9080 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP_FAST_QPMD;
9081 }
9082
9083 ctxm = &ctx->ctx_arr[BNXT_CTX_QP];
9084 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps,
9085 pg_lvl);
9086 if (rc)
9087 return rc;
9088
9089 ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ];
9090 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, srqs + extra_srqs, pg_lvl);
9091 if (rc)
9092 return rc;
9093
9094 ctxm = &ctx->ctx_arr[BNXT_CTX_CQ];
9095 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->cq_l2_entries +
9096 extra_qps * 2, pg_lvl);
9097 if (rc)
9098 return rc;
9099
9100 ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC];
9101 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
9102 if (rc)
9103 return rc;
9104
9105 ctxm = &ctx->ctx_arr[BNXT_CTX_STAT];
9106 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, ctxm->max_entries, 1);
9107 if (rc)
9108 return rc;
9109
9110 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
9111 goto skip_rdma;
9112
9113 ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV];
9114 /* 128K extra is needed to accommodate static AH context
9115 * allocation by f/w.
9116 */
9117 num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256);
9118 num_ah = min_t(u32, num_mr, 1024 * 128);
9119 ctxm->split_entry_cnt = BNXT_CTX_MRAV_AV_SPLIT_ENTRY + 1;
9120 if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah)
9121 ctxm->mrav_av_entries = num_ah;
9122
9123 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, num_mr + num_ah, 2);
9124 if (rc)
9125 return rc;
9126 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
9127
9128 ctxm = &ctx->ctx_arr[BNXT_CTX_TIM];
9129 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, l2_qps + qp1_qps + extra_qps, 1);
9130 if (rc)
9131 return rc;
9132 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
9133
9134 skip_rdma:
9135 ctxm = &ctx->ctx_arr[BNXT_CTX_STQM];
9136 min = ctxm->min_entries;
9137 entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps +
9138 2 * (extra_qps + qp1_qps) + min;
9139 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries_sp, 2);
9140 if (rc)
9141 return rc;
9142
9143 ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM];
9144 entries = l2_qps + 2 * (extra_qps + qp1_qps);
9145 rc = bnxt_setup_ctxm_pg_tbls(bp, ctxm, entries, 2);
9146 if (rc)
9147 return rc;
9148 for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
9149 ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
9150 ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
9151
9152 if (bp->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2)
9153 rc = bnxt_backing_store_cfg_v2(bp, ena);
9154 else
9155 rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
9156 if (rc) {
9157 netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
9158 rc);
9159 return rc;
9160 }
9161 ctx->flags |= BNXT_CTX_FLAG_INITED;
9162 return 0;
9163 }
9164
9165 static int bnxt_hwrm_crash_dump_mem_cfg(struct bnxt *bp)
9166 {
9167 struct hwrm_dbg_crashdump_medium_cfg_input *req;
9168 u16 page_attr;
9169 int rc;
9170
9171 if (!(bp->fw_dbg_cap & DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR))
9172 return 0;
9173
9174 rc = hwrm_req_init(bp, req, HWRM_DBG_CRASHDUMP_MEDIUM_CFG);
9175 if (rc)
9176 return rc;
9177
9178 if (BNXT_PAGE_SIZE == 0x2000)
9179 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_8K;
9180 else if (BNXT_PAGE_SIZE == 0x10000)
9181 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_64K;
9182 else
9183 page_attr = DBG_CRASHDUMP_MEDIUM_CFG_REQ_PG_SIZE_PG_4K;
9184 req->pg_size_lvl = cpu_to_le16(page_attr |
9185 bp->fw_crash_mem->ring_mem.depth);
9186 req->pbl = cpu_to_le64(bp->fw_crash_mem->ring_mem.pg_tbl_map);
9187 req->size = cpu_to_le32(bp->fw_crash_len);
9188 req->output_dest_flags = cpu_to_le16(BNXT_DBG_CR_DUMP_MDM_CFG_DDR);
9189 return hwrm_req_send(bp, req);
9190 }
9191
9192 static void bnxt_free_crash_dump_mem(struct bnxt *bp)
9193 {
9194 if (bp->fw_crash_mem) {
9195 bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem);
9196 kfree(bp->fw_crash_mem);
9197 bp->fw_crash_mem = NULL;
9198 }
9199 }
9200
9201 static int bnxt_alloc_crash_dump_mem(struct bnxt *bp)
9202 {
9203 u32 mem_size = 0;
9204 int rc;
9205
9206 if (!(bp->fw_dbg_cap & DBG_QCAPS_RESP_FLAGS_CRASHDUMP_HOST_DDR))
9207 return 0;
9208
9209 rc = bnxt_hwrm_get_dump_len(bp, BNXT_DUMP_CRASH, &mem_size);
9210 if (rc)
9211 return rc;
9212
9213 mem_size = round_up(mem_size, 4);
9214
9215 /* keep and use the existing pages */
9216 if (bp->fw_crash_mem &&
9217 mem_size <= bp->fw_crash_mem->nr_pages * BNXT_PAGE_SIZE)
9218 goto alloc_done;
9219
9220 if (bp->fw_crash_mem)
9221 bnxt_free_ctx_pg_tbls(bp, bp->fw_crash_mem);
9222 else
9223 bp->fw_crash_mem = kzalloc(sizeof(*bp->fw_crash_mem),
9224 GFP_KERNEL);
9225 if (!bp->fw_crash_mem)
9226 return -ENOMEM;
9227
9228 rc = bnxt_alloc_ctx_pg_tbls(bp, bp->fw_crash_mem, mem_size, 1, NULL);
9229 if (rc) {
9230 bnxt_free_crash_dump_mem(bp);
9231 return rc;
9232 }
9233
9234 alloc_done:
9235 bp->fw_crash_len = mem_size;
9236 return 0;
9237 }
9238
9239 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
9240 {
9241 struct hwrm_func_resource_qcaps_output *resp;
9242 struct hwrm_func_resource_qcaps_input *req;
9243 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9244 int rc;
9245
9246 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
9247 if (rc)
9248 return rc;
9249
9250 req->fid = cpu_to_le16(0xffff);
9251 resp = hwrm_req_hold(bp, req);
9252 rc = hwrm_req_send_silent(bp, req);
9253 if (rc)
9254 goto hwrm_func_resc_qcaps_exit;
9255
9256 hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
9257 if (!all)
9258 goto hwrm_func_resc_qcaps_exit;
9259
9260 hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
9261 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
9262 hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
9263 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
9264 hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
9265 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
9266 hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
9267 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
9268 hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
9269 hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
9270 hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
9271 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
9272 hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
9273 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
9274 hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
9275 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
9276
9277 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
9278 u16 max_msix = le16_to_cpu(resp->max_msix);
9279
9280 hw_resc->max_nqs = max_msix;
9281 hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
9282 }
9283
9284 if (BNXT_PF(bp)) {
9285 struct bnxt_pf_info *pf = &bp->pf;
9286
9287 pf->vf_resv_strategy =
9288 le16_to_cpu(resp->vf_reservation_strategy);
9289 if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
9290 pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
9291 }
9292 hwrm_func_resc_qcaps_exit:
9293 hwrm_req_drop(bp, req);
9294 return rc;
9295 }
9296
9297 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
9298 {
9299 struct hwrm_port_mac_ptp_qcfg_output *resp;
9300 struct hwrm_port_mac_ptp_qcfg_input *req;
9301 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
9302 u8 flags;
9303 int rc;
9304
9305 if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_PLUS(bp)) {
9306 rc = -ENODEV;
9307 goto no_ptp;
9308 }
9309
9310 rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
9311 if (rc)
9312 goto no_ptp;
9313
9314 req->port_id = cpu_to_le16(bp->pf.port_id);
9315 resp = hwrm_req_hold(bp, req);
9316 rc = hwrm_req_send(bp, req);
9317 if (rc)
9318 goto exit;
9319
9320 flags = resp->flags;
9321 if (BNXT_CHIP_P5_AND_MINUS(bp) &&
9322 !(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
9323 rc = -ENODEV;
9324 goto exit;
9325 }
9326 if (!ptp) {
9327 ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
9328 if (!ptp) {
9329 rc = -ENOMEM;
9330 goto exit;
9331 }
9332 ptp->bp = bp;
9333 bp->ptp_cfg = ptp;
9334 }
9335
9336 if (flags &
9337 (PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK |
9338 PORT_MAC_PTP_QCFG_RESP_FLAGS_64B_PHC_TIME)) {
9339 ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
9340 ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
9341 } else if (BNXT_CHIP_P5(bp)) {
9342 ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
9343 ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
9344 } else {
9345 rc = -ENODEV;
9346 goto exit;
9347 }
9348 ptp->rtc_configured =
9349 (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
9350 rc = bnxt_ptp_init(bp);
9351 if (rc)
9352 netdev_warn(bp->dev, "PTP initialization failed.\n");
9353 exit:
9354 hwrm_req_drop(bp, req);
9355 if (!rc)
9356 return 0;
9357
9358 no_ptp:
9359 bnxt_ptp_clear(bp);
9360 kfree(ptp);
9361 bp->ptp_cfg = NULL;
9362 return rc;
9363 }
9364
9365 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
9366 {
9367 struct hwrm_func_qcaps_output *resp;
9368 struct hwrm_func_qcaps_input *req;
9369 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9370 u32 flags, flags_ext, flags_ext2;
9371 int rc;
9372
9373 rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
9374 if (rc)
9375 return rc;
9376
9377 req->fid = cpu_to_le16(0xffff);
9378 resp = hwrm_req_hold(bp, req);
9379 rc = hwrm_req_send(bp, req);
9380 if (rc)
9381 goto hwrm_func_qcaps_exit;
9382
9383 flags = le32_to_cpu(resp->flags);
9384 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
9385 bp->flags |= BNXT_FLAG_ROCEV1_CAP;
9386 if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
9387 bp->flags |= BNXT_FLAG_ROCEV2_CAP;
9388 if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
9389 bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
9390 if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
9391 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
9392 if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
9393 bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
9394 if (flags & FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
9395 bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
9396 if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
9397 bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
9398 if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
9399 bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
9400 if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
9401 bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
9402
9403 flags_ext = le32_to_cpu(resp->flags_ext);
9404 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
9405 bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
9406 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
9407 bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
9408 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
9409 bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
9410 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
9411 bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
9412 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
9413 bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
9414 if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_DFLT_VLAN_TPID_PCP_SUPPORTED))
9415 bp->fw_cap |= BNXT_FW_CAP_DFLT_VLAN_TPID_PCP;
9416 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_BS_V2_SUPPORTED)
9417 bp->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2;
9418 if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_TX_COAL_CMPL_CAP)
9419 bp->flags |= BNXT_FLAG_TX_COAL_CMPL;
9420
9421 flags_ext2 = le32_to_cpu(resp->flags_ext2);
9422 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
9423 bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
9424 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_UDP_GSO_SUPPORTED)
9425 bp->flags |= BNXT_FLAG_UDP_GSO_CAP;
9426 if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_TX_PKT_TS_CMPL_SUPPORTED)
9427 bp->fw_cap |= BNXT_FW_CAP_TX_TS_CMP;
9428 if (BNXT_PF(bp) &&
9429 (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_ROCE_VF_RESOURCE_MGMT_SUPPORTED))
9430 bp->fw_cap |= BNXT_FW_CAP_ROCE_VF_RESC_MGMT_SUPPORTED;
9431
9432 bp->tx_push_thresh = 0;
9433 if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
9434 BNXT_FW_MAJ(bp) > 217)
9435 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
9436
9437 hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
9438 hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
9439 hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
9440 hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
9441 hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
9442 if (!hw_resc->max_hw_ring_grps)
9443 hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
9444 hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
9445 hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
9446 hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
9447
9448 hw_resc->max_encap_records = le32_to_cpu(resp->max_encap_records);
9449 hw_resc->max_decap_records = le32_to_cpu(resp->max_decap_records);
9450 hw_resc->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
9451 hw_resc->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
9452 hw_resc->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
9453 hw_resc->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
9454
9455 if (BNXT_PF(bp)) {
9456 struct bnxt_pf_info *pf = &bp->pf;
9457
9458 pf->fw_fid = le16_to_cpu(resp->fid);
9459 pf->port_id = le16_to_cpu(resp->port_id);
9460 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
9461 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
9462 pf->max_vfs = le16_to_cpu(resp->max_vfs);
9463 bp->flags &= ~BNXT_FLAG_WOL_CAP;
9464 if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
9465 bp->flags |= BNXT_FLAG_WOL_CAP;
9466 if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
9467 bp->fw_cap |= BNXT_FW_CAP_PTP;
9468 } else {
9469 bnxt_ptp_clear(bp);
9470 kfree(bp->ptp_cfg);
9471 bp->ptp_cfg = NULL;
9472 }
9473 } else {
9474 #ifdef CONFIG_BNXT_SRIOV
9475 struct bnxt_vf_info *vf = &bp->vf;
9476
9477 vf->fw_fid = le16_to_cpu(resp->fid);
9478 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
9479 #endif
9480 }
9481 bp->tso_max_segs = le16_to_cpu(resp->max_tso_segs);
9482
9483 hwrm_func_qcaps_exit:
9484 hwrm_req_drop(bp, req);
9485 return rc;
9486 }
9487
9488 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
9489 {
9490 struct hwrm_dbg_qcaps_output *resp;
9491 struct hwrm_dbg_qcaps_input *req;
9492 int rc;
9493
9494 bp->fw_dbg_cap = 0;
9495 if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
9496 return;
9497
9498 rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
9499 if (rc)
9500 return;
9501
9502 req->fid = cpu_to_le16(0xffff);
9503 resp = hwrm_req_hold(bp, req);
9504 rc = hwrm_req_send(bp, req);
9505 if (rc)
9506 goto hwrm_dbg_qcaps_exit;
9507
9508 bp->fw_dbg_cap = le32_to_cpu(resp->flags);
9509
9510 hwrm_dbg_qcaps_exit:
9511 hwrm_req_drop(bp, req);
9512 }
9513
9514 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
9515
9516 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
9517 {
9518 int rc;
9519
9520 rc = __bnxt_hwrm_func_qcaps(bp);
9521 if (rc)
9522 return rc;
9523
9524 bnxt_hwrm_dbg_qcaps(bp);
9525
9526 rc = bnxt_hwrm_queue_qportcfg(bp);
9527 if (rc) {
9528 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
9529 return rc;
9530 }
9531 if (bp->hwrm_spec_code >= 0x10803) {
9532 rc = bnxt_alloc_ctx_mem(bp);
9533 if (rc)
9534 return rc;
9535 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9536 if (!rc)
9537 bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
9538 }
9539 return 0;
9540 }
9541
9542 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
9543 {
9544 struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
9545 struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
9546 u32 flags;
9547 int rc;
9548
9549 if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
9550 return 0;
9551
9552 rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
9553 if (rc)
9554 return rc;
9555
9556 resp = hwrm_req_hold(bp, req);
9557 rc = hwrm_req_send(bp, req);
9558 if (rc)
9559 goto hwrm_cfa_adv_qcaps_exit;
9560
9561 flags = le32_to_cpu(resp->flags);
9562 if (flags &
9563 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
9564 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
9565
9566 if (flags &
9567 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED)
9568 bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3;
9569
9570 if (flags &
9571 CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED)
9572 bp->fw_cap |= BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO;
9573
9574 hwrm_cfa_adv_qcaps_exit:
9575 hwrm_req_drop(bp, req);
9576 return rc;
9577 }
9578
9579 static int __bnxt_alloc_fw_health(struct bnxt *bp)
9580 {
9581 if (bp->fw_health)
9582 return 0;
9583
9584 bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
9585 if (!bp->fw_health)
9586 return -ENOMEM;
9587
9588 mutex_init(&bp->fw_health->lock);
9589 return 0;
9590 }
9591
9592 static int bnxt_alloc_fw_health(struct bnxt *bp)
9593 {
9594 int rc;
9595
9596 if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
9597 !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
9598 return 0;
9599
9600 rc = __bnxt_alloc_fw_health(bp);
9601 if (rc) {
9602 bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
9603 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9604 return rc;
9605 }
9606
9607 return 0;
9608 }
9609
9610 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
9611 {
9612 writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
9613 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
9614 BNXT_FW_HEALTH_WIN_MAP_OFF);
9615 }
9616
9617 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
9618 {
9619 struct bnxt_fw_health *fw_health = bp->fw_health;
9620 u32 reg_type;
9621
9622 if (!fw_health)
9623 return;
9624
9625 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
9626 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
9627 fw_health->status_reliable = false;
9628
9629 reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
9630 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
9631 fw_health->resets_reliable = false;
9632 }
9633
9634 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
9635 {
9636 void __iomem *hs;
9637 u32 status_loc;
9638 u32 reg_type;
9639 u32 sig;
9640
9641 if (bp->fw_health)
9642 bp->fw_health->status_reliable = false;
9643
9644 __bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
9645 hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
9646
9647 sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
9648 if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
9649 if (!bp->chip_num) {
9650 __bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
9651 bp->chip_num = readl(bp->bar0 +
9652 BNXT_FW_HEALTH_WIN_BASE +
9653 BNXT_GRC_REG_CHIP_NUM);
9654 }
9655 if (!BNXT_CHIP_P5_PLUS(bp))
9656 return;
9657
9658 status_loc = BNXT_GRC_REG_STATUS_P5 |
9659 BNXT_FW_HEALTH_REG_TYPE_BAR0;
9660 } else {
9661 status_loc = readl(hs + offsetof(struct hcomm_status,
9662 fw_status_loc));
9663 }
9664
9665 if (__bnxt_alloc_fw_health(bp)) {
9666 netdev_warn(bp->dev, "no memory for firmware status checks\n");
9667 return;
9668 }
9669
9670 bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
9671 reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
9672 if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
9673 __bnxt_map_fw_health_reg(bp, status_loc);
9674 bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
9675 BNXT_FW_HEALTH_WIN_OFF(status_loc);
9676 }
9677
9678 bp->fw_health->status_reliable = true;
9679 }
9680
9681 static int bnxt_map_fw_health_regs(struct bnxt *bp)
9682 {
9683 struct bnxt_fw_health *fw_health = bp->fw_health;
9684 u32 reg_base = 0xffffffff;
9685 int i;
9686
9687 bp->fw_health->status_reliable = false;
9688 bp->fw_health->resets_reliable = false;
9689 /* Only pre-map the monitoring GRC registers using window 3 */
9690 for (i = 0; i < 4; i++) {
9691 u32 reg = fw_health->regs[i];
9692
9693 if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
9694 continue;
9695 if (reg_base == 0xffffffff)
9696 reg_base = reg & BNXT_GRC_BASE_MASK;
9697 if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
9698 return -ERANGE;
9699 fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
9700 }
9701 bp->fw_health->status_reliable = true;
9702 bp->fw_health->resets_reliable = true;
9703 if (reg_base == 0xffffffff)
9704 return 0;
9705
9706 __bnxt_map_fw_health_reg(bp, reg_base);
9707 return 0;
9708 }
9709
9710 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
9711 {
9712 if (!bp->fw_health)
9713 return;
9714
9715 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
9716 bp->fw_health->status_reliable = true;
9717 bp->fw_health->resets_reliable = true;
9718 } else {
9719 bnxt_try_map_fw_health_reg(bp);
9720 }
9721 }
9722
9723 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
9724 {
9725 struct bnxt_fw_health *fw_health = bp->fw_health;
9726 struct hwrm_error_recovery_qcfg_output *resp;
9727 struct hwrm_error_recovery_qcfg_input *req;
9728 int rc, i;
9729
9730 if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
9731 return 0;
9732
9733 rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
9734 if (rc)
9735 return rc;
9736
9737 resp = hwrm_req_hold(bp, req);
9738 rc = hwrm_req_send(bp, req);
9739 if (rc)
9740 goto err_recovery_out;
9741 fw_health->flags = le32_to_cpu(resp->flags);
9742 if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
9743 !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
9744 rc = -EINVAL;
9745 goto err_recovery_out;
9746 }
9747 fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
9748 fw_health->master_func_wait_dsecs =
9749 le32_to_cpu(resp->master_func_wait_period);
9750 fw_health->normal_func_wait_dsecs =
9751 le32_to_cpu(resp->normal_func_wait_period);
9752 fw_health->post_reset_wait_dsecs =
9753 le32_to_cpu(resp->master_func_wait_period_after_reset);
9754 fw_health->post_reset_max_wait_dsecs =
9755 le32_to_cpu(resp->max_bailout_time_after_reset);
9756 fw_health->regs[BNXT_FW_HEALTH_REG] =
9757 le32_to_cpu(resp->fw_health_status_reg);
9758 fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
9759 le32_to_cpu(resp->fw_heartbeat_reg);
9760 fw_health->regs[BNXT_FW_RESET_CNT_REG] =
9761 le32_to_cpu(resp->fw_reset_cnt_reg);
9762 fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
9763 le32_to_cpu(resp->reset_inprogress_reg);
9764 fw_health->fw_reset_inprog_reg_mask =
9765 le32_to_cpu(resp->reset_inprogress_reg_mask);
9766 fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
9767 if (fw_health->fw_reset_seq_cnt >= 16) {
9768 rc = -EINVAL;
9769 goto err_recovery_out;
9770 }
9771 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
9772 fw_health->fw_reset_seq_regs[i] =
9773 le32_to_cpu(resp->reset_reg[i]);
9774 fw_health->fw_reset_seq_vals[i] =
9775 le32_to_cpu(resp->reset_reg_val[i]);
9776 fw_health->fw_reset_seq_delay_msec[i] =
9777 resp->delay_after_reset[i];
9778 }
9779 err_recovery_out:
9780 hwrm_req_drop(bp, req);
9781 if (!rc)
9782 rc = bnxt_map_fw_health_regs(bp);
9783 if (rc)
9784 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
9785 return rc;
9786 }
9787
9788 static int bnxt_hwrm_func_reset(struct bnxt *bp)
9789 {
9790 struct hwrm_func_reset_input *req;
9791 int rc;
9792
9793 rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
9794 if (rc)
9795 return rc;
9796
9797 req->enables = 0;
9798 hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
9799 return hwrm_req_send(bp, req);
9800 }
9801
9802 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
9803 {
9804 struct hwrm_nvm_get_dev_info_output nvm_info;
9805
9806 if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
9807 snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
9808 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
9809 nvm_info.nvm_cfg_ver_upd);
9810 }
9811
9812 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
9813 {
9814 struct hwrm_queue_qportcfg_output *resp;
9815 struct hwrm_queue_qportcfg_input *req;
9816 u8 i, j, *qptr;
9817 bool no_rdma;
9818 int rc = 0;
9819
9820 rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
9821 if (rc)
9822 return rc;
9823
9824 resp = hwrm_req_hold(bp, req);
9825 rc = hwrm_req_send(bp, req);
9826 if (rc)
9827 goto qportcfg_exit;
9828
9829 if (!resp->max_configurable_queues) {
9830 rc = -EINVAL;
9831 goto qportcfg_exit;
9832 }
9833 bp->max_tc = resp->max_configurable_queues;
9834 bp->max_lltc = resp->max_configurable_lossless_queues;
9835 if (bp->max_tc > BNXT_MAX_QUEUE)
9836 bp->max_tc = BNXT_MAX_QUEUE;
9837
9838 no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
9839 qptr = &resp->queue_id0;
9840 for (i = 0, j = 0; i < bp->max_tc; i++) {
9841 bp->q_info[j].queue_id = *qptr;
9842 bp->q_ids[i] = *qptr++;
9843 bp->q_info[j].queue_profile = *qptr++;
9844 bp->tc_to_qidx[j] = j;
9845 if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
9846 (no_rdma && BNXT_PF(bp)))
9847 j++;
9848 }
9849 bp->max_q = bp->max_tc;
9850 bp->max_tc = max_t(u8, j, 1);
9851
9852 if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
9853 bp->max_tc = 1;
9854
9855 if (bp->max_lltc > bp->max_tc)
9856 bp->max_lltc = bp->max_tc;
9857
9858 qportcfg_exit:
9859 hwrm_req_drop(bp, req);
9860 return rc;
9861 }
9862
9863 static int bnxt_hwrm_poll(struct bnxt *bp)
9864 {
9865 struct hwrm_ver_get_input *req;
9866 int rc;
9867
9868 rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9869 if (rc)
9870 return rc;
9871
9872 req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9873 req->hwrm_intf_min = HWRM_VERSION_MINOR;
9874 req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9875
9876 hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
9877 rc = hwrm_req_send(bp, req);
9878 return rc;
9879 }
9880
9881 static int bnxt_hwrm_ver_get(struct bnxt *bp)
9882 {
9883 struct hwrm_ver_get_output *resp;
9884 struct hwrm_ver_get_input *req;
9885 u16 fw_maj, fw_min, fw_bld, fw_rsv;
9886 u32 dev_caps_cfg, hwrm_ver;
9887 int rc, len;
9888
9889 rc = hwrm_req_init(bp, req, HWRM_VER_GET);
9890 if (rc)
9891 return rc;
9892
9893 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
9894 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
9895 req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
9896 req->hwrm_intf_min = HWRM_VERSION_MINOR;
9897 req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
9898
9899 resp = hwrm_req_hold(bp, req);
9900 rc = hwrm_req_send(bp, req);
9901 if (rc)
9902 goto hwrm_ver_get_exit;
9903
9904 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
9905
9906 bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
9907 resp->hwrm_intf_min_8b << 8 |
9908 resp->hwrm_intf_upd_8b;
9909 if (resp->hwrm_intf_maj_8b < 1) {
9910 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
9911 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9912 resp->hwrm_intf_upd_8b);
9913 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
9914 }
9915
9916 hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
9917 HWRM_VERSION_UPDATE;
9918
9919 if (bp->hwrm_spec_code > hwrm_ver)
9920 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9921 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
9922 HWRM_VERSION_UPDATE);
9923 else
9924 snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
9925 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
9926 resp->hwrm_intf_upd_8b);
9927
9928 fw_maj = le16_to_cpu(resp->hwrm_fw_major);
9929 if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
9930 fw_min = le16_to_cpu(resp->hwrm_fw_minor);
9931 fw_bld = le16_to_cpu(resp->hwrm_fw_build);
9932 fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
9933 len = FW_VER_STR_LEN;
9934 } else {
9935 fw_maj = resp->hwrm_fw_maj_8b;
9936 fw_min = resp->hwrm_fw_min_8b;
9937 fw_bld = resp->hwrm_fw_bld_8b;
9938 fw_rsv = resp->hwrm_fw_rsvd_8b;
9939 len = BC_HWRM_STR_LEN;
9940 }
9941 bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
9942 snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
9943 fw_rsv);
9944
9945 if (strlen(resp->active_pkg_name)) {
9946 int fw_ver_len = strlen(bp->fw_ver_str);
9947
9948 snprintf(bp->fw_ver_str + fw_ver_len,
9949 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
9950 resp->active_pkg_name);
9951 bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
9952 }
9953
9954 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
9955 if (!bp->hwrm_cmd_timeout)
9956 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
9957 bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
9958 if (!bp->hwrm_cmd_max_timeout)
9959 bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
9960 else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
9961 netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
9962 bp->hwrm_cmd_max_timeout / 1000);
9963
9964 if (resp->hwrm_intf_maj_8b >= 1) {
9965 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
9966 bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
9967 }
9968 if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
9969 bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
9970
9971 bp->chip_num = le16_to_cpu(resp->chip_num);
9972 bp->chip_rev = resp->chip_rev;
9973 if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
9974 !resp->chip_metal)
9975 bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
9976
9977 dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
9978 if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
9979 (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
9980 bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
9981
9982 if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
9983 bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
9984
9985 if (dev_caps_cfg &
9986 VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
9987 bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
9988
9989 if (dev_caps_cfg &
9990 VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
9991 bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
9992
9993 if (dev_caps_cfg &
9994 VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
9995 bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
9996
9997 hwrm_ver_get_exit:
9998 hwrm_req_drop(bp, req);
9999 return rc;
10000 }
10001
10002 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
10003 {
10004 struct hwrm_fw_set_time_input *req;
10005 struct tm tm;
10006 time64_t now = ktime_get_real_seconds();
10007 int rc;
10008
10009 if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
10010 bp->hwrm_spec_code < 0x10400)
10011 return -EOPNOTSUPP;
10012
10013 time64_to_tm(now, 0, &tm);
10014 rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
10015 if (rc)
10016 return rc;
10017
10018 req->year = cpu_to_le16(1900 + tm.tm_year);
10019 req->month = 1 + tm.tm_mon;
10020 req->day = tm.tm_mday;
10021 req->hour = tm.tm_hour;
10022 req->minute = tm.tm_min;
10023 req->second = tm.tm_sec;
10024 return hwrm_req_send(bp, req);
10025 }
10026
10027 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
10028 {
10029 u64 sw_tmp;
10030
10031 hw &= mask;
10032 sw_tmp = (*sw & ~mask) | hw;
10033 if (hw < (*sw & mask))
10034 sw_tmp += mask + 1;
10035 WRITE_ONCE(*sw, sw_tmp);
10036 }
10037
10038 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
10039 int count, bool ignore_zero)
10040 {
10041 int i;
10042
10043 for (i = 0; i < count; i++) {
10044 u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
10045
10046 if (ignore_zero && !hw)
10047 continue;
10048
10049 if (masks[i] == -1ULL)
10050 sw_stats[i] = hw;
10051 else
10052 bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
10053 }
10054 }
10055
10056 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
10057 {
10058 if (!stats->hw_stats)
10059 return;
10060
10061 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
10062 stats->hw_masks, stats->len / 8, false);
10063 }
10064
10065 static void bnxt_accumulate_all_stats(struct bnxt *bp)
10066 {
10067 struct bnxt_stats_mem *ring0_stats;
10068 bool ignore_zero = false;
10069 int i;
10070
10071 /* Chip bug. Counter intermittently becomes 0. */
10072 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10073 ignore_zero = true;
10074
10075 for (i = 0; i < bp->cp_nr_rings; i++) {
10076 struct bnxt_napi *bnapi = bp->bnapi[i];
10077 struct bnxt_cp_ring_info *cpr;
10078 struct bnxt_stats_mem *stats;
10079
10080 cpr = &bnapi->cp_ring;
10081 stats = &cpr->stats;
10082 if (!i)
10083 ring0_stats = stats;
10084 __bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
10085 ring0_stats->hw_masks,
10086 ring0_stats->len / 8, ignore_zero);
10087 }
10088 if (bp->flags & BNXT_FLAG_PORT_STATS) {
10089 struct bnxt_stats_mem *stats = &bp->port_stats;
10090 __le64 *hw_stats = stats->hw_stats;
10091 u64 *sw_stats = stats->sw_stats;
10092 u64 *masks = stats->hw_masks;
10093 int cnt;
10094
10095 cnt = sizeof(struct rx_port_stats) / 8;
10096 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
10097
10098 hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10099 sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10100 masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10101 cnt = sizeof(struct tx_port_stats) / 8;
10102 __bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
10103 }
10104 if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
10105 bnxt_accumulate_stats(&bp->rx_port_stats_ext);
10106 bnxt_accumulate_stats(&bp->tx_port_stats_ext);
10107 }
10108 }
10109
10110 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
10111 {
10112 struct hwrm_port_qstats_input *req;
10113 struct bnxt_pf_info *pf = &bp->pf;
10114 int rc;
10115
10116 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
10117 return 0;
10118
10119 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
10120 return -EOPNOTSUPP;
10121
10122 rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
10123 if (rc)
10124 return rc;
10125
10126 req->flags = flags;
10127 req->port_id = cpu_to_le16(pf->port_id);
10128 req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
10129 BNXT_TX_PORT_STATS_BYTE_OFFSET);
10130 req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
10131 return hwrm_req_send(bp, req);
10132 }
10133
10134 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
10135 {
10136 struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
10137 struct hwrm_queue_pri2cos_qcfg_input *req_qc;
10138 struct hwrm_port_qstats_ext_output *resp_qs;
10139 struct hwrm_port_qstats_ext_input *req_qs;
10140 struct bnxt_pf_info *pf = &bp->pf;
10141 u32 tx_stat_size;
10142 int rc;
10143
10144 if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
10145 return 0;
10146
10147 if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
10148 return -EOPNOTSUPP;
10149
10150 rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
10151 if (rc)
10152 return rc;
10153
10154 req_qs->flags = flags;
10155 req_qs->port_id = cpu_to_le16(pf->port_id);
10156 req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
10157 req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
10158 tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
10159 sizeof(struct tx_port_stats_ext) : 0;
10160 req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
10161 req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
10162 resp_qs = hwrm_req_hold(bp, req_qs);
10163 rc = hwrm_req_send(bp, req_qs);
10164 if (!rc) {
10165 bp->fw_rx_stats_ext_size =
10166 le16_to_cpu(resp_qs->rx_stat_size) / 8;
10167 if (BNXT_FW_MAJ(bp) < 220 &&
10168 bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
10169 bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
10170
10171 bp->fw_tx_stats_ext_size = tx_stat_size ?
10172 le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
10173 } else {
10174 bp->fw_rx_stats_ext_size = 0;
10175 bp->fw_tx_stats_ext_size = 0;
10176 }
10177 hwrm_req_drop(bp, req_qs);
10178
10179 if (flags)
10180 return rc;
10181
10182 if (bp->fw_tx_stats_ext_size <=
10183 offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
10184 bp->pri2cos_valid = 0;
10185 return rc;
10186 }
10187
10188 rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
10189 if (rc)
10190 return rc;
10191
10192 req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
10193
10194 resp_qc = hwrm_req_hold(bp, req_qc);
10195 rc = hwrm_req_send(bp, req_qc);
10196 if (!rc) {
10197 u8 *pri2cos;
10198 int i, j;
10199
10200 pri2cos = &resp_qc->pri0_cos_queue_id;
10201 for (i = 0; i < 8; i++) {
10202 u8 queue_id = pri2cos[i];
10203 u8 queue_idx;
10204
10205 /* Per port queue IDs start from 0, 10, 20, etc */
10206 queue_idx = queue_id % 10;
10207 if (queue_idx > BNXT_MAX_QUEUE) {
10208 bp->pri2cos_valid = false;
10209 hwrm_req_drop(bp, req_qc);
10210 return rc;
10211 }
10212 for (j = 0; j < bp->max_q; j++) {
10213 if (bp->q_ids[j] == queue_id)
10214 bp->pri2cos_idx[i] = queue_idx;
10215 }
10216 }
10217 bp->pri2cos_valid = true;
10218 }
10219 hwrm_req_drop(bp, req_qc);
10220
10221 return rc;
10222 }
10223
10224 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
10225 {
10226 bnxt_hwrm_tunnel_dst_port_free(bp,
10227 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
10228 bnxt_hwrm_tunnel_dst_port_free(bp,
10229 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
10230 }
10231
10232 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
10233 {
10234 int rc, i;
10235 u32 tpa_flags = 0;
10236
10237 if (set_tpa)
10238 tpa_flags = bp->flags & BNXT_FLAG_TPA;
10239 else if (BNXT_NO_FW_ACCESS(bp))
10240 return 0;
10241 for (i = 0; i < bp->nr_vnics; i++) {
10242 rc = bnxt_hwrm_vnic_set_tpa(bp, &bp->vnic_info[i], tpa_flags);
10243 if (rc) {
10244 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
10245 i, rc);
10246 return rc;
10247 }
10248 }
10249 return 0;
10250 }
10251
10252 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
10253 {
10254 int i;
10255
10256 for (i = 0; i < bp->nr_vnics; i++)
10257 bnxt_hwrm_vnic_set_rss(bp, &bp->vnic_info[i], false);
10258 }
10259
10260 static void bnxt_clear_vnic(struct bnxt *bp)
10261 {
10262 if (!bp->vnic_info)
10263 return;
10264
10265 bnxt_hwrm_clear_vnic_filter(bp);
10266 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS)) {
10267 /* clear all RSS setting before free vnic ctx */
10268 bnxt_hwrm_clear_vnic_rss(bp);
10269 bnxt_hwrm_vnic_ctx_free(bp);
10270 }
10271 /* before free the vnic, undo the vnic tpa settings */
10272 if (bp->flags & BNXT_FLAG_TPA)
10273 bnxt_set_tpa(bp, false);
10274 bnxt_hwrm_vnic_free(bp);
10275 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10276 bnxt_hwrm_vnic_ctx_free(bp);
10277 }
10278
10279 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
10280 bool irq_re_init)
10281 {
10282 bnxt_clear_vnic(bp);
10283 bnxt_hwrm_ring_free(bp, close_path);
10284 bnxt_hwrm_ring_grp_free(bp);
10285 if (irq_re_init) {
10286 bnxt_hwrm_stat_ctx_free(bp);
10287 bnxt_hwrm_free_tunnel_ports(bp);
10288 }
10289 }
10290
10291 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
10292 {
10293 struct hwrm_func_cfg_input *req;
10294 u8 evb_mode;
10295 int rc;
10296
10297 if (br_mode == BRIDGE_MODE_VEB)
10298 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
10299 else if (br_mode == BRIDGE_MODE_VEPA)
10300 evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
10301 else
10302 return -EINVAL;
10303
10304 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
10305 if (rc)
10306 return rc;
10307
10308 req->fid = cpu_to_le16(0xffff);
10309 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
10310 req->evb_mode = evb_mode;
10311 return hwrm_req_send(bp, req);
10312 }
10313
10314 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
10315 {
10316 struct hwrm_func_cfg_input *req;
10317 int rc;
10318
10319 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
10320 return 0;
10321
10322 rc = bnxt_hwrm_func_cfg_short_req_init(bp, &req);
10323 if (rc)
10324 return rc;
10325
10326 req->fid = cpu_to_le16(0xffff);
10327 req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
10328 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
10329 if (size == 128)
10330 req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
10331
10332 return hwrm_req_send(bp, req);
10333 }
10334
10335 static int __bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10336 {
10337 int rc;
10338
10339 if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
10340 goto skip_rss_ctx;
10341
10342 /* allocate context for vnic */
10343 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 0);
10344 if (rc) {
10345 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
10346 vnic->vnic_id, rc);
10347 goto vnic_setup_err;
10348 }
10349 bp->rsscos_nr_ctxs++;
10350
10351 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10352 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, 1);
10353 if (rc) {
10354 netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
10355 vnic->vnic_id, rc);
10356 goto vnic_setup_err;
10357 }
10358 bp->rsscos_nr_ctxs++;
10359 }
10360
10361 skip_rss_ctx:
10362 /* configure default vnic, ring grp */
10363 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10364 if (rc) {
10365 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
10366 vnic->vnic_id, rc);
10367 goto vnic_setup_err;
10368 }
10369
10370 /* Enable RSS hashing on vnic */
10371 rc = bnxt_hwrm_vnic_set_rss(bp, vnic, true);
10372 if (rc) {
10373 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
10374 vnic->vnic_id, rc);
10375 goto vnic_setup_err;
10376 }
10377
10378 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
10379 rc = bnxt_hwrm_vnic_set_hds(bp, vnic);
10380 if (rc) {
10381 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
10382 vnic->vnic_id, rc);
10383 }
10384 }
10385
10386 vnic_setup_err:
10387 return rc;
10388 }
10389
10390 int bnxt_hwrm_vnic_update(struct bnxt *bp, struct bnxt_vnic_info *vnic,
10391 u8 valid)
10392 {
10393 struct hwrm_vnic_update_input *req;
10394 int rc;
10395
10396 rc = hwrm_req_init(bp, req, HWRM_VNIC_UPDATE);
10397 if (rc)
10398 return rc;
10399
10400 req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
10401
10402 if (valid & VNIC_UPDATE_REQ_ENABLES_MRU_VALID)
10403 req->mru = cpu_to_le16(vnic->mru);
10404
10405 req->enables = cpu_to_le32(valid);
10406
10407 return hwrm_req_send(bp, req);
10408 }
10409
10410 int bnxt_hwrm_vnic_rss_cfg_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10411 {
10412 int rc;
10413
10414 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true);
10415 if (rc) {
10416 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
10417 vnic->vnic_id, rc);
10418 return rc;
10419 }
10420 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10421 if (rc)
10422 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
10423 vnic->vnic_id, rc);
10424 return rc;
10425 }
10426
10427 int __bnxt_setup_vnic_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10428 {
10429 int rc, i, nr_ctxs;
10430
10431 nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
10432 for (i = 0; i < nr_ctxs; i++) {
10433 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, i);
10434 if (rc) {
10435 netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
10436 vnic->vnic_id, i, rc);
10437 break;
10438 }
10439 bp->rsscos_nr_ctxs++;
10440 }
10441 if (i < nr_ctxs)
10442 return -ENOMEM;
10443
10444 rc = bnxt_hwrm_vnic_rss_cfg_p5(bp, vnic);
10445 if (rc)
10446 return rc;
10447
10448 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
10449 rc = bnxt_hwrm_vnic_set_hds(bp, vnic);
10450 if (rc) {
10451 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
10452 vnic->vnic_id, rc);
10453 }
10454 }
10455 return rc;
10456 }
10457
10458 static int bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic)
10459 {
10460 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10461 return __bnxt_setup_vnic_p5(bp, vnic);
10462 else
10463 return __bnxt_setup_vnic(bp, vnic);
10464 }
10465
10466 static int bnxt_alloc_and_setup_vnic(struct bnxt *bp,
10467 struct bnxt_vnic_info *vnic,
10468 u16 start_rx_ring_idx, int rx_rings)
10469 {
10470 int rc;
10471
10472 rc = bnxt_hwrm_vnic_alloc(bp, vnic, start_rx_ring_idx, rx_rings);
10473 if (rc) {
10474 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
10475 vnic->vnic_id, rc);
10476 return rc;
10477 }
10478 return bnxt_setup_vnic(bp, vnic);
10479 }
10480
10481 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
10482 {
10483 struct bnxt_vnic_info *vnic;
10484 int i, rc = 0;
10485
10486 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) {
10487 vnic = &bp->vnic_info[BNXT_VNIC_NTUPLE];
10488 return bnxt_alloc_and_setup_vnic(bp, vnic, 0, bp->rx_nr_rings);
10489 }
10490
10491 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10492 return 0;
10493
10494 for (i = 0; i < bp->rx_nr_rings; i++) {
10495 u16 vnic_id = i + 1;
10496 u16 ring_id = i;
10497
10498 if (vnic_id >= bp->nr_vnics)
10499 break;
10500
10501 vnic = &bp->vnic_info[vnic_id];
10502 vnic->flags |= BNXT_VNIC_RFS_FLAG;
10503 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
10504 vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
10505 if (bnxt_alloc_and_setup_vnic(bp, &bp->vnic_info[vnic_id], ring_id, 1))
10506 break;
10507 }
10508 return rc;
10509 }
10510
10511 void bnxt_del_one_rss_ctx(struct bnxt *bp, struct bnxt_rss_ctx *rss_ctx,
10512 bool all)
10513 {
10514 struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
10515 struct bnxt_filter_base *usr_fltr, *tmp;
10516 struct bnxt_ntuple_filter *ntp_fltr;
10517 int i;
10518
10519 if (netif_running(bp->dev)) {
10520 bnxt_hwrm_vnic_free_one(bp, &rss_ctx->vnic);
10521 for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++) {
10522 if (vnic->fw_rss_cos_lb_ctx[i] != INVALID_HW_RING_ID)
10523 bnxt_hwrm_vnic_ctx_free_one(bp, vnic, i);
10524 }
10525 }
10526 if (!all)
10527 return;
10528
10529 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list) {
10530 if ((usr_fltr->flags & BNXT_ACT_RSS_CTX) &&
10531 usr_fltr->fw_vnic_id == rss_ctx->index) {
10532 ntp_fltr = container_of(usr_fltr,
10533 struct bnxt_ntuple_filter,
10534 base);
10535 bnxt_hwrm_cfa_ntuple_filter_free(bp, ntp_fltr);
10536 bnxt_del_ntp_filter(bp, ntp_fltr);
10537 bnxt_del_one_usr_fltr(bp, usr_fltr);
10538 }
10539 }
10540
10541 if (vnic->rss_table)
10542 dma_free_coherent(&bp->pdev->dev, vnic->rss_table_size,
10543 vnic->rss_table,
10544 vnic->rss_table_dma_addr);
10545 bp->num_rss_ctx--;
10546 }
10547
10548 static void bnxt_hwrm_realloc_rss_ctx_vnic(struct bnxt *bp)
10549 {
10550 bool set_tpa = !!(bp->flags & BNXT_FLAG_TPA);
10551 struct ethtool_rxfh_context *ctx;
10552 unsigned long context;
10553
10554 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) {
10555 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx);
10556 struct bnxt_vnic_info *vnic = &rss_ctx->vnic;
10557
10558 if (bnxt_hwrm_vnic_alloc(bp, vnic, 0, bp->rx_nr_rings) ||
10559 bnxt_hwrm_vnic_set_tpa(bp, vnic, set_tpa) ||
10560 __bnxt_setup_vnic_p5(bp, vnic)) {
10561 netdev_err(bp->dev, "Failed to restore RSS ctx %d\n",
10562 rss_ctx->index);
10563 bnxt_del_one_rss_ctx(bp, rss_ctx, true);
10564 ethtool_rxfh_context_lost(bp->dev, rss_ctx->index);
10565 }
10566 }
10567 }
10568
10569 static void bnxt_clear_rss_ctxs(struct bnxt *bp)
10570 {
10571 struct ethtool_rxfh_context *ctx;
10572 unsigned long context;
10573
10574 xa_for_each(&bp->dev->ethtool->rss_ctx, context, ctx) {
10575 struct bnxt_rss_ctx *rss_ctx = ethtool_rxfh_context_priv(ctx);
10576
10577 bnxt_del_one_rss_ctx(bp, rss_ctx, false);
10578 }
10579 }
10580
10581 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
10582 static bool bnxt_promisc_ok(struct bnxt *bp)
10583 {
10584 #ifdef CONFIG_BNXT_SRIOV
10585 if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
10586 return false;
10587 #endif
10588 return true;
10589 }
10590
10591 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
10592 {
10593 struct bnxt_vnic_info *vnic = &bp->vnic_info[1];
10594 unsigned int rc = 0;
10595
10596 rc = bnxt_hwrm_vnic_alloc(bp, vnic, bp->rx_nr_rings - 1, 1);
10597 if (rc) {
10598 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
10599 rc);
10600 return rc;
10601 }
10602
10603 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
10604 if (rc) {
10605 netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
10606 rc);
10607 return rc;
10608 }
10609 return rc;
10610 }
10611
10612 static int bnxt_cfg_rx_mode(struct bnxt *);
10613 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
10614
10615 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
10616 {
10617 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
10618 int rc = 0;
10619 unsigned int rx_nr_rings = bp->rx_nr_rings;
10620
10621 if (irq_re_init) {
10622 rc = bnxt_hwrm_stat_ctx_alloc(bp);
10623 if (rc) {
10624 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
10625 rc);
10626 goto err_out;
10627 }
10628 }
10629
10630 rc = bnxt_hwrm_ring_alloc(bp);
10631 if (rc) {
10632 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
10633 goto err_out;
10634 }
10635
10636 rc = bnxt_hwrm_ring_grp_alloc(bp);
10637 if (rc) {
10638 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
10639 goto err_out;
10640 }
10641
10642 if (BNXT_CHIP_TYPE_NITRO_A0(bp))
10643 rx_nr_rings--;
10644
10645 /* default vnic 0 */
10646 rc = bnxt_hwrm_vnic_alloc(bp, vnic, 0, rx_nr_rings);
10647 if (rc) {
10648 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
10649 goto err_out;
10650 }
10651
10652 if (BNXT_VF(bp))
10653 bnxt_hwrm_func_qcfg(bp);
10654
10655 rc = bnxt_setup_vnic(bp, vnic);
10656 if (rc)
10657 goto err_out;
10658 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
10659 bnxt_hwrm_update_rss_hash_cfg(bp);
10660
10661 if (bp->flags & BNXT_FLAG_RFS) {
10662 rc = bnxt_alloc_rfs_vnics(bp);
10663 if (rc)
10664 goto err_out;
10665 }
10666
10667 if (bp->flags & BNXT_FLAG_TPA) {
10668 rc = bnxt_set_tpa(bp, true);
10669 if (rc)
10670 goto err_out;
10671 }
10672
10673 if (BNXT_VF(bp))
10674 bnxt_update_vf_mac(bp);
10675
10676 /* Filter for default vnic 0 */
10677 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
10678 if (rc) {
10679 if (BNXT_VF(bp) && rc == -ENODEV)
10680 netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
10681 else
10682 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
10683 goto err_out;
10684 }
10685 vnic->uc_filter_count = 1;
10686
10687 vnic->rx_mask = 0;
10688 if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
10689 goto skip_rx_mask;
10690
10691 if (bp->dev->flags & IFF_BROADCAST)
10692 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10693
10694 if (bp->dev->flags & IFF_PROMISC)
10695 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10696
10697 if (bp->dev->flags & IFF_ALLMULTI) {
10698 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10699 vnic->mc_list_count = 0;
10700 } else if (bp->dev->flags & IFF_MULTICAST) {
10701 u32 mask = 0;
10702
10703 bnxt_mc_list_updated(bp, &mask);
10704 vnic->rx_mask |= mask;
10705 }
10706
10707 rc = bnxt_cfg_rx_mode(bp);
10708 if (rc)
10709 goto err_out;
10710
10711 skip_rx_mask:
10712 rc = bnxt_hwrm_set_coal(bp);
10713 if (rc)
10714 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
10715 rc);
10716
10717 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
10718 rc = bnxt_setup_nitroa0_vnic(bp);
10719 if (rc)
10720 netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
10721 rc);
10722 }
10723
10724 if (BNXT_VF(bp)) {
10725 bnxt_hwrm_func_qcfg(bp);
10726 netdev_update_features(bp->dev);
10727 }
10728
10729 return 0;
10730
10731 err_out:
10732 bnxt_hwrm_resource_free(bp, 0, true);
10733
10734 return rc;
10735 }
10736
10737 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
10738 {
10739 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
10740 return 0;
10741 }
10742
10743 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
10744 {
10745 bnxt_init_cp_rings(bp);
10746 bnxt_init_rx_rings(bp);
10747 bnxt_init_tx_rings(bp);
10748 bnxt_init_ring_grps(bp, irq_re_init);
10749 bnxt_init_vnics(bp);
10750
10751 return bnxt_init_chip(bp, irq_re_init);
10752 }
10753
10754 static int bnxt_set_real_num_queues(struct bnxt *bp)
10755 {
10756 int rc;
10757 struct net_device *dev = bp->dev;
10758
10759 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
10760 bp->tx_nr_rings_xdp);
10761 if (rc)
10762 return rc;
10763
10764 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
10765 if (rc)
10766 return rc;
10767
10768 #ifdef CONFIG_RFS_ACCEL
10769 if (bp->flags & BNXT_FLAG_RFS)
10770 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
10771 #endif
10772
10773 return rc;
10774 }
10775
10776 static int __bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
10777 bool shared)
10778 {
10779 int _rx = *rx, _tx = *tx;
10780
10781 if (shared) {
10782 *rx = min_t(int, _rx, max);
10783 *tx = min_t(int, _tx, max);
10784 } else {
10785 if (max < 2)
10786 return -ENOMEM;
10787
10788 while (_rx + _tx > max) {
10789 if (_rx > _tx && _rx > 1)
10790 _rx--;
10791 else if (_tx > 1)
10792 _tx--;
10793 }
10794 *rx = _rx;
10795 *tx = _tx;
10796 }
10797 return 0;
10798 }
10799
10800 static int __bnxt_num_tx_to_cp(struct bnxt *bp, int tx, int tx_sets, int tx_xdp)
10801 {
10802 return (tx - tx_xdp) / tx_sets + tx_xdp;
10803 }
10804
10805 int bnxt_num_tx_to_cp(struct bnxt *bp, int tx)
10806 {
10807 int tcs = bp->num_tc;
10808
10809 if (!tcs)
10810 tcs = 1;
10811 return __bnxt_num_tx_to_cp(bp, tx, tcs, bp->tx_nr_rings_xdp);
10812 }
10813
10814 static int bnxt_num_cp_to_tx(struct bnxt *bp, int tx_cp)
10815 {
10816 int tcs = bp->num_tc;
10817
10818 return (tx_cp - bp->tx_nr_rings_xdp) * tcs +
10819 bp->tx_nr_rings_xdp;
10820 }
10821
10822 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
10823 bool sh)
10824 {
10825 int tx_cp = bnxt_num_tx_to_cp(bp, *tx);
10826
10827 if (tx_cp != *tx) {
10828 int tx_saved = tx_cp, rc;
10829
10830 rc = __bnxt_trim_rings(bp, rx, &tx_cp, max, sh);
10831 if (rc)
10832 return rc;
10833 if (tx_cp != tx_saved)
10834 *tx = bnxt_num_cp_to_tx(bp, tx_cp);
10835 return 0;
10836 }
10837 return __bnxt_trim_rings(bp, rx, tx, max, sh);
10838 }
10839
10840 static void bnxt_setup_msix(struct bnxt *bp)
10841 {
10842 const int len = sizeof(bp->irq_tbl[0].name);
10843 struct net_device *dev = bp->dev;
10844 int tcs, i;
10845
10846 tcs = bp->num_tc;
10847 if (tcs) {
10848 int i, off, count;
10849
10850 for (i = 0; i < tcs; i++) {
10851 count = bp->tx_nr_rings_per_tc;
10852 off = BNXT_TC_TO_RING_BASE(bp, i);
10853 netdev_set_tc_queue(dev, i, count, off);
10854 }
10855 }
10856
10857 for (i = 0; i < bp->cp_nr_rings; i++) {
10858 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
10859 char *attr;
10860
10861 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
10862 attr = "TxRx";
10863 else if (i < bp->rx_nr_rings)
10864 attr = "rx";
10865 else
10866 attr = "tx";
10867
10868 snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
10869 attr, i);
10870 bp->irq_tbl[map_idx].handler = bnxt_msix;
10871 }
10872 }
10873
10874 static int bnxt_init_int_mode(struct bnxt *bp);
10875
10876 static int bnxt_change_msix(struct bnxt *bp, int total)
10877 {
10878 struct msi_map map;
10879 int i;
10880
10881 /* add MSIX to the end if needed */
10882 for (i = bp->total_irqs; i < total; i++) {
10883 map = pci_msix_alloc_irq_at(bp->pdev, i, NULL);
10884 if (map.index < 0)
10885 return bp->total_irqs;
10886 bp->irq_tbl[i].vector = map.virq;
10887 bp->total_irqs++;
10888 }
10889
10890 /* trim MSIX from the end if needed */
10891 for (i = bp->total_irqs; i > total; i--) {
10892 map.index = i - 1;
10893 map.virq = bp->irq_tbl[i - 1].vector;
10894 pci_msix_free_irq(bp->pdev, map);
10895 bp->total_irqs--;
10896 }
10897 return bp->total_irqs;
10898 }
10899
10900 static int bnxt_setup_int_mode(struct bnxt *bp)
10901 {
10902 int rc;
10903
10904 if (!bp->irq_tbl) {
10905 rc = bnxt_init_int_mode(bp);
10906 if (rc || !bp->irq_tbl)
10907 return rc ?: -ENODEV;
10908 }
10909
10910 bnxt_setup_msix(bp);
10911
10912 rc = bnxt_set_real_num_queues(bp);
10913 return rc;
10914 }
10915
10916 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
10917 {
10918 return bp->hw_resc.max_rsscos_ctxs;
10919 }
10920
10921 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
10922 {
10923 return bp->hw_resc.max_vnics;
10924 }
10925
10926 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
10927 {
10928 return bp->hw_resc.max_stat_ctxs;
10929 }
10930
10931 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
10932 {
10933 return bp->hw_resc.max_cp_rings;
10934 }
10935
10936 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
10937 {
10938 unsigned int cp = bp->hw_resc.max_cp_rings;
10939
10940 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
10941 cp -= bnxt_get_ulp_msix_num(bp);
10942
10943 return cp;
10944 }
10945
10946 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
10947 {
10948 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10949
10950 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10951 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
10952
10953 return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
10954 }
10955
10956 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
10957 {
10958 bp->hw_resc.max_irqs = max_irqs;
10959 }
10960
10961 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
10962 {
10963 unsigned int cp;
10964
10965 cp = bnxt_get_max_func_cp_rings_for_en(bp);
10966 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
10967 return cp - bp->rx_nr_rings - bp->tx_nr_rings;
10968 else
10969 return cp - bp->cp_nr_rings;
10970 }
10971
10972 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
10973 {
10974 return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
10975 }
10976
10977 static int bnxt_get_avail_msix(struct bnxt *bp, int num)
10978 {
10979 int max_irq = bnxt_get_max_func_irqs(bp);
10980 int total_req = bp->cp_nr_rings + num;
10981
10982 if (max_irq < total_req) {
10983 num = max_irq - bp->cp_nr_rings;
10984 if (num <= 0)
10985 return 0;
10986 }
10987 return num;
10988 }
10989
10990 static int bnxt_get_num_msix(struct bnxt *bp)
10991 {
10992 if (!BNXT_NEW_RM(bp))
10993 return bnxt_get_max_func_irqs(bp);
10994
10995 return bnxt_nq_rings_in_use(bp);
10996 }
10997
10998 static int bnxt_init_int_mode(struct bnxt *bp)
10999 {
11000 int i, total_vecs, max, rc = 0, min = 1, ulp_msix, tx_cp, tbl_size;
11001
11002 total_vecs = bnxt_get_num_msix(bp);
11003 max = bnxt_get_max_func_irqs(bp);
11004 if (total_vecs > max)
11005 total_vecs = max;
11006
11007 if (!total_vecs)
11008 return 0;
11009
11010 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
11011 min = 2;
11012
11013 total_vecs = pci_alloc_irq_vectors(bp->pdev, min, total_vecs,
11014 PCI_IRQ_MSIX);
11015 ulp_msix = bnxt_get_ulp_msix_num(bp);
11016 if (total_vecs < 0 || total_vecs < ulp_msix) {
11017 rc = -ENODEV;
11018 goto msix_setup_exit;
11019 }
11020
11021 tbl_size = total_vecs;
11022 if (pci_msix_can_alloc_dyn(bp->pdev))
11023 tbl_size = max;
11024 bp->irq_tbl = kcalloc(tbl_size, sizeof(*bp->irq_tbl), GFP_KERNEL);
11025 if (bp->irq_tbl) {
11026 for (i = 0; i < total_vecs; i++)
11027 bp->irq_tbl[i].vector = pci_irq_vector(bp->pdev, i);
11028
11029 bp->total_irqs = total_vecs;
11030 /* Trim rings based upon num of vectors allocated */
11031 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
11032 total_vecs - ulp_msix, min == 1);
11033 if (rc)
11034 goto msix_setup_exit;
11035
11036 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
11037 bp->cp_nr_rings = (min == 1) ?
11038 max_t(int, tx_cp, bp->rx_nr_rings) :
11039 tx_cp + bp->rx_nr_rings;
11040
11041 } else {
11042 rc = -ENOMEM;
11043 goto msix_setup_exit;
11044 }
11045 return 0;
11046
11047 msix_setup_exit:
11048 netdev_err(bp->dev, "bnxt_init_int_mode err: %x\n", rc);
11049 kfree(bp->irq_tbl);
11050 bp->irq_tbl = NULL;
11051 pci_free_irq_vectors(bp->pdev);
11052 return rc;
11053 }
11054
11055 static void bnxt_clear_int_mode(struct bnxt *bp)
11056 {
11057 pci_free_irq_vectors(bp->pdev);
11058
11059 kfree(bp->irq_tbl);
11060 bp->irq_tbl = NULL;
11061 }
11062
11063 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
11064 {
11065 bool irq_cleared = false;
11066 bool irq_change = false;
11067 int tcs = bp->num_tc;
11068 int irqs_required;
11069 int rc;
11070
11071 if (!bnxt_need_reserve_rings(bp))
11072 return 0;
11073
11074 if (BNXT_NEW_RM(bp) && !bnxt_ulp_registered(bp->edev)) {
11075 int ulp_msix = bnxt_get_avail_msix(bp, bp->ulp_num_msix_want);
11076
11077 if (ulp_msix > bp->ulp_num_msix_want)
11078 ulp_msix = bp->ulp_num_msix_want;
11079 irqs_required = ulp_msix + bp->cp_nr_rings;
11080 } else {
11081 irqs_required = bnxt_get_num_msix(bp);
11082 }
11083
11084 if (irq_re_init && BNXT_NEW_RM(bp) && irqs_required != bp->total_irqs) {
11085 irq_change = true;
11086 if (!pci_msix_can_alloc_dyn(bp->pdev)) {
11087 bnxt_ulp_irq_stop(bp);
11088 bnxt_clear_int_mode(bp);
11089 irq_cleared = true;
11090 }
11091 }
11092 rc = __bnxt_reserve_rings(bp);
11093 if (irq_cleared) {
11094 if (!rc)
11095 rc = bnxt_init_int_mode(bp);
11096 bnxt_ulp_irq_restart(bp, rc);
11097 } else if (irq_change && !rc) {
11098 if (bnxt_change_msix(bp, irqs_required) != irqs_required)
11099 rc = -ENOSPC;
11100 }
11101 if (rc) {
11102 netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
11103 return rc;
11104 }
11105 if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
11106 bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
11107 netdev_err(bp->dev, "tx ring reservation failure\n");
11108 netdev_reset_tc(bp->dev);
11109 bp->num_tc = 0;
11110 if (bp->tx_nr_rings_xdp)
11111 bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
11112 else
11113 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11114 return -ENOMEM;
11115 }
11116 return 0;
11117 }
11118
11119 static void bnxt_free_irq(struct bnxt *bp)
11120 {
11121 struct bnxt_irq *irq;
11122 int i;
11123
11124 #ifdef CONFIG_RFS_ACCEL
11125 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
11126 bp->dev->rx_cpu_rmap = NULL;
11127 #endif
11128 if (!bp->irq_tbl || !bp->bnapi)
11129 return;
11130
11131 for (i = 0; i < bp->cp_nr_rings; i++) {
11132 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
11133
11134 irq = &bp->irq_tbl[map_idx];
11135 if (irq->requested) {
11136 if (irq->have_cpumask) {
11137 irq_update_affinity_hint(irq->vector, NULL);
11138 free_cpumask_var(irq->cpu_mask);
11139 irq->have_cpumask = 0;
11140 }
11141 free_irq(irq->vector, bp->bnapi[i]);
11142 }
11143
11144 irq->requested = 0;
11145 }
11146 }
11147
11148 static int bnxt_request_irq(struct bnxt *bp)
11149 {
11150 int i, j, rc = 0;
11151 unsigned long flags = 0;
11152 #ifdef CONFIG_RFS_ACCEL
11153 struct cpu_rmap *rmap;
11154 #endif
11155
11156 rc = bnxt_setup_int_mode(bp);
11157 if (rc) {
11158 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
11159 rc);
11160 return rc;
11161 }
11162 #ifdef CONFIG_RFS_ACCEL
11163 rmap = bp->dev->rx_cpu_rmap;
11164 #endif
11165 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
11166 int map_idx = bnxt_cp_num_to_irq_num(bp, i);
11167 struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
11168
11169 #ifdef CONFIG_RFS_ACCEL
11170 if (rmap && bp->bnapi[i]->rx_ring) {
11171 rc = irq_cpu_rmap_add(rmap, irq->vector);
11172 if (rc)
11173 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
11174 j);
11175 j++;
11176 }
11177 #endif
11178 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
11179 bp->bnapi[i]);
11180 if (rc)
11181 break;
11182
11183 netif_napi_set_irq(&bp->bnapi[i]->napi, irq->vector);
11184 irq->requested = 1;
11185
11186 if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
11187 int numa_node = dev_to_node(&bp->pdev->dev);
11188
11189 irq->have_cpumask = 1;
11190 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
11191 irq->cpu_mask);
11192 rc = irq_update_affinity_hint(irq->vector, irq->cpu_mask);
11193 if (rc) {
11194 netdev_warn(bp->dev,
11195 "Update affinity hint failed, IRQ = %d\n",
11196 irq->vector);
11197 break;
11198 }
11199 }
11200 }
11201 return rc;
11202 }
11203
11204 static void bnxt_del_napi(struct bnxt *bp)
11205 {
11206 int i;
11207
11208 if (!bp->bnapi)
11209 return;
11210
11211 for (i = 0; i < bp->rx_nr_rings; i++)
11212 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL);
11213 for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++)
11214 netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_TX, NULL);
11215
11216 for (i = 0; i < bp->cp_nr_rings; i++) {
11217 struct bnxt_napi *bnapi = bp->bnapi[i];
11218
11219 __netif_napi_del(&bnapi->napi);
11220 }
11221 /* We called __netif_napi_del(), we need
11222 * to respect an RCU grace period before freeing napi structures.
11223 */
11224 synchronize_net();
11225 }
11226
11227 static void bnxt_init_napi(struct bnxt *bp)
11228 {
11229 int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
11230 unsigned int cp_nr_rings = bp->cp_nr_rings;
11231 struct bnxt_napi *bnapi;
11232 int i;
11233
11234 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
11235 poll_fn = bnxt_poll_p5;
11236 else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
11237 cp_nr_rings--;
11238 for (i = 0; i < cp_nr_rings; i++) {
11239 bnapi = bp->bnapi[i];
11240 netif_napi_add_config(bp->dev, &bnapi->napi, poll_fn,
11241 bnapi->index);
11242 }
11243 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
11244 bnapi = bp->bnapi[cp_nr_rings];
11245 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll_nitroa0);
11246 }
11247 }
11248
11249 static void bnxt_disable_napi(struct bnxt *bp)
11250 {
11251 int i;
11252
11253 if (!bp->bnapi ||
11254 test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
11255 return;
11256
11257 for (i = 0; i < bp->cp_nr_rings; i++) {
11258 struct bnxt_napi *bnapi = bp->bnapi[i];
11259 struct bnxt_cp_ring_info *cpr;
11260
11261 cpr = &bnapi->cp_ring;
11262 if (bnapi->tx_fault)
11263 cpr->sw_stats->tx.tx_resets++;
11264 if (bnapi->in_reset)
11265 cpr->sw_stats->rx.rx_resets++;
11266 napi_disable(&bnapi->napi);
11267 if (bnapi->rx_ring)
11268 cancel_work_sync(&cpr->dim.work);
11269 }
11270 }
11271
11272 static void bnxt_enable_napi(struct bnxt *bp)
11273 {
11274 int i;
11275
11276 clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
11277 for (i = 0; i < bp->cp_nr_rings; i++) {
11278 struct bnxt_napi *bnapi = bp->bnapi[i];
11279 struct bnxt_cp_ring_info *cpr;
11280
11281 bnapi->tx_fault = 0;
11282
11283 cpr = &bnapi->cp_ring;
11284 bnapi->in_reset = false;
11285
11286 if (bnapi->rx_ring) {
11287 INIT_WORK(&cpr->dim.work, bnxt_dim_work);
11288 cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
11289 }
11290 napi_enable(&bnapi->napi);
11291 }
11292 }
11293
11294 void bnxt_tx_disable(struct bnxt *bp)
11295 {
11296 int i;
11297 struct bnxt_tx_ring_info *txr;
11298
11299 if (bp->tx_ring) {
11300 for (i = 0; i < bp->tx_nr_rings; i++) {
11301 txr = &bp->tx_ring[i];
11302 WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
11303 }
11304 }
11305 /* Make sure napi polls see @dev_state change */
11306 synchronize_net();
11307 /* Drop carrier first to prevent TX timeout */
11308 netif_carrier_off(bp->dev);
11309 /* Stop all TX queues */
11310 netif_tx_disable(bp->dev);
11311 }
11312
11313 void bnxt_tx_enable(struct bnxt *bp)
11314 {
11315 int i;
11316 struct bnxt_tx_ring_info *txr;
11317
11318 for (i = 0; i < bp->tx_nr_rings; i++) {
11319 txr = &bp->tx_ring[i];
11320 WRITE_ONCE(txr->dev_state, 0);
11321 }
11322 /* Make sure napi polls see @dev_state change */
11323 synchronize_net();
11324 netif_tx_wake_all_queues(bp->dev);
11325 if (BNXT_LINK_IS_UP(bp))
11326 netif_carrier_on(bp->dev);
11327 }
11328
11329 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
11330 {
11331 u8 active_fec = link_info->active_fec_sig_mode &
11332 PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
11333
11334 switch (active_fec) {
11335 default:
11336 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
11337 return "None";
11338 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
11339 return "Clause 74 BaseR";
11340 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
11341 return "Clause 91 RS(528,514)";
11342 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
11343 return "Clause 91 RS544_1XN";
11344 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
11345 return "Clause 91 RS(544,514)";
11346 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
11347 return "Clause 91 RS272_1XN";
11348 case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
11349 return "Clause 91 RS(272,257)";
11350 }
11351 }
11352
11353 void bnxt_report_link(struct bnxt *bp)
11354 {
11355 if (BNXT_LINK_IS_UP(bp)) {
11356 const char *signal = "";
11357 const char *flow_ctrl;
11358 const char *duplex;
11359 u32 speed;
11360 u16 fec;
11361
11362 netif_carrier_on(bp->dev);
11363 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
11364 if (speed == SPEED_UNKNOWN) {
11365 netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
11366 return;
11367 }
11368 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
11369 duplex = "full";
11370 else
11371 duplex = "half";
11372 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
11373 flow_ctrl = "ON - receive & transmit";
11374 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
11375 flow_ctrl = "ON - transmit";
11376 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
11377 flow_ctrl = "ON - receive";
11378 else
11379 flow_ctrl = "none";
11380 if (bp->link_info.phy_qcfg_resp.option_flags &
11381 PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
11382 u8 sig_mode = bp->link_info.active_fec_sig_mode &
11383 PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
11384 switch (sig_mode) {
11385 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
11386 signal = "(NRZ) ";
11387 break;
11388 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
11389 signal = "(PAM4 56Gbps) ";
11390 break;
11391 case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112:
11392 signal = "(PAM4 112Gbps) ";
11393 break;
11394 default:
11395 break;
11396 }
11397 }
11398 netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
11399 speed, signal, duplex, flow_ctrl);
11400 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
11401 netdev_info(bp->dev, "EEE is %s\n",
11402 bp->eee.eee_active ? "active" :
11403 "not active");
11404 fec = bp->link_info.fec_cfg;
11405 if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
11406 netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
11407 (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
11408 bnxt_report_fec(&bp->link_info));
11409 } else {
11410 netif_carrier_off(bp->dev);
11411 netdev_err(bp->dev, "NIC Link is Down\n");
11412 }
11413 }
11414
11415 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
11416 {
11417 if (!resp->supported_speeds_auto_mode &&
11418 !resp->supported_speeds_force_mode &&
11419 !resp->supported_pam4_speeds_auto_mode &&
11420 !resp->supported_pam4_speeds_force_mode &&
11421 !resp->supported_speeds2_auto_mode &&
11422 !resp->supported_speeds2_force_mode)
11423 return true;
11424 return false;
11425 }
11426
11427 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
11428 {
11429 struct bnxt_link_info *link_info = &bp->link_info;
11430 struct hwrm_port_phy_qcaps_output *resp;
11431 struct hwrm_port_phy_qcaps_input *req;
11432 int rc = 0;
11433
11434 if (bp->hwrm_spec_code < 0x10201)
11435 return 0;
11436
11437 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
11438 if (rc)
11439 return rc;
11440
11441 resp = hwrm_req_hold(bp, req);
11442 rc = hwrm_req_send(bp, req);
11443 if (rc)
11444 goto hwrm_phy_qcaps_exit;
11445
11446 bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
11447 if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
11448 struct ethtool_keee *eee = &bp->eee;
11449 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
11450
11451 _bnxt_fw_to_linkmode(eee->supported, fw_speeds);
11452 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
11453 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
11454 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
11455 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
11456 }
11457
11458 if (bp->hwrm_spec_code >= 0x10a01) {
11459 if (bnxt_phy_qcaps_no_speed(resp)) {
11460 link_info->phy_state = BNXT_PHY_STATE_DISABLED;
11461 netdev_warn(bp->dev, "Ethernet link disabled\n");
11462 } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
11463 link_info->phy_state = BNXT_PHY_STATE_ENABLED;
11464 netdev_info(bp->dev, "Ethernet link enabled\n");
11465 /* Phy re-enabled, reprobe the speeds */
11466 link_info->support_auto_speeds = 0;
11467 link_info->support_pam4_auto_speeds = 0;
11468 link_info->support_auto_speeds2 = 0;
11469 }
11470 }
11471 if (resp->supported_speeds_auto_mode)
11472 link_info->support_auto_speeds =
11473 le16_to_cpu(resp->supported_speeds_auto_mode);
11474 if (resp->supported_pam4_speeds_auto_mode)
11475 link_info->support_pam4_auto_speeds =
11476 le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
11477 if (resp->supported_speeds2_auto_mode)
11478 link_info->support_auto_speeds2 =
11479 le16_to_cpu(resp->supported_speeds2_auto_mode);
11480
11481 bp->port_count = resp->port_cnt;
11482
11483 hwrm_phy_qcaps_exit:
11484 hwrm_req_drop(bp, req);
11485 return rc;
11486 }
11487
11488 static bool bnxt_support_dropped(u16 advertising, u16 supported)
11489 {
11490 u16 diff = advertising ^ supported;
11491
11492 return ((supported | diff) != supported);
11493 }
11494
11495 static bool bnxt_support_speed_dropped(struct bnxt_link_info *link_info)
11496 {
11497 struct bnxt *bp = container_of(link_info, struct bnxt, link_info);
11498
11499 /* Check if any advertised speeds are no longer supported. The caller
11500 * holds the link_lock mutex, so we can modify link_info settings.
11501 */
11502 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11503 if (bnxt_support_dropped(link_info->advertising,
11504 link_info->support_auto_speeds2)) {
11505 link_info->advertising = link_info->support_auto_speeds2;
11506 return true;
11507 }
11508 return false;
11509 }
11510 if (bnxt_support_dropped(link_info->advertising,
11511 link_info->support_auto_speeds)) {
11512 link_info->advertising = link_info->support_auto_speeds;
11513 return true;
11514 }
11515 if (bnxt_support_dropped(link_info->advertising_pam4,
11516 link_info->support_pam4_auto_speeds)) {
11517 link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
11518 return true;
11519 }
11520 return false;
11521 }
11522
11523 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
11524 {
11525 struct bnxt_link_info *link_info = &bp->link_info;
11526 struct hwrm_port_phy_qcfg_output *resp;
11527 struct hwrm_port_phy_qcfg_input *req;
11528 u8 link_state = link_info->link_state;
11529 bool support_changed;
11530 int rc;
11531
11532 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
11533 if (rc)
11534 return rc;
11535
11536 resp = hwrm_req_hold(bp, req);
11537 rc = hwrm_req_send(bp, req);
11538 if (rc) {
11539 hwrm_req_drop(bp, req);
11540 if (BNXT_VF(bp) && rc == -ENODEV) {
11541 netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
11542 rc = 0;
11543 }
11544 return rc;
11545 }
11546
11547 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
11548 link_info->phy_link_status = resp->link;
11549 link_info->duplex = resp->duplex_cfg;
11550 if (bp->hwrm_spec_code >= 0x10800)
11551 link_info->duplex = resp->duplex_state;
11552 link_info->pause = resp->pause;
11553 link_info->auto_mode = resp->auto_mode;
11554 link_info->auto_pause_setting = resp->auto_pause;
11555 link_info->lp_pause = resp->link_partner_adv_pause;
11556 link_info->force_pause_setting = resp->force_pause;
11557 link_info->duplex_setting = resp->duplex_cfg;
11558 if (link_info->phy_link_status == BNXT_LINK_LINK) {
11559 link_info->link_speed = le16_to_cpu(resp->link_speed);
11560 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2)
11561 link_info->active_lanes = resp->active_lanes;
11562 } else {
11563 link_info->link_speed = 0;
11564 link_info->active_lanes = 0;
11565 }
11566 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
11567 link_info->force_pam4_link_speed =
11568 le16_to_cpu(resp->force_pam4_link_speed);
11569 link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2);
11570 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
11571 link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
11572 link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2);
11573 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
11574 link_info->auto_pam4_link_speeds =
11575 le16_to_cpu(resp->auto_pam4_link_speed_mask);
11576 link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2);
11577 link_info->lp_auto_link_speeds =
11578 le16_to_cpu(resp->link_partner_adv_speeds);
11579 link_info->lp_auto_pam4_link_speeds =
11580 resp->link_partner_pam4_adv_speeds;
11581 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
11582 link_info->phy_ver[0] = resp->phy_maj;
11583 link_info->phy_ver[1] = resp->phy_min;
11584 link_info->phy_ver[2] = resp->phy_bld;
11585 link_info->media_type = resp->media_type;
11586 link_info->phy_type = resp->phy_type;
11587 link_info->transceiver = resp->xcvr_pkg_type;
11588 link_info->phy_addr = resp->eee_config_phy_addr &
11589 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
11590 link_info->module_status = resp->module_status;
11591
11592 if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
11593 struct ethtool_keee *eee = &bp->eee;
11594 u16 fw_speeds;
11595
11596 eee->eee_active = 0;
11597 if (resp->eee_config_phy_addr &
11598 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
11599 eee->eee_active = 1;
11600 fw_speeds = le16_to_cpu(
11601 resp->link_partner_adv_eee_link_speed_mask);
11602 _bnxt_fw_to_linkmode(eee->lp_advertised, fw_speeds);
11603 }
11604
11605 /* Pull initial EEE config */
11606 if (!chng_link_state) {
11607 if (resp->eee_config_phy_addr &
11608 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
11609 eee->eee_enabled = 1;
11610
11611 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
11612 _bnxt_fw_to_linkmode(eee->advertised, fw_speeds);
11613
11614 if (resp->eee_config_phy_addr &
11615 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
11616 __le32 tmr;
11617
11618 eee->tx_lpi_enabled = 1;
11619 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
11620 eee->tx_lpi_timer = le32_to_cpu(tmr) &
11621 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
11622 }
11623 }
11624 }
11625
11626 link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
11627 if (bp->hwrm_spec_code >= 0x10504) {
11628 link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
11629 link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
11630 }
11631 /* TODO: need to add more logic to report VF link */
11632 if (chng_link_state) {
11633 if (link_info->phy_link_status == BNXT_LINK_LINK)
11634 link_info->link_state = BNXT_LINK_STATE_UP;
11635 else
11636 link_info->link_state = BNXT_LINK_STATE_DOWN;
11637 if (link_state != link_info->link_state)
11638 bnxt_report_link(bp);
11639 } else {
11640 /* always link down if not require to update link state */
11641 link_info->link_state = BNXT_LINK_STATE_DOWN;
11642 }
11643 hwrm_req_drop(bp, req);
11644
11645 if (!BNXT_PHY_CFG_ABLE(bp))
11646 return 0;
11647
11648 support_changed = bnxt_support_speed_dropped(link_info);
11649 if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
11650 bnxt_hwrm_set_link_setting(bp, true, false);
11651 return 0;
11652 }
11653
11654 static void bnxt_get_port_module_status(struct bnxt *bp)
11655 {
11656 struct bnxt_link_info *link_info = &bp->link_info;
11657 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
11658 u8 module_status;
11659
11660 if (bnxt_update_link(bp, true))
11661 return;
11662
11663 module_status = link_info->module_status;
11664 switch (module_status) {
11665 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
11666 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
11667 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
11668 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
11669 bp->pf.port_id);
11670 if (bp->hwrm_spec_code >= 0x10201) {
11671 netdev_warn(bp->dev, "Module part number %s\n",
11672 resp->phy_vendor_partnumber);
11673 }
11674 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
11675 netdev_warn(bp->dev, "TX is disabled\n");
11676 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
11677 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
11678 }
11679 }
11680
11681 static void
11682 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
11683 {
11684 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
11685 if (bp->hwrm_spec_code >= 0x10201)
11686 req->auto_pause =
11687 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
11688 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
11689 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
11690 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
11691 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
11692 req->enables |=
11693 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
11694 } else {
11695 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
11696 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
11697 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
11698 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
11699 req->enables |=
11700 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
11701 if (bp->hwrm_spec_code >= 0x10201) {
11702 req->auto_pause = req->force_pause;
11703 req->enables |= cpu_to_le32(
11704 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
11705 }
11706 }
11707 }
11708
11709 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
11710 {
11711 if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
11712 req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
11713 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11714 req->enables |=
11715 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK);
11716 req->auto_link_speeds2_mask = cpu_to_le16(bp->link_info.advertising);
11717 } else if (bp->link_info.advertising) {
11718 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
11719 req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
11720 }
11721 if (bp->link_info.advertising_pam4) {
11722 req->enables |=
11723 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
11724 req->auto_link_pam4_speed_mask =
11725 cpu_to_le16(bp->link_info.advertising_pam4);
11726 }
11727 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
11728 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
11729 } else {
11730 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
11731 if (bp->phy_flags & BNXT_PHY_FL_SPEEDS2) {
11732 req->force_link_speeds2 = cpu_to_le16(bp->link_info.req_link_speed);
11733 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2);
11734 netif_info(bp, link, bp->dev, "Forcing FW speed2: %d\n",
11735 (u32)bp->link_info.req_link_speed);
11736 } else if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
11737 req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
11738 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
11739 } else {
11740 req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
11741 }
11742 }
11743
11744 /* tell chimp that the setting takes effect immediately */
11745 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
11746 }
11747
11748 int bnxt_hwrm_set_pause(struct bnxt *bp)
11749 {
11750 struct hwrm_port_phy_cfg_input *req;
11751 int rc;
11752
11753 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11754 if (rc)
11755 return rc;
11756
11757 bnxt_hwrm_set_pause_common(bp, req);
11758
11759 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
11760 bp->link_info.force_link_chng)
11761 bnxt_hwrm_set_link_common(bp, req);
11762
11763 rc = hwrm_req_send(bp, req);
11764 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
11765 /* since changing of pause setting doesn't trigger any link
11766 * change event, the driver needs to update the current pause
11767 * result upon successfully return of the phy_cfg command
11768 */
11769 bp->link_info.pause =
11770 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
11771 bp->link_info.auto_pause_setting = 0;
11772 if (!bp->link_info.force_link_chng)
11773 bnxt_report_link(bp);
11774 }
11775 bp->link_info.force_link_chng = false;
11776 return rc;
11777 }
11778
11779 static void bnxt_hwrm_set_eee(struct bnxt *bp,
11780 struct hwrm_port_phy_cfg_input *req)
11781 {
11782 struct ethtool_keee *eee = &bp->eee;
11783
11784 if (eee->eee_enabled) {
11785 u16 eee_speeds;
11786 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
11787
11788 if (eee->tx_lpi_enabled)
11789 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
11790 else
11791 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
11792
11793 req->flags |= cpu_to_le32(flags);
11794 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
11795 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
11796 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
11797 } else {
11798 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
11799 }
11800 }
11801
11802 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
11803 {
11804 struct hwrm_port_phy_cfg_input *req;
11805 int rc;
11806
11807 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11808 if (rc)
11809 return rc;
11810
11811 if (set_pause)
11812 bnxt_hwrm_set_pause_common(bp, req);
11813
11814 bnxt_hwrm_set_link_common(bp, req);
11815
11816 if (set_eee)
11817 bnxt_hwrm_set_eee(bp, req);
11818 return hwrm_req_send(bp, req);
11819 }
11820
11821 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
11822 {
11823 struct hwrm_port_phy_cfg_input *req;
11824 int rc;
11825
11826 if (!BNXT_SINGLE_PF(bp))
11827 return 0;
11828
11829 if (pci_num_vf(bp->pdev) &&
11830 !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
11831 return 0;
11832
11833 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
11834 if (rc)
11835 return rc;
11836
11837 req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
11838 rc = hwrm_req_send(bp, req);
11839 if (!rc) {
11840 mutex_lock(&bp->link_lock);
11841 /* Device is not obliged link down in certain scenarios, even
11842 * when forced. Setting the state unknown is consistent with
11843 * driver startup and will force link state to be reported
11844 * during subsequent open based on PORT_PHY_QCFG.
11845 */
11846 bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
11847 mutex_unlock(&bp->link_lock);
11848 }
11849 return rc;
11850 }
11851
11852 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11853 {
11854 #ifdef CONFIG_TEE_BNXT_FW
11855 int rc = tee_bnxt_fw_load();
11856
11857 if (rc)
11858 netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11859
11860 return rc;
11861 #else
11862 netdev_err(bp->dev, "OP-TEE not supported\n");
11863 return -ENODEV;
11864 #endif
11865 }
11866
11867 static int bnxt_try_recover_fw(struct bnxt *bp)
11868 {
11869 if (bp->fw_health && bp->fw_health->status_reliable) {
11870 int retry = 0, rc;
11871 u32 sts;
11872
11873 do {
11874 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11875 rc = bnxt_hwrm_poll(bp);
11876 if (!BNXT_FW_IS_BOOTING(sts) &&
11877 !BNXT_FW_IS_RECOVERING(sts))
11878 break;
11879 retry++;
11880 } while (rc == -EBUSY && retry < BNXT_FW_RETRY);
11881
11882 if (!BNXT_FW_IS_HEALTHY(sts)) {
11883 netdev_err(bp->dev,
11884 "Firmware not responding, status: 0x%x\n",
11885 sts);
11886 rc = -ENODEV;
11887 }
11888 if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11889 netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11890 return bnxt_fw_reset_via_optee(bp);
11891 }
11892 return rc;
11893 }
11894
11895 return -ENODEV;
11896 }
11897
11898 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
11899 {
11900 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11901
11902 if (!BNXT_NEW_RM(bp))
11903 return; /* no resource reservations required */
11904
11905 hw_resc->resv_cp_rings = 0;
11906 hw_resc->resv_stat_ctxs = 0;
11907 hw_resc->resv_irqs = 0;
11908 hw_resc->resv_tx_rings = 0;
11909 hw_resc->resv_rx_rings = 0;
11910 hw_resc->resv_hw_ring_grps = 0;
11911 hw_resc->resv_vnics = 0;
11912 hw_resc->resv_rsscos_ctxs = 0;
11913 if (!fw_reset) {
11914 bp->tx_nr_rings = 0;
11915 bp->rx_nr_rings = 0;
11916 }
11917 }
11918
11919 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
11920 {
11921 int rc;
11922
11923 if (!BNXT_NEW_RM(bp))
11924 return 0; /* no resource reservations required */
11925
11926 rc = bnxt_hwrm_func_resc_qcaps(bp, true);
11927 if (rc)
11928 netdev_err(bp->dev, "resc_qcaps failed\n");
11929
11930 bnxt_clear_reservations(bp, fw_reset);
11931
11932 return rc;
11933 }
11934
11935 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
11936 {
11937 struct hwrm_func_drv_if_change_output *resp;
11938 struct hwrm_func_drv_if_change_input *req;
11939 bool fw_reset = !bp->irq_tbl;
11940 bool resc_reinit = false;
11941 int rc, retry = 0;
11942 u32 flags = 0;
11943
11944 if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
11945 return 0;
11946
11947 rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
11948 if (rc)
11949 return rc;
11950
11951 if (up)
11952 req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
11953 resp = hwrm_req_hold(bp, req);
11954
11955 hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
11956 while (retry < BNXT_FW_IF_RETRY) {
11957 rc = hwrm_req_send(bp, req);
11958 if (rc != -EAGAIN)
11959 break;
11960
11961 msleep(50);
11962 retry++;
11963 }
11964
11965 if (rc == -EAGAIN) {
11966 hwrm_req_drop(bp, req);
11967 return rc;
11968 } else if (!rc) {
11969 flags = le32_to_cpu(resp->flags);
11970 } else if (up) {
11971 rc = bnxt_try_recover_fw(bp);
11972 fw_reset = true;
11973 }
11974 hwrm_req_drop(bp, req);
11975 if (rc)
11976 return rc;
11977
11978 if (!up) {
11979 bnxt_inv_fw_health_reg(bp);
11980 return 0;
11981 }
11982
11983 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
11984 resc_reinit = true;
11985 if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
11986 test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
11987 fw_reset = true;
11988 else
11989 bnxt_remap_fw_health_regs(bp);
11990
11991 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
11992 netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
11993 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
11994 return -ENODEV;
11995 }
11996 if (resc_reinit || fw_reset) {
11997 if (fw_reset) {
11998 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
11999 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12000 bnxt_ulp_irq_stop(bp);
12001 bnxt_free_ctx_mem(bp, false);
12002 bnxt_dcb_free(bp);
12003 rc = bnxt_fw_init_one(bp);
12004 if (rc) {
12005 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
12006 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
12007 return rc;
12008 }
12009 bnxt_clear_int_mode(bp);
12010 rc = bnxt_init_int_mode(bp);
12011 if (rc) {
12012 clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
12013 netdev_err(bp->dev, "init int mode failed\n");
12014 return rc;
12015 }
12016 }
12017 rc = bnxt_cancel_reservations(bp, fw_reset);
12018 }
12019 return rc;
12020 }
12021
12022 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
12023 {
12024 struct hwrm_port_led_qcaps_output *resp;
12025 struct hwrm_port_led_qcaps_input *req;
12026 struct bnxt_pf_info *pf = &bp->pf;
12027 int rc;
12028
12029 bp->num_leds = 0;
12030 if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
12031 return 0;
12032
12033 rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
12034 if (rc)
12035 return rc;
12036
12037 req->port_id = cpu_to_le16(pf->port_id);
12038 resp = hwrm_req_hold(bp, req);
12039 rc = hwrm_req_send(bp, req);
12040 if (rc) {
12041 hwrm_req_drop(bp, req);
12042 return rc;
12043 }
12044 if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
12045 int i;
12046
12047 bp->num_leds = resp->num_leds;
12048 memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
12049 bp->num_leds);
12050 for (i = 0; i < bp->num_leds; i++) {
12051 struct bnxt_led_info *led = &bp->leds[i];
12052 __le16 caps = led->led_state_caps;
12053
12054 if (!led->led_group_id ||
12055 !BNXT_LED_ALT_BLINK_CAP(caps)) {
12056 bp->num_leds = 0;
12057 break;
12058 }
12059 }
12060 }
12061 hwrm_req_drop(bp, req);
12062 return 0;
12063 }
12064
12065 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
12066 {
12067 struct hwrm_wol_filter_alloc_output *resp;
12068 struct hwrm_wol_filter_alloc_input *req;
12069 int rc;
12070
12071 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
12072 if (rc)
12073 return rc;
12074
12075 req->port_id = cpu_to_le16(bp->pf.port_id);
12076 req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
12077 req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
12078 memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
12079
12080 resp = hwrm_req_hold(bp, req);
12081 rc = hwrm_req_send(bp, req);
12082 if (!rc)
12083 bp->wol_filter_id = resp->wol_filter_id;
12084 hwrm_req_drop(bp, req);
12085 return rc;
12086 }
12087
12088 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
12089 {
12090 struct hwrm_wol_filter_free_input *req;
12091 int rc;
12092
12093 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
12094 if (rc)
12095 return rc;
12096
12097 req->port_id = cpu_to_le16(bp->pf.port_id);
12098 req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
12099 req->wol_filter_id = bp->wol_filter_id;
12100
12101 return hwrm_req_send(bp, req);
12102 }
12103
12104 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
12105 {
12106 struct hwrm_wol_filter_qcfg_output *resp;
12107 struct hwrm_wol_filter_qcfg_input *req;
12108 u16 next_handle = 0;
12109 int rc;
12110
12111 rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
12112 if (rc)
12113 return rc;
12114
12115 req->port_id = cpu_to_le16(bp->pf.port_id);
12116 req->handle = cpu_to_le16(handle);
12117 resp = hwrm_req_hold(bp, req);
12118 rc = hwrm_req_send(bp, req);
12119 if (!rc) {
12120 next_handle = le16_to_cpu(resp->next_handle);
12121 if (next_handle != 0) {
12122 if (resp->wol_type ==
12123 WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
12124 bp->wol = 1;
12125 bp->wol_filter_id = resp->wol_filter_id;
12126 }
12127 }
12128 }
12129 hwrm_req_drop(bp, req);
12130 return next_handle;
12131 }
12132
12133 static void bnxt_get_wol_settings(struct bnxt *bp)
12134 {
12135 u16 handle = 0;
12136
12137 bp->wol = 0;
12138 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
12139 return;
12140
12141 do {
12142 handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
12143 } while (handle && handle != 0xffff);
12144 }
12145
12146 static bool bnxt_eee_config_ok(struct bnxt *bp)
12147 {
12148 struct ethtool_keee *eee = &bp->eee;
12149 struct bnxt_link_info *link_info = &bp->link_info;
12150
12151 if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
12152 return true;
12153
12154 if (eee->eee_enabled) {
12155 __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising);
12156 __ETHTOOL_DECLARE_LINK_MODE_MASK(tmp);
12157
12158 _bnxt_fw_to_linkmode(advertising, link_info->advertising);
12159
12160 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
12161 eee->eee_enabled = 0;
12162 return false;
12163 }
12164 if (linkmode_andnot(tmp, eee->advertised, advertising)) {
12165 linkmode_and(eee->advertised, advertising,
12166 eee->supported);
12167 return false;
12168 }
12169 }
12170 return true;
12171 }
12172
12173 static int bnxt_update_phy_setting(struct bnxt *bp)
12174 {
12175 int rc;
12176 bool update_link = false;
12177 bool update_pause = false;
12178 bool update_eee = false;
12179 struct bnxt_link_info *link_info = &bp->link_info;
12180
12181 rc = bnxt_update_link(bp, true);
12182 if (rc) {
12183 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
12184 rc);
12185 return rc;
12186 }
12187 if (!BNXT_SINGLE_PF(bp))
12188 return 0;
12189
12190 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
12191 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
12192 link_info->req_flow_ctrl)
12193 update_pause = true;
12194 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
12195 link_info->force_pause_setting != link_info->req_flow_ctrl)
12196 update_pause = true;
12197 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
12198 if (BNXT_AUTO_MODE(link_info->auto_mode))
12199 update_link = true;
12200 if (bnxt_force_speed_updated(link_info))
12201 update_link = true;
12202 if (link_info->req_duplex != link_info->duplex_setting)
12203 update_link = true;
12204 } else {
12205 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
12206 update_link = true;
12207 if (bnxt_auto_speed_updated(link_info))
12208 update_link = true;
12209 }
12210
12211 /* The last close may have shutdown the link, so need to call
12212 * PHY_CFG to bring it back up.
12213 */
12214 if (!BNXT_LINK_IS_UP(bp))
12215 update_link = true;
12216
12217 if (!bnxt_eee_config_ok(bp))
12218 update_eee = true;
12219
12220 if (update_link)
12221 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
12222 else if (update_pause)
12223 rc = bnxt_hwrm_set_pause(bp);
12224 if (rc) {
12225 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
12226 rc);
12227 return rc;
12228 }
12229
12230 return rc;
12231 }
12232
12233 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
12234
12235 static int bnxt_reinit_after_abort(struct bnxt *bp)
12236 {
12237 int rc;
12238
12239 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12240 return -EBUSY;
12241
12242 if (bp->dev->reg_state == NETREG_UNREGISTERED)
12243 return -ENODEV;
12244
12245 rc = bnxt_fw_init_one(bp);
12246 if (!rc) {
12247 bnxt_clear_int_mode(bp);
12248 rc = bnxt_init_int_mode(bp);
12249 if (!rc) {
12250 clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
12251 set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
12252 }
12253 }
12254 return rc;
12255 }
12256
12257 static void bnxt_cfg_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr)
12258 {
12259 struct bnxt_ntuple_filter *ntp_fltr;
12260 struct bnxt_l2_filter *l2_fltr;
12261
12262 if (list_empty(&fltr->list))
12263 return;
12264
12265 if (fltr->type == BNXT_FLTR_TYPE_NTUPLE) {
12266 ntp_fltr = container_of(fltr, struct bnxt_ntuple_filter, base);
12267 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0];
12268 atomic_inc(&l2_fltr->refcnt);
12269 ntp_fltr->l2_fltr = l2_fltr;
12270 if (bnxt_hwrm_cfa_ntuple_filter_alloc(bp, ntp_fltr)) {
12271 bnxt_del_ntp_filter(bp, ntp_fltr);
12272 netdev_err(bp->dev, "restoring previously configured ntuple filter id %d failed\n",
12273 fltr->sw_id);
12274 }
12275 } else if (fltr->type == BNXT_FLTR_TYPE_L2) {
12276 l2_fltr = container_of(fltr, struct bnxt_l2_filter, base);
12277 if (bnxt_hwrm_l2_filter_alloc(bp, l2_fltr)) {
12278 bnxt_del_l2_filter(bp, l2_fltr);
12279 netdev_err(bp->dev, "restoring previously configured l2 filter id %d failed\n",
12280 fltr->sw_id);
12281 }
12282 }
12283 }
12284
12285 static void bnxt_cfg_usr_fltrs(struct bnxt *bp)
12286 {
12287 struct bnxt_filter_base *usr_fltr, *tmp;
12288
12289 list_for_each_entry_safe(usr_fltr, tmp, &bp->usr_fltr_list, list)
12290 bnxt_cfg_one_usr_fltr(bp, usr_fltr);
12291 }
12292
12293 static int bnxt_set_xps_mapping(struct bnxt *bp)
12294 {
12295 int numa_node = dev_to_node(&bp->pdev->dev);
12296 unsigned int q_idx, map_idx, cpu, i;
12297 const struct cpumask *cpu_mask_ptr;
12298 int nr_cpus = num_online_cpus();
12299 cpumask_t *q_map;
12300 int rc = 0;
12301
12302 q_map = kcalloc(bp->tx_nr_rings_per_tc, sizeof(*q_map), GFP_KERNEL);
12303 if (!q_map)
12304 return -ENOMEM;
12305
12306 /* Create CPU mask for all TX queues across MQPRIO traffic classes.
12307 * Each TC has the same number of TX queues. The nth TX queue for each
12308 * TC will have the same CPU mask.
12309 */
12310 for (i = 0; i < nr_cpus; i++) {
12311 map_idx = i % bp->tx_nr_rings_per_tc;
12312 cpu = cpumask_local_spread(i, numa_node);
12313 cpu_mask_ptr = get_cpu_mask(cpu);
12314 cpumask_or(&q_map[map_idx], &q_map[map_idx], cpu_mask_ptr);
12315 }
12316
12317 /* Register CPU mask for each TX queue except the ones marked for XDP */
12318 for (q_idx = 0; q_idx < bp->dev->real_num_tx_queues; q_idx++) {
12319 map_idx = q_idx % bp->tx_nr_rings_per_tc;
12320 rc = netif_set_xps_queue(bp->dev, &q_map[map_idx], q_idx);
12321 if (rc) {
12322 netdev_warn(bp->dev, "Error setting XPS for q:%d\n",
12323 q_idx);
12324 break;
12325 }
12326 }
12327
12328 kfree(q_map);
12329
12330 return rc;
12331 }
12332
12333 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12334 {
12335 int rc = 0;
12336
12337 netif_carrier_off(bp->dev);
12338 if (irq_re_init) {
12339 /* Reserve rings now if none were reserved at driver probe. */
12340 rc = bnxt_init_dflt_ring_mode(bp);
12341 if (rc) {
12342 netdev_err(bp->dev, "Failed to reserve default rings at open\n");
12343 return rc;
12344 }
12345 }
12346 rc = bnxt_reserve_rings(bp, irq_re_init);
12347 if (rc)
12348 return rc;
12349
12350 rc = bnxt_alloc_mem(bp, irq_re_init);
12351 if (rc) {
12352 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
12353 goto open_err_free_mem;
12354 }
12355
12356 if (irq_re_init) {
12357 bnxt_init_napi(bp);
12358 rc = bnxt_request_irq(bp);
12359 if (rc) {
12360 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
12361 goto open_err_irq;
12362 }
12363 }
12364
12365 rc = bnxt_init_nic(bp, irq_re_init);
12366 if (rc) {
12367 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
12368 goto open_err_irq;
12369 }
12370
12371 bnxt_enable_napi(bp);
12372 bnxt_debug_dev_init(bp);
12373
12374 if (link_re_init) {
12375 mutex_lock(&bp->link_lock);
12376 rc = bnxt_update_phy_setting(bp);
12377 mutex_unlock(&bp->link_lock);
12378 if (rc) {
12379 netdev_warn(bp->dev, "failed to update phy settings\n");
12380 if (BNXT_SINGLE_PF(bp)) {
12381 bp->link_info.phy_retry = true;
12382 bp->link_info.phy_retry_expires =
12383 jiffies + 5 * HZ;
12384 }
12385 }
12386 }
12387
12388 if (irq_re_init) {
12389 udp_tunnel_nic_reset_ntf(bp->dev);
12390 rc = bnxt_set_xps_mapping(bp);
12391 if (rc)
12392 netdev_warn(bp->dev, "failed to set xps mapping\n");
12393 }
12394
12395 if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
12396 if (!static_key_enabled(&bnxt_xdp_locking_key))
12397 static_branch_enable(&bnxt_xdp_locking_key);
12398 } else if (static_key_enabled(&bnxt_xdp_locking_key)) {
12399 static_branch_disable(&bnxt_xdp_locking_key);
12400 }
12401 set_bit(BNXT_STATE_OPEN, &bp->state);
12402 bnxt_enable_int(bp);
12403 /* Enable TX queues */
12404 bnxt_tx_enable(bp);
12405 mod_timer(&bp->timer, jiffies + bp->current_interval);
12406 /* Poll link status and check for SFP+ module status */
12407 mutex_lock(&bp->link_lock);
12408 bnxt_get_port_module_status(bp);
12409 mutex_unlock(&bp->link_lock);
12410
12411 /* VF-reps may need to be re-opened after the PF is re-opened */
12412 if (BNXT_PF(bp))
12413 bnxt_vf_reps_open(bp);
12414 if (bp->ptp_cfg && !(bp->fw_cap & BNXT_FW_CAP_TX_TS_CMP))
12415 WRITE_ONCE(bp->ptp_cfg->tx_avail, BNXT_MAX_TX_TS);
12416 bnxt_ptp_init_rtc(bp, true);
12417 bnxt_ptp_cfg_tstamp_filters(bp);
12418 if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
12419 bnxt_hwrm_realloc_rss_ctx_vnic(bp);
12420 bnxt_cfg_usr_fltrs(bp);
12421 return 0;
12422
12423 open_err_irq:
12424 bnxt_del_napi(bp);
12425
12426 open_err_free_mem:
12427 bnxt_free_skbs(bp);
12428 bnxt_free_irq(bp);
12429 bnxt_free_mem(bp, true);
12430 return rc;
12431 }
12432
12433 /* rtnl_lock held */
12434 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12435 {
12436 int rc = 0;
12437
12438 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
12439 rc = -EIO;
12440 if (!rc)
12441 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
12442 if (rc) {
12443 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
12444 dev_close(bp->dev);
12445 }
12446 return rc;
12447 }
12448
12449 /* rtnl_lock held, open the NIC half way by allocating all resources, but
12450 * NAPI, IRQ, and TX are not enabled. This is mainly used for offline
12451 * self tests.
12452 */
12453 int bnxt_half_open_nic(struct bnxt *bp)
12454 {
12455 int rc = 0;
12456
12457 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12458 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
12459 rc = -ENODEV;
12460 goto half_open_err;
12461 }
12462
12463 rc = bnxt_alloc_mem(bp, true);
12464 if (rc) {
12465 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
12466 goto half_open_err;
12467 }
12468 bnxt_init_napi(bp);
12469 set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12470 rc = bnxt_init_nic(bp, true);
12471 if (rc) {
12472 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12473 bnxt_del_napi(bp);
12474 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
12475 goto half_open_err;
12476 }
12477 return 0;
12478
12479 half_open_err:
12480 bnxt_free_skbs(bp);
12481 bnxt_free_mem(bp, true);
12482 dev_close(bp->dev);
12483 return rc;
12484 }
12485
12486 /* rtnl_lock held, this call can only be made after a previous successful
12487 * call to bnxt_half_open_nic().
12488 */
12489 void bnxt_half_close_nic(struct bnxt *bp)
12490 {
12491 bnxt_hwrm_resource_free(bp, false, true);
12492 bnxt_del_napi(bp);
12493 bnxt_free_skbs(bp);
12494 bnxt_free_mem(bp, true);
12495 clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
12496 }
12497
12498 void bnxt_reenable_sriov(struct bnxt *bp)
12499 {
12500 if (BNXT_PF(bp)) {
12501 struct bnxt_pf_info *pf = &bp->pf;
12502 int n = pf->active_vfs;
12503
12504 if (n)
12505 bnxt_cfg_hw_sriov(bp, &n, true);
12506 }
12507 }
12508
12509 static int bnxt_open(struct net_device *dev)
12510 {
12511 struct bnxt *bp = netdev_priv(dev);
12512 int rc;
12513
12514 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12515 rc = bnxt_reinit_after_abort(bp);
12516 if (rc) {
12517 if (rc == -EBUSY)
12518 netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
12519 else
12520 netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
12521 return -ENODEV;
12522 }
12523 }
12524
12525 rc = bnxt_hwrm_if_change(bp, true);
12526 if (rc)
12527 return rc;
12528
12529 rc = __bnxt_open_nic(bp, true, true);
12530 if (rc) {
12531 bnxt_hwrm_if_change(bp, false);
12532 } else {
12533 if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
12534 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
12535 bnxt_queue_sp_work(bp,
12536 BNXT_RESTART_ULP_SP_EVENT);
12537 }
12538 }
12539
12540 return rc;
12541 }
12542
12543 static bool bnxt_drv_busy(struct bnxt *bp)
12544 {
12545 return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
12546 test_bit(BNXT_STATE_READ_STATS, &bp->state));
12547 }
12548
12549 static void bnxt_get_ring_stats(struct bnxt *bp,
12550 struct rtnl_link_stats64 *stats);
12551
12552 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
12553 bool link_re_init)
12554 {
12555 /* Close the VF-reps before closing PF */
12556 if (BNXT_PF(bp))
12557 bnxt_vf_reps_close(bp);
12558
12559 /* Change device state to avoid TX queue wake up's */
12560 bnxt_tx_disable(bp);
12561
12562 clear_bit(BNXT_STATE_OPEN, &bp->state);
12563 smp_mb__after_atomic();
12564 while (bnxt_drv_busy(bp))
12565 msleep(20);
12566
12567 if (BNXT_SUPPORTS_MULTI_RSS_CTX(bp))
12568 bnxt_clear_rss_ctxs(bp);
12569 /* Flush rings and disable interrupts */
12570 bnxt_shutdown_nic(bp, irq_re_init);
12571
12572 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
12573
12574 bnxt_debug_dev_exit(bp);
12575 bnxt_disable_napi(bp);
12576 del_timer_sync(&bp->timer);
12577 bnxt_free_skbs(bp);
12578
12579 /* Save ring stats before shutdown */
12580 if (bp->bnapi && irq_re_init) {
12581 bnxt_get_ring_stats(bp, &bp->net_stats_prev);
12582 bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
12583 }
12584 if (irq_re_init) {
12585 bnxt_free_irq(bp);
12586 bnxt_del_napi(bp);
12587 }
12588 bnxt_free_mem(bp, irq_re_init);
12589 }
12590
12591 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
12592 {
12593 if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12594 /* If we get here, it means firmware reset is in progress
12595 * while we are trying to close. We can safely proceed with
12596 * the close because we are holding rtnl_lock(). Some firmware
12597 * messages may fail as we proceed to close. We set the
12598 * ABORT_ERR flag here so that the FW reset thread will later
12599 * abort when it gets the rtnl_lock() and sees the flag.
12600 */
12601 netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
12602 set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
12603 }
12604
12605 #ifdef CONFIG_BNXT_SRIOV
12606 if (bp->sriov_cfg) {
12607 int rc;
12608
12609 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
12610 !bp->sriov_cfg,
12611 BNXT_SRIOV_CFG_WAIT_TMO);
12612 if (!rc)
12613 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
12614 else if (rc < 0)
12615 netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
12616 }
12617 #endif
12618 __bnxt_close_nic(bp, irq_re_init, link_re_init);
12619 }
12620
12621 static int bnxt_close(struct net_device *dev)
12622 {
12623 struct bnxt *bp = netdev_priv(dev);
12624
12625 bnxt_close_nic(bp, true, true);
12626 bnxt_hwrm_shutdown_link(bp);
12627 bnxt_hwrm_if_change(bp, false);
12628 return 0;
12629 }
12630
12631 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
12632 u16 *val)
12633 {
12634 struct hwrm_port_phy_mdio_read_output *resp;
12635 struct hwrm_port_phy_mdio_read_input *req;
12636 int rc;
12637
12638 if (bp->hwrm_spec_code < 0x10a00)
12639 return -EOPNOTSUPP;
12640
12641 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
12642 if (rc)
12643 return rc;
12644
12645 req->port_id = cpu_to_le16(bp->pf.port_id);
12646 req->phy_addr = phy_addr;
12647 req->reg_addr = cpu_to_le16(reg & 0x1f);
12648 if (mdio_phy_id_is_c45(phy_addr)) {
12649 req->cl45_mdio = 1;
12650 req->phy_addr = mdio_phy_id_prtad(phy_addr);
12651 req->dev_addr = mdio_phy_id_devad(phy_addr);
12652 req->reg_addr = cpu_to_le16(reg);
12653 }
12654
12655 resp = hwrm_req_hold(bp, req);
12656 rc = hwrm_req_send(bp, req);
12657 if (!rc)
12658 *val = le16_to_cpu(resp->reg_data);
12659 hwrm_req_drop(bp, req);
12660 return rc;
12661 }
12662
12663 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
12664 u16 val)
12665 {
12666 struct hwrm_port_phy_mdio_write_input *req;
12667 int rc;
12668
12669 if (bp->hwrm_spec_code < 0x10a00)
12670 return -EOPNOTSUPP;
12671
12672 rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
12673 if (rc)
12674 return rc;
12675
12676 req->port_id = cpu_to_le16(bp->pf.port_id);
12677 req->phy_addr = phy_addr;
12678 req->reg_addr = cpu_to_le16(reg & 0x1f);
12679 if (mdio_phy_id_is_c45(phy_addr)) {
12680 req->cl45_mdio = 1;
12681 req->phy_addr = mdio_phy_id_prtad(phy_addr);
12682 req->dev_addr = mdio_phy_id_devad(phy_addr);
12683 req->reg_addr = cpu_to_le16(reg);
12684 }
12685 req->reg_data = cpu_to_le16(val);
12686
12687 return hwrm_req_send(bp, req);
12688 }
12689
12690 /* rtnl_lock held */
12691 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
12692 {
12693 struct mii_ioctl_data *mdio = if_mii(ifr);
12694 struct bnxt *bp = netdev_priv(dev);
12695 int rc;
12696
12697 switch (cmd) {
12698 case SIOCGMIIPHY:
12699 mdio->phy_id = bp->link_info.phy_addr;
12700
12701 fallthrough;
12702 case SIOCGMIIREG: {
12703 u16 mii_regval = 0;
12704
12705 if (!netif_running(dev))
12706 return -EAGAIN;
12707
12708 rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
12709 &mii_regval);
12710 mdio->val_out = mii_regval;
12711 return rc;
12712 }
12713
12714 case SIOCSMIIREG:
12715 if (!netif_running(dev))
12716 return -EAGAIN;
12717
12718 return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
12719 mdio->val_in);
12720
12721 case SIOCSHWTSTAMP:
12722 return bnxt_hwtstamp_set(dev, ifr);
12723
12724 case SIOCGHWTSTAMP:
12725 return bnxt_hwtstamp_get(dev, ifr);
12726
12727 default:
12728 /* do nothing */
12729 break;
12730 }
12731 return -EOPNOTSUPP;
12732 }
12733
12734 static void bnxt_get_ring_stats(struct bnxt *bp,
12735 struct rtnl_link_stats64 *stats)
12736 {
12737 int i;
12738
12739 for (i = 0; i < bp->cp_nr_rings; i++) {
12740 struct bnxt_napi *bnapi = bp->bnapi[i];
12741 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
12742 u64 *sw = cpr->stats.sw_stats;
12743
12744 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
12745 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
12746 stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
12747
12748 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
12749 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
12750 stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
12751
12752 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
12753 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
12754 stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
12755
12756 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
12757 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
12758 stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
12759
12760 stats->rx_missed_errors +=
12761 BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
12762
12763 stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
12764
12765 stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
12766
12767 stats->rx_dropped +=
12768 cpr->sw_stats->rx.rx_netpoll_discards +
12769 cpr->sw_stats->rx.rx_oom_discards;
12770 }
12771 }
12772
12773 static void bnxt_add_prev_stats(struct bnxt *bp,
12774 struct rtnl_link_stats64 *stats)
12775 {
12776 struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
12777
12778 stats->rx_packets += prev_stats->rx_packets;
12779 stats->tx_packets += prev_stats->tx_packets;
12780 stats->rx_bytes += prev_stats->rx_bytes;
12781 stats->tx_bytes += prev_stats->tx_bytes;
12782 stats->rx_missed_errors += prev_stats->rx_missed_errors;
12783 stats->multicast += prev_stats->multicast;
12784 stats->rx_dropped += prev_stats->rx_dropped;
12785 stats->tx_dropped += prev_stats->tx_dropped;
12786 }
12787
12788 static void
12789 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
12790 {
12791 struct bnxt *bp = netdev_priv(dev);
12792
12793 set_bit(BNXT_STATE_READ_STATS, &bp->state);
12794 /* Make sure bnxt_close_nic() sees that we are reading stats before
12795 * we check the BNXT_STATE_OPEN flag.
12796 */
12797 smp_mb__after_atomic();
12798 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12799 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
12800 *stats = bp->net_stats_prev;
12801 return;
12802 }
12803
12804 bnxt_get_ring_stats(bp, stats);
12805 bnxt_add_prev_stats(bp, stats);
12806
12807 if (bp->flags & BNXT_FLAG_PORT_STATS) {
12808 u64 *rx = bp->port_stats.sw_stats;
12809 u64 *tx = bp->port_stats.sw_stats +
12810 BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
12811
12812 stats->rx_crc_errors =
12813 BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
12814 stats->rx_frame_errors =
12815 BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
12816 stats->rx_length_errors =
12817 BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
12818 BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
12819 BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
12820 stats->rx_errors =
12821 BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
12822 BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
12823 stats->collisions =
12824 BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
12825 stats->tx_fifo_errors =
12826 BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
12827 stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
12828 }
12829 clear_bit(BNXT_STATE_READ_STATS, &bp->state);
12830 }
12831
12832 static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
12833 struct bnxt_total_ring_err_stats *stats,
12834 struct bnxt_cp_ring_info *cpr)
12835 {
12836 struct bnxt_sw_stats *sw_stats = cpr->sw_stats;
12837 u64 *hw_stats = cpr->stats.sw_stats;
12838
12839 stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
12840 stats->rx_total_resets += sw_stats->rx.rx_resets;
12841 stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
12842 stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
12843 stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
12844 stats->rx_total_ring_discards +=
12845 BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
12846 stats->tx_total_resets += sw_stats->tx.tx_resets;
12847 stats->tx_total_ring_discards +=
12848 BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
12849 stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
12850 }
12851
12852 void bnxt_get_ring_err_stats(struct bnxt *bp,
12853 struct bnxt_total_ring_err_stats *stats)
12854 {
12855 int i;
12856
12857 for (i = 0; i < bp->cp_nr_rings; i++)
12858 bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
12859 }
12860
12861 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
12862 {
12863 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12864 struct net_device *dev = bp->dev;
12865 struct netdev_hw_addr *ha;
12866 u8 *haddr;
12867 int mc_count = 0;
12868 bool update = false;
12869 int off = 0;
12870
12871 netdev_for_each_mc_addr(ha, dev) {
12872 if (mc_count >= BNXT_MAX_MC_ADDRS) {
12873 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12874 vnic->mc_list_count = 0;
12875 return false;
12876 }
12877 haddr = ha->addr;
12878 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
12879 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
12880 update = true;
12881 }
12882 off += ETH_ALEN;
12883 mc_count++;
12884 }
12885 if (mc_count)
12886 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
12887
12888 if (mc_count != vnic->mc_list_count) {
12889 vnic->mc_list_count = mc_count;
12890 update = true;
12891 }
12892 return update;
12893 }
12894
12895 static bool bnxt_uc_list_updated(struct bnxt *bp)
12896 {
12897 struct net_device *dev = bp->dev;
12898 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12899 struct netdev_hw_addr *ha;
12900 int off = 0;
12901
12902 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
12903 return true;
12904
12905 netdev_for_each_uc_addr(ha, dev) {
12906 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
12907 return true;
12908
12909 off += ETH_ALEN;
12910 }
12911 return false;
12912 }
12913
12914 static void bnxt_set_rx_mode(struct net_device *dev)
12915 {
12916 struct bnxt *bp = netdev_priv(dev);
12917 struct bnxt_vnic_info *vnic;
12918 bool mc_update = false;
12919 bool uc_update;
12920 u32 mask;
12921
12922 if (!test_bit(BNXT_STATE_OPEN, &bp->state))
12923 return;
12924
12925 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12926 mask = vnic->rx_mask;
12927 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
12928 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
12929 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
12930 CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
12931
12932 if (dev->flags & IFF_PROMISC)
12933 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12934
12935 uc_update = bnxt_uc_list_updated(bp);
12936
12937 if (dev->flags & IFF_BROADCAST)
12938 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
12939 if (dev->flags & IFF_ALLMULTI) {
12940 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
12941 vnic->mc_list_count = 0;
12942 } else if (dev->flags & IFF_MULTICAST) {
12943 mc_update = bnxt_mc_list_updated(bp, &mask);
12944 }
12945
12946 if (mask != vnic->rx_mask || uc_update || mc_update) {
12947 vnic->rx_mask = mask;
12948
12949 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
12950 }
12951 }
12952
12953 static int bnxt_cfg_rx_mode(struct bnxt *bp)
12954 {
12955 struct net_device *dev = bp->dev;
12956 struct bnxt_vnic_info *vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
12957 struct netdev_hw_addr *ha;
12958 int i, off = 0, rc;
12959 bool uc_update;
12960
12961 netif_addr_lock_bh(dev);
12962 uc_update = bnxt_uc_list_updated(bp);
12963 netif_addr_unlock_bh(dev);
12964
12965 if (!uc_update)
12966 goto skip_uc;
12967
12968 for (i = 1; i < vnic->uc_filter_count; i++) {
12969 struct bnxt_l2_filter *fltr = vnic->l2_filters[i];
12970
12971 bnxt_hwrm_l2_filter_free(bp, fltr);
12972 bnxt_del_l2_filter(bp, fltr);
12973 }
12974
12975 vnic->uc_filter_count = 1;
12976
12977 netif_addr_lock_bh(dev);
12978 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
12979 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
12980 } else {
12981 netdev_for_each_uc_addr(ha, dev) {
12982 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
12983 off += ETH_ALEN;
12984 vnic->uc_filter_count++;
12985 }
12986 }
12987 netif_addr_unlock_bh(dev);
12988
12989 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
12990 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
12991 if (rc) {
12992 if (BNXT_VF(bp) && rc == -ENODEV) {
12993 if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
12994 netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
12995 else
12996 netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
12997 rc = 0;
12998 } else {
12999 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
13000 }
13001 vnic->uc_filter_count = i;
13002 return rc;
13003 }
13004 }
13005 if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
13006 netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
13007
13008 skip_uc:
13009 if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
13010 !bnxt_promisc_ok(bp))
13011 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
13012 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
13013 if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
13014 netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
13015 rc);
13016 vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
13017 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
13018 vnic->mc_list_count = 0;
13019 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
13020 }
13021 if (rc)
13022 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
13023 rc);
13024
13025 return rc;
13026 }
13027
13028 static bool bnxt_can_reserve_rings(struct bnxt *bp)
13029 {
13030 #ifdef CONFIG_BNXT_SRIOV
13031 if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
13032 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
13033
13034 /* No minimum rings were provisioned by the PF. Don't
13035 * reserve rings by default when device is down.
13036 */
13037 if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
13038 return true;
13039
13040 if (!netif_running(bp->dev))
13041 return false;
13042 }
13043 #endif
13044 return true;
13045 }
13046
13047 /* If the chip and firmware supports RFS */
13048 static bool bnxt_rfs_supported(struct bnxt *bp)
13049 {
13050 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
13051 if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
13052 return true;
13053 return false;
13054 }
13055 /* 212 firmware is broken for aRFS */
13056 if (BNXT_FW_MAJ(bp) == 212)
13057 return false;
13058 if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
13059 return true;
13060 if (bp->rss_cap & BNXT_RSS_CAP_NEW_RSS_CAP)
13061 return true;
13062 return false;
13063 }
13064
13065 /* If runtime conditions support RFS */
13066 bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx)
13067 {
13068 struct bnxt_hw_rings hwr = {0};
13069 int max_vnics, max_rss_ctxs;
13070
13071 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
13072 !BNXT_SUPPORTS_NTUPLE_VNIC(bp))
13073 return bnxt_rfs_supported(bp);
13074
13075 if (!bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
13076 return false;
13077
13078 hwr.grp = bp->rx_nr_rings;
13079 hwr.vnic = bnxt_get_total_vnics(bp, bp->rx_nr_rings);
13080 if (new_rss_ctx)
13081 hwr.vnic++;
13082 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
13083 max_vnics = bnxt_get_max_func_vnics(bp);
13084 max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
13085
13086 if (hwr.vnic > max_vnics || hwr.rss_ctx > max_rss_ctxs) {
13087 if (bp->rx_nr_rings > 1)
13088 netdev_warn(bp->dev,
13089 "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
13090 min(max_rss_ctxs - 1, max_vnics - 1));
13091 return false;
13092 }
13093
13094 if (!BNXT_NEW_RM(bp))
13095 return true;
13096
13097 /* Do not reduce VNIC and RSS ctx reservations. There is a FW
13098 * issue that will mess up the default VNIC if we reduce the
13099 * reservations.
13100 */
13101 if (hwr.vnic <= bp->hw_resc.resv_vnics &&
13102 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs)
13103 return true;
13104
13105 bnxt_hwrm_reserve_rings(bp, &hwr);
13106 if (hwr.vnic <= bp->hw_resc.resv_vnics &&
13107 hwr.rss_ctx <= bp->hw_resc.resv_rsscos_ctxs)
13108 return true;
13109
13110 netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
13111 hwr.vnic = 1;
13112 hwr.rss_ctx = 0;
13113 bnxt_hwrm_reserve_rings(bp, &hwr);
13114 return false;
13115 }
13116
13117 static netdev_features_t bnxt_fix_features(struct net_device *dev,
13118 netdev_features_t features)
13119 {
13120 struct bnxt *bp = netdev_priv(dev);
13121 netdev_features_t vlan_features;
13122
13123 if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp, false))
13124 features &= ~NETIF_F_NTUPLE;
13125
13126 if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
13127 features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13128
13129 if (!(features & NETIF_F_GRO))
13130 features &= ~NETIF_F_GRO_HW;
13131
13132 if (features & NETIF_F_GRO_HW)
13133 features &= ~NETIF_F_LRO;
13134
13135 /* Both CTAG and STAG VLAN acceleration on the RX side have to be
13136 * turned on or off together.
13137 */
13138 vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
13139 if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
13140 if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13141 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
13142 else if (vlan_features)
13143 features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13144 }
13145 #ifdef CONFIG_BNXT_SRIOV
13146 if (BNXT_VF(bp) && bp->vf.vlan)
13147 features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
13148 #endif
13149 return features;
13150 }
13151
13152 static int bnxt_reinit_features(struct bnxt *bp, bool irq_re_init,
13153 bool link_re_init, u32 flags, bool update_tpa)
13154 {
13155 bnxt_close_nic(bp, irq_re_init, link_re_init);
13156 bp->flags = flags;
13157 if (update_tpa)
13158 bnxt_set_ring_params(bp);
13159 return bnxt_open_nic(bp, irq_re_init, link_re_init);
13160 }
13161
13162 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
13163 {
13164 bool update_tpa = false, update_ntuple = false;
13165 struct bnxt *bp = netdev_priv(dev);
13166 u32 flags = bp->flags;
13167 u32 changes;
13168 int rc = 0;
13169 bool re_init = false;
13170
13171 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
13172 if (features & NETIF_F_GRO_HW)
13173 flags |= BNXT_FLAG_GRO;
13174 else if (features & NETIF_F_LRO)
13175 flags |= BNXT_FLAG_LRO;
13176
13177 if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
13178 flags &= ~BNXT_FLAG_TPA;
13179
13180 if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13181 flags |= BNXT_FLAG_STRIP_VLAN;
13182
13183 if (features & NETIF_F_NTUPLE)
13184 flags |= BNXT_FLAG_RFS;
13185 else
13186 bnxt_clear_usr_fltrs(bp, true);
13187
13188 changes = flags ^ bp->flags;
13189 if (changes & BNXT_FLAG_TPA) {
13190 update_tpa = true;
13191 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
13192 (flags & BNXT_FLAG_TPA) == 0 ||
13193 (bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
13194 re_init = true;
13195 }
13196
13197 if (changes & ~BNXT_FLAG_TPA)
13198 re_init = true;
13199
13200 if (changes & BNXT_FLAG_RFS)
13201 update_ntuple = true;
13202
13203 if (flags != bp->flags) {
13204 u32 old_flags = bp->flags;
13205
13206 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13207 bp->flags = flags;
13208 if (update_tpa)
13209 bnxt_set_ring_params(bp);
13210 return rc;
13211 }
13212
13213 if (update_ntuple)
13214 return bnxt_reinit_features(bp, true, false, flags, update_tpa);
13215
13216 if (re_init)
13217 return bnxt_reinit_features(bp, false, false, flags, update_tpa);
13218
13219 if (update_tpa) {
13220 bp->flags = flags;
13221 rc = bnxt_set_tpa(bp,
13222 (flags & BNXT_FLAG_TPA) ?
13223 true : false);
13224 if (rc)
13225 bp->flags = old_flags;
13226 }
13227 }
13228 return rc;
13229 }
13230
13231 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
13232 u8 **nextp)
13233 {
13234 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
13235 struct hop_jumbo_hdr *jhdr;
13236 int hdr_count = 0;
13237 u8 *nexthdr;
13238 int start;
13239
13240 /* Check that there are at most 2 IPv6 extension headers, no
13241 * fragment header, and each is <= 64 bytes.
13242 */
13243 start = nw_off + sizeof(*ip6h);
13244 nexthdr = &ip6h->nexthdr;
13245 while (ipv6_ext_hdr(*nexthdr)) {
13246 struct ipv6_opt_hdr *hp;
13247 int hdrlen;
13248
13249 if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
13250 *nexthdr == NEXTHDR_FRAGMENT)
13251 return false;
13252 hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
13253 skb_headlen(skb), NULL);
13254 if (!hp)
13255 return false;
13256 if (*nexthdr == NEXTHDR_AUTH)
13257 hdrlen = ipv6_authlen(hp);
13258 else
13259 hdrlen = ipv6_optlen(hp);
13260
13261 if (hdrlen > 64)
13262 return false;
13263
13264 /* The ext header may be a hop-by-hop header inserted for
13265 * big TCP purposes. This will be removed before sending
13266 * from NIC, so do not count it.
13267 */
13268 if (*nexthdr == NEXTHDR_HOP) {
13269 if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
13270 goto increment_hdr;
13271
13272 jhdr = (struct hop_jumbo_hdr *)hp;
13273 if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
13274 jhdr->nexthdr != IPPROTO_TCP)
13275 goto increment_hdr;
13276
13277 goto next_hdr;
13278 }
13279 increment_hdr:
13280 hdr_count++;
13281 next_hdr:
13282 nexthdr = &hp->nexthdr;
13283 start += hdrlen;
13284 }
13285 if (nextp) {
13286 /* Caller will check inner protocol */
13287 if (skb->encapsulation) {
13288 *nextp = nexthdr;
13289 return true;
13290 }
13291 *nextp = NULL;
13292 }
13293 /* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
13294 return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
13295 }
13296
13297 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
13298 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
13299 {
13300 struct udphdr *uh = udp_hdr(skb);
13301 __be16 udp_port = uh->dest;
13302
13303 if (udp_port != bp->vxlan_port && udp_port != bp->nge_port &&
13304 udp_port != bp->vxlan_gpe_port)
13305 return false;
13306 if (skb->inner_protocol == htons(ETH_P_TEB)) {
13307 struct ethhdr *eh = inner_eth_hdr(skb);
13308
13309 switch (eh->h_proto) {
13310 case htons(ETH_P_IP):
13311 return true;
13312 case htons(ETH_P_IPV6):
13313 return bnxt_exthdr_check(bp, skb,
13314 skb_inner_network_offset(skb),
13315 NULL);
13316 }
13317 } else if (skb->inner_protocol == htons(ETH_P_IP)) {
13318 return true;
13319 } else if (skb->inner_protocol == htons(ETH_P_IPV6)) {
13320 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
13321 NULL);
13322 }
13323 return false;
13324 }
13325
13326 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
13327 {
13328 switch (l4_proto) {
13329 case IPPROTO_UDP:
13330 return bnxt_udp_tunl_check(bp, skb);
13331 case IPPROTO_IPIP:
13332 return true;
13333 case IPPROTO_GRE: {
13334 switch (skb->inner_protocol) {
13335 default:
13336 return false;
13337 case htons(ETH_P_IP):
13338 return true;
13339 case htons(ETH_P_IPV6):
13340 fallthrough;
13341 }
13342 }
13343 case IPPROTO_IPV6:
13344 /* Check ext headers of inner ipv6 */
13345 return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
13346 NULL);
13347 }
13348 return false;
13349 }
13350
13351 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
13352 struct net_device *dev,
13353 netdev_features_t features)
13354 {
13355 struct bnxt *bp = netdev_priv(dev);
13356 u8 *l4_proto;
13357
13358 features = vlan_features_check(skb, features);
13359 switch (vlan_get_protocol(skb)) {
13360 case htons(ETH_P_IP):
13361 if (!skb->encapsulation)
13362 return features;
13363 l4_proto = &ip_hdr(skb)->protocol;
13364 if (bnxt_tunl_check(bp, skb, *l4_proto))
13365 return features;
13366 break;
13367 case htons(ETH_P_IPV6):
13368 if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
13369 &l4_proto))
13370 break;
13371 if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
13372 return features;
13373 break;
13374 }
13375 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13376 }
13377
13378 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
13379 u32 *reg_buf)
13380 {
13381 struct hwrm_dbg_read_direct_output *resp;
13382 struct hwrm_dbg_read_direct_input *req;
13383 __le32 *dbg_reg_buf;
13384 dma_addr_t mapping;
13385 int rc, i;
13386
13387 rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
13388 if (rc)
13389 return rc;
13390
13391 dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
13392 &mapping);
13393 if (!dbg_reg_buf) {
13394 rc = -ENOMEM;
13395 goto dbg_rd_reg_exit;
13396 }
13397
13398 req->host_dest_addr = cpu_to_le64(mapping);
13399
13400 resp = hwrm_req_hold(bp, req);
13401 req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
13402 req->read_len32 = cpu_to_le32(num_words);
13403
13404 rc = hwrm_req_send(bp, req);
13405 if (rc || resp->error_code) {
13406 rc = -EIO;
13407 goto dbg_rd_reg_exit;
13408 }
13409 for (i = 0; i < num_words; i++)
13410 reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
13411
13412 dbg_rd_reg_exit:
13413 hwrm_req_drop(bp, req);
13414 return rc;
13415 }
13416
13417 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
13418 u32 ring_id, u32 *prod, u32 *cons)
13419 {
13420 struct hwrm_dbg_ring_info_get_output *resp;
13421 struct hwrm_dbg_ring_info_get_input *req;
13422 int rc;
13423
13424 rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
13425 if (rc)
13426 return rc;
13427
13428 req->ring_type = ring_type;
13429 req->fw_ring_id = cpu_to_le32(ring_id);
13430 resp = hwrm_req_hold(bp, req);
13431 rc = hwrm_req_send(bp, req);
13432 if (!rc) {
13433 *prod = le32_to_cpu(resp->producer_index);
13434 *cons = le32_to_cpu(resp->consumer_index);
13435 }
13436 hwrm_req_drop(bp, req);
13437 return rc;
13438 }
13439
13440 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
13441 {
13442 struct bnxt_tx_ring_info *txr;
13443 int i = bnapi->index, j;
13444
13445 bnxt_for_each_napi_tx(j, bnapi, txr)
13446 netdev_info(bnapi->bp->dev, "[%d.%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
13447 i, j, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
13448 txr->tx_cons);
13449 }
13450
13451 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
13452 {
13453 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
13454 int i = bnapi->index;
13455
13456 if (!rxr)
13457 return;
13458
13459 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
13460 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
13461 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
13462 rxr->rx_sw_agg_prod);
13463 }
13464
13465 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
13466 {
13467 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
13468 int i = bnapi->index;
13469
13470 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
13471 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
13472 }
13473
13474 static void bnxt_dbg_dump_states(struct bnxt *bp)
13475 {
13476 int i;
13477 struct bnxt_napi *bnapi;
13478
13479 for (i = 0; i < bp->cp_nr_rings; i++) {
13480 bnapi = bp->bnapi[i];
13481 if (netif_msg_drv(bp)) {
13482 bnxt_dump_tx_sw_state(bnapi);
13483 bnxt_dump_rx_sw_state(bnapi);
13484 bnxt_dump_cp_sw_state(bnapi);
13485 }
13486 }
13487 }
13488
13489 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
13490 {
13491 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
13492 struct hwrm_ring_reset_input *req;
13493 struct bnxt_napi *bnapi = rxr->bnapi;
13494 struct bnxt_cp_ring_info *cpr;
13495 u16 cp_ring_id;
13496 int rc;
13497
13498 rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
13499 if (rc)
13500 return rc;
13501
13502 cpr = &bnapi->cp_ring;
13503 cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
13504 req->cmpl_ring = cpu_to_le16(cp_ring_id);
13505 req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
13506 req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
13507 return hwrm_req_send_silent(bp, req);
13508 }
13509
13510 static void bnxt_reset_task(struct bnxt *bp, bool silent)
13511 {
13512 if (!silent)
13513 bnxt_dbg_dump_states(bp);
13514 if (netif_running(bp->dev)) {
13515 bnxt_close_nic(bp, !silent, false);
13516 bnxt_open_nic(bp, !silent, false);
13517 }
13518 }
13519
13520 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
13521 {
13522 struct bnxt *bp = netdev_priv(dev);
13523
13524 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
13525 bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
13526 }
13527
13528 static void bnxt_fw_health_check(struct bnxt *bp)
13529 {
13530 struct bnxt_fw_health *fw_health = bp->fw_health;
13531 struct pci_dev *pdev = bp->pdev;
13532 u32 val;
13533
13534 if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
13535 return;
13536
13537 /* Make sure it is enabled before checking the tmr_counter. */
13538 smp_rmb();
13539 if (fw_health->tmr_counter) {
13540 fw_health->tmr_counter--;
13541 return;
13542 }
13543
13544 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
13545 if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
13546 fw_health->arrests++;
13547 goto fw_reset;
13548 }
13549
13550 fw_health->last_fw_heartbeat = val;
13551
13552 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13553 if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
13554 fw_health->discoveries++;
13555 goto fw_reset;
13556 }
13557
13558 fw_health->tmr_counter = fw_health->tmr_multiplier;
13559 return;
13560
13561 fw_reset:
13562 bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
13563 }
13564
13565 static void bnxt_timer(struct timer_list *t)
13566 {
13567 struct bnxt *bp = from_timer(bp, t, timer);
13568 struct net_device *dev = bp->dev;
13569
13570 if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
13571 return;
13572
13573 if (atomic_read(&bp->intr_sem) != 0)
13574 goto bnxt_restart_timer;
13575
13576 if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
13577 bnxt_fw_health_check(bp);
13578
13579 if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
13580 bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
13581
13582 if (bnxt_tc_flower_enabled(bp))
13583 bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
13584
13585 #ifdef CONFIG_RFS_ACCEL
13586 if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
13587 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
13588 #endif /*CONFIG_RFS_ACCEL*/
13589
13590 if (bp->link_info.phy_retry) {
13591 if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
13592 bp->link_info.phy_retry = false;
13593 netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
13594 } else {
13595 bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
13596 }
13597 }
13598
13599 if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
13600 bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
13601
13602 if ((BNXT_CHIP_P5(bp)) && !bp->chip_rev && netif_carrier_ok(dev))
13603 bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
13604
13605 bnxt_restart_timer:
13606 mod_timer(&bp->timer, jiffies + bp->current_interval);
13607 }
13608
13609 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
13610 {
13611 /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
13612 * set. If the device is being closed, bnxt_close() may be holding
13613 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
13614 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
13615 */
13616 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13617 rtnl_lock();
13618 }
13619
13620 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
13621 {
13622 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13623 rtnl_unlock();
13624 }
13625
13626 /* Only called from bnxt_sp_task() */
13627 static void bnxt_reset(struct bnxt *bp, bool silent)
13628 {
13629 bnxt_rtnl_lock_sp(bp);
13630 if (test_bit(BNXT_STATE_OPEN, &bp->state))
13631 bnxt_reset_task(bp, silent);
13632 bnxt_rtnl_unlock_sp(bp);
13633 }
13634
13635 /* Only called from bnxt_sp_task() */
13636 static void bnxt_rx_ring_reset(struct bnxt *bp)
13637 {
13638 int i;
13639
13640 bnxt_rtnl_lock_sp(bp);
13641 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13642 bnxt_rtnl_unlock_sp(bp);
13643 return;
13644 }
13645 /* Disable and flush TPA before resetting the RX ring */
13646 if (bp->flags & BNXT_FLAG_TPA)
13647 bnxt_set_tpa(bp, false);
13648 for (i = 0; i < bp->rx_nr_rings; i++) {
13649 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
13650 struct bnxt_cp_ring_info *cpr;
13651 int rc;
13652
13653 if (!rxr->bnapi->in_reset)
13654 continue;
13655
13656 rc = bnxt_hwrm_rx_ring_reset(bp, i);
13657 if (rc) {
13658 if (rc == -EINVAL || rc == -EOPNOTSUPP)
13659 netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
13660 else
13661 netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
13662 rc);
13663 bnxt_reset_task(bp, true);
13664 break;
13665 }
13666 bnxt_free_one_rx_ring_skbs(bp, i);
13667 rxr->rx_prod = 0;
13668 rxr->rx_agg_prod = 0;
13669 rxr->rx_sw_agg_prod = 0;
13670 rxr->rx_next_cons = 0;
13671 rxr->bnapi->in_reset = false;
13672 bnxt_alloc_one_rx_ring(bp, i);
13673 cpr = &rxr->bnapi->cp_ring;
13674 cpr->sw_stats->rx.rx_resets++;
13675 if (bp->flags & BNXT_FLAG_AGG_RINGS)
13676 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
13677 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
13678 }
13679 if (bp->flags & BNXT_FLAG_TPA)
13680 bnxt_set_tpa(bp, true);
13681 bnxt_rtnl_unlock_sp(bp);
13682 }
13683
13684 static void bnxt_fw_fatal_close(struct bnxt *bp)
13685 {
13686 bnxt_tx_disable(bp);
13687 bnxt_disable_napi(bp);
13688 bnxt_disable_int_sync(bp);
13689 bnxt_free_irq(bp);
13690 bnxt_clear_int_mode(bp);
13691 pci_disable_device(bp->pdev);
13692 }
13693
13694 static void bnxt_fw_reset_close(struct bnxt *bp)
13695 {
13696 /* When firmware is in fatal state, quiesce device and disable
13697 * bus master to prevent any potential bad DMAs before freeing
13698 * kernel memory.
13699 */
13700 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
13701 u16 val = 0;
13702
13703 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
13704 if (val == 0xffff)
13705 bp->fw_reset_min_dsecs = 0;
13706 bnxt_fw_fatal_close(bp);
13707 }
13708 __bnxt_close_nic(bp, true, false);
13709 bnxt_vf_reps_free(bp);
13710 bnxt_clear_int_mode(bp);
13711 bnxt_hwrm_func_drv_unrgtr(bp);
13712 if (pci_is_enabled(bp->pdev))
13713 pci_disable_device(bp->pdev);
13714 bnxt_free_ctx_mem(bp, false);
13715 }
13716
13717 static bool is_bnxt_fw_ok(struct bnxt *bp)
13718 {
13719 struct bnxt_fw_health *fw_health = bp->fw_health;
13720 bool no_heartbeat = false, has_reset = false;
13721 u32 val;
13722
13723 val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
13724 if (val == fw_health->last_fw_heartbeat)
13725 no_heartbeat = true;
13726
13727 val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
13728 if (val != fw_health->last_fw_reset_cnt)
13729 has_reset = true;
13730
13731 if (!no_heartbeat && has_reset)
13732 return true;
13733
13734 return false;
13735 }
13736
13737 /* rtnl_lock is acquired before calling this function */
13738 static void bnxt_force_fw_reset(struct bnxt *bp)
13739 {
13740 struct bnxt_fw_health *fw_health = bp->fw_health;
13741 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
13742 u32 wait_dsecs;
13743
13744 if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
13745 test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
13746 return;
13747
13748 /* we have to serialize with bnxt_refclk_read()*/
13749 if (ptp) {
13750 unsigned long flags;
13751
13752 write_seqlock_irqsave(&ptp->ptp_lock, flags);
13753 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13754 write_sequnlock_irqrestore(&ptp->ptp_lock, flags);
13755 } else {
13756 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13757 }
13758 bnxt_fw_reset_close(bp);
13759 wait_dsecs = fw_health->master_func_wait_dsecs;
13760 if (fw_health->primary) {
13761 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
13762 wait_dsecs = 0;
13763 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
13764 } else {
13765 bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
13766 wait_dsecs = fw_health->normal_func_wait_dsecs;
13767 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13768 }
13769
13770 bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
13771 bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
13772 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
13773 }
13774
13775 void bnxt_fw_exception(struct bnxt *bp)
13776 {
13777 netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
13778 set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
13779 bnxt_ulp_stop(bp);
13780 bnxt_rtnl_lock_sp(bp);
13781 bnxt_force_fw_reset(bp);
13782 bnxt_rtnl_unlock_sp(bp);
13783 }
13784
13785 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
13786 * < 0 on error.
13787 */
13788 static int bnxt_get_registered_vfs(struct bnxt *bp)
13789 {
13790 #ifdef CONFIG_BNXT_SRIOV
13791 int rc;
13792
13793 if (!BNXT_PF(bp))
13794 return 0;
13795
13796 rc = bnxt_hwrm_func_qcfg(bp);
13797 if (rc) {
13798 netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
13799 return rc;
13800 }
13801 if (bp->pf.registered_vfs)
13802 return bp->pf.registered_vfs;
13803 if (bp->sriov_cfg)
13804 return 1;
13805 #endif
13806 return 0;
13807 }
13808
13809 void bnxt_fw_reset(struct bnxt *bp)
13810 {
13811 bnxt_ulp_stop(bp);
13812 bnxt_rtnl_lock_sp(bp);
13813 if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
13814 !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13815 struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
13816 int n = 0, tmo;
13817
13818 /* we have to serialize with bnxt_refclk_read()*/
13819 if (ptp) {
13820 unsigned long flags;
13821
13822 write_seqlock_irqsave(&ptp->ptp_lock, flags);
13823 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13824 write_sequnlock_irqrestore(&ptp->ptp_lock, flags);
13825 } else {
13826 set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13827 }
13828 if (bp->pf.active_vfs &&
13829 !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
13830 n = bnxt_get_registered_vfs(bp);
13831 if (n < 0) {
13832 netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
13833 n);
13834 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13835 dev_close(bp->dev);
13836 goto fw_reset_exit;
13837 } else if (n > 0) {
13838 u16 vf_tmo_dsecs = n * 10;
13839
13840 if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
13841 bp->fw_reset_max_dsecs = vf_tmo_dsecs;
13842 bp->fw_reset_state =
13843 BNXT_FW_RESET_STATE_POLL_VF;
13844 bnxt_queue_fw_reset_work(bp, HZ / 10);
13845 goto fw_reset_exit;
13846 }
13847 bnxt_fw_reset_close(bp);
13848 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
13849 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
13850 tmo = HZ / 10;
13851 } else {
13852 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
13853 tmo = bp->fw_reset_min_dsecs * HZ / 10;
13854 }
13855 bnxt_queue_fw_reset_work(bp, tmo);
13856 }
13857 fw_reset_exit:
13858 bnxt_rtnl_unlock_sp(bp);
13859 }
13860
13861 static void bnxt_chk_missed_irq(struct bnxt *bp)
13862 {
13863 int i;
13864
13865 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
13866 return;
13867
13868 for (i = 0; i < bp->cp_nr_rings; i++) {
13869 struct bnxt_napi *bnapi = bp->bnapi[i];
13870 struct bnxt_cp_ring_info *cpr;
13871 u32 fw_ring_id;
13872 int j;
13873
13874 if (!bnapi)
13875 continue;
13876
13877 cpr = &bnapi->cp_ring;
13878 for (j = 0; j < cpr->cp_ring_count; j++) {
13879 struct bnxt_cp_ring_info *cpr2 = &cpr->cp_ring_arr[j];
13880 u32 val[2];
13881
13882 if (cpr2->has_more_work || !bnxt_has_work(bp, cpr2))
13883 continue;
13884
13885 if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
13886 cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
13887 continue;
13888 }
13889 fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
13890 bnxt_dbg_hwrm_ring_info_get(bp,
13891 DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
13892 fw_ring_id, &val[0], &val[1]);
13893 cpr->sw_stats->cmn.missed_irqs++;
13894 }
13895 }
13896 }
13897
13898 static void bnxt_cfg_ntp_filters(struct bnxt *);
13899
13900 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
13901 {
13902 struct bnxt_link_info *link_info = &bp->link_info;
13903
13904 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
13905 link_info->autoneg = BNXT_AUTONEG_SPEED;
13906 if (bp->hwrm_spec_code >= 0x10201) {
13907 if (link_info->auto_pause_setting &
13908 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
13909 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
13910 } else {
13911 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
13912 }
13913 bnxt_set_auto_speed(link_info);
13914 } else {
13915 bnxt_set_force_speed(link_info);
13916 link_info->req_duplex = link_info->duplex_setting;
13917 }
13918 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
13919 link_info->req_flow_ctrl =
13920 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
13921 else
13922 link_info->req_flow_ctrl = link_info->force_pause_setting;
13923 }
13924
13925 static void bnxt_fw_echo_reply(struct bnxt *bp)
13926 {
13927 struct bnxt_fw_health *fw_health = bp->fw_health;
13928 struct hwrm_func_echo_response_input *req;
13929 int rc;
13930
13931 rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
13932 if (rc)
13933 return;
13934 req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
13935 req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
13936 hwrm_req_send(bp, req);
13937 }
13938
13939 static void bnxt_ulp_restart(struct bnxt *bp)
13940 {
13941 bnxt_ulp_stop(bp);
13942 bnxt_ulp_start(bp, 0);
13943 }
13944
13945 static void bnxt_sp_task(struct work_struct *work)
13946 {
13947 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
13948
13949 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13950 smp_mb__after_atomic();
13951 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
13952 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
13953 return;
13954 }
13955
13956 if (test_and_clear_bit(BNXT_RESTART_ULP_SP_EVENT, &bp->sp_event)) {
13957 bnxt_ulp_restart(bp);
13958 bnxt_reenable_sriov(bp);
13959 }
13960
13961 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
13962 bnxt_cfg_rx_mode(bp);
13963
13964 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
13965 bnxt_cfg_ntp_filters(bp);
13966 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
13967 bnxt_hwrm_exec_fwd_req(bp);
13968 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
13969 netdev_info(bp->dev, "Receive PF driver unload event!\n");
13970 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
13971 bnxt_hwrm_port_qstats(bp, 0);
13972 bnxt_hwrm_port_qstats_ext(bp, 0);
13973 bnxt_accumulate_all_stats(bp);
13974 }
13975
13976 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
13977 int rc;
13978
13979 mutex_lock(&bp->link_lock);
13980 if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
13981 &bp->sp_event))
13982 bnxt_hwrm_phy_qcaps(bp);
13983
13984 rc = bnxt_update_link(bp, true);
13985 if (rc)
13986 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
13987 rc);
13988
13989 if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
13990 &bp->sp_event))
13991 bnxt_init_ethtool_link_settings(bp);
13992 mutex_unlock(&bp->link_lock);
13993 }
13994 if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
13995 int rc;
13996
13997 mutex_lock(&bp->link_lock);
13998 rc = bnxt_update_phy_setting(bp);
13999 mutex_unlock(&bp->link_lock);
14000 if (rc) {
14001 netdev_warn(bp->dev, "update phy settings retry failed\n");
14002 } else {
14003 bp->link_info.phy_retry = false;
14004 netdev_info(bp->dev, "update phy settings retry succeeded\n");
14005 }
14006 }
14007 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
14008 mutex_lock(&bp->link_lock);
14009 bnxt_get_port_module_status(bp);
14010 mutex_unlock(&bp->link_lock);
14011 }
14012
14013 if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
14014 bnxt_tc_flow_stats_work(bp);
14015
14016 if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
14017 bnxt_chk_missed_irq(bp);
14018
14019 if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
14020 bnxt_fw_echo_reply(bp);
14021
14022 if (test_and_clear_bit(BNXT_THERMAL_THRESHOLD_SP_EVENT, &bp->sp_event))
14023 bnxt_hwmon_notify_event(bp);
14024
14025 /* These functions below will clear BNXT_STATE_IN_SP_TASK. They
14026 * must be the last functions to be called before exiting.
14027 */
14028 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
14029 bnxt_reset(bp, false);
14030
14031 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
14032 bnxt_reset(bp, true);
14033
14034 if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
14035 bnxt_rx_ring_reset(bp);
14036
14037 if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
14038 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
14039 test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
14040 bnxt_devlink_health_fw_report(bp);
14041 else
14042 bnxt_fw_reset(bp);
14043 }
14044
14045 if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
14046 if (!is_bnxt_fw_ok(bp))
14047 bnxt_devlink_health_fw_report(bp);
14048 }
14049
14050 smp_mb__before_atomic();
14051 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
14052 }
14053
14054 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
14055 int *max_cp);
14056
14057 /* Under rtnl_lock */
14058 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
14059 int tx_xdp)
14060 {
14061 int max_rx, max_tx, max_cp, tx_sets = 1, tx_cp;
14062 struct bnxt_hw_rings hwr = {0};
14063 int rx_rings = rx;
14064 int rc;
14065
14066 if (tcs)
14067 tx_sets = tcs;
14068
14069 _bnxt_get_max_rings(bp, &max_rx, &max_tx, &max_cp);
14070
14071 if (max_rx < rx_rings)
14072 return -ENOMEM;
14073
14074 if (bp->flags & BNXT_FLAG_AGG_RINGS)
14075 rx_rings <<= 1;
14076
14077 hwr.rx = rx_rings;
14078 hwr.tx = tx * tx_sets + tx_xdp;
14079 if (max_tx < hwr.tx)
14080 return -ENOMEM;
14081
14082 hwr.vnic = bnxt_get_total_vnics(bp, rx);
14083
14084 tx_cp = __bnxt_num_tx_to_cp(bp, hwr.tx, tx_sets, tx_xdp);
14085 hwr.cp = sh ? max_t(int, tx_cp, rx) : tx_cp + rx;
14086 if (max_cp < hwr.cp)
14087 return -ENOMEM;
14088 hwr.stat = hwr.cp;
14089 if (BNXT_NEW_RM(bp)) {
14090 hwr.cp += bnxt_get_ulp_msix_num_in_use(bp);
14091 hwr.stat += bnxt_get_ulp_stat_ctxs_in_use(bp);
14092 hwr.grp = rx;
14093 hwr.rss_ctx = bnxt_get_total_rss_ctxs(bp, &hwr);
14094 }
14095 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)
14096 hwr.cp_p5 = hwr.tx + rx;
14097 rc = bnxt_hwrm_check_rings(bp, &hwr);
14098 if (!rc && pci_msix_can_alloc_dyn(bp->pdev)) {
14099 if (!bnxt_ulp_registered(bp->edev)) {
14100 hwr.cp += bnxt_get_ulp_msix_num(bp);
14101 hwr.cp = min_t(int, hwr.cp, bnxt_get_max_func_irqs(bp));
14102 }
14103 if (hwr.cp > bp->total_irqs) {
14104 int total_msix = bnxt_change_msix(bp, hwr.cp);
14105
14106 if (total_msix < hwr.cp) {
14107 netdev_warn(bp->dev, "Unable to allocate %d MSIX vectors, maximum available %d\n",
14108 hwr.cp, total_msix);
14109 rc = -ENOSPC;
14110 }
14111 }
14112 }
14113 return rc;
14114 }
14115
14116 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
14117 {
14118 if (bp->bar2) {
14119 pci_iounmap(pdev, bp->bar2);
14120 bp->bar2 = NULL;
14121 }
14122
14123 if (bp->bar1) {
14124 pci_iounmap(pdev, bp->bar1);
14125 bp->bar1 = NULL;
14126 }
14127
14128 if (bp->bar0) {
14129 pci_iounmap(pdev, bp->bar0);
14130 bp->bar0 = NULL;
14131 }
14132 }
14133
14134 static void bnxt_cleanup_pci(struct bnxt *bp)
14135 {
14136 bnxt_unmap_bars(bp, bp->pdev);
14137 pci_release_regions(bp->pdev);
14138 if (pci_is_enabled(bp->pdev))
14139 pci_disable_device(bp->pdev);
14140 }
14141
14142 static void bnxt_init_dflt_coal(struct bnxt *bp)
14143 {
14144 struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
14145 struct bnxt_coal *coal;
14146 u16 flags = 0;
14147
14148 if (coal_cap->cmpl_params &
14149 RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
14150 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
14151
14152 /* Tick values in micro seconds.
14153 * 1 coal_buf x bufs_per_record = 1 completion record.
14154 */
14155 coal = &bp->rx_coal;
14156 coal->coal_ticks = 10;
14157 coal->coal_bufs = 30;
14158 coal->coal_ticks_irq = 1;
14159 coal->coal_bufs_irq = 2;
14160 coal->idle_thresh = 50;
14161 coal->bufs_per_record = 2;
14162 coal->budget = 64; /* NAPI budget */
14163 coal->flags = flags;
14164
14165 coal = &bp->tx_coal;
14166 coal->coal_ticks = 28;
14167 coal->coal_bufs = 30;
14168 coal->coal_ticks_irq = 2;
14169 coal->coal_bufs_irq = 2;
14170 coal->bufs_per_record = 1;
14171 coal->flags = flags;
14172
14173 bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
14174 }
14175
14176 /* FW that pre-reserves 1 VNIC per function */
14177 static bool bnxt_fw_pre_resv_vnics(struct bnxt *bp)
14178 {
14179 u16 fw_maj = BNXT_FW_MAJ(bp), fw_bld = BNXT_FW_BLD(bp);
14180
14181 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
14182 (fw_maj > 218 || (fw_maj == 218 && fw_bld >= 18)))
14183 return true;
14184 if ((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
14185 (fw_maj > 216 || (fw_maj == 216 && fw_bld >= 172)))
14186 return true;
14187 return false;
14188 }
14189
14190 static int bnxt_fw_init_one_p1(struct bnxt *bp)
14191 {
14192 int rc;
14193
14194 bp->fw_cap = 0;
14195 rc = bnxt_hwrm_ver_get(bp);
14196 /* FW may be unresponsive after FLR. FLR must complete within 100 msec
14197 * so wait before continuing with recovery.
14198 */
14199 if (rc)
14200 msleep(100);
14201 bnxt_try_map_fw_health_reg(bp);
14202 if (rc) {
14203 rc = bnxt_try_recover_fw(bp);
14204 if (rc)
14205 return rc;
14206 rc = bnxt_hwrm_ver_get(bp);
14207 if (rc)
14208 return rc;
14209 }
14210
14211 bnxt_nvm_cfg_ver_get(bp);
14212
14213 rc = bnxt_hwrm_func_reset(bp);
14214 if (rc)
14215 return -ENODEV;
14216
14217 bnxt_hwrm_fw_set_time(bp);
14218 return 0;
14219 }
14220
14221 static int bnxt_fw_init_one_p2(struct bnxt *bp)
14222 {
14223 int rc;
14224
14225 /* Get the MAX capabilities for this function */
14226 rc = bnxt_hwrm_func_qcaps(bp);
14227 if (rc) {
14228 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
14229 rc);
14230 return -ENODEV;
14231 }
14232
14233 rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
14234 if (rc)
14235 netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
14236 rc);
14237
14238 if (bnxt_alloc_fw_health(bp)) {
14239 netdev_warn(bp->dev, "no memory for firmware error recovery\n");
14240 } else {
14241 rc = bnxt_hwrm_error_recovery_qcfg(bp);
14242 if (rc)
14243 netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
14244 rc);
14245 }
14246
14247 rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
14248 if (rc)
14249 return -ENODEV;
14250
14251 rc = bnxt_alloc_crash_dump_mem(bp);
14252 if (rc)
14253 netdev_warn(bp->dev, "crash dump mem alloc failure rc: %d\n",
14254 rc);
14255 if (!rc) {
14256 rc = bnxt_hwrm_crash_dump_mem_cfg(bp);
14257 if (rc) {
14258 bnxt_free_crash_dump_mem(bp);
14259 netdev_warn(bp->dev,
14260 "hwrm crash dump mem failure rc: %d\n", rc);
14261 }
14262 }
14263
14264 if (bnxt_fw_pre_resv_vnics(bp))
14265 bp->fw_cap |= BNXT_FW_CAP_PRE_RESV_VNICS;
14266
14267 bnxt_hwrm_func_qcfg(bp);
14268 bnxt_hwrm_vnic_qcaps(bp);
14269 bnxt_hwrm_port_led_qcaps(bp);
14270 bnxt_ethtool_init(bp);
14271 if (bp->fw_cap & BNXT_FW_CAP_PTP)
14272 __bnxt_hwrm_ptp_qcfg(bp);
14273 bnxt_dcb_init(bp);
14274 bnxt_hwmon_init(bp);
14275 return 0;
14276 }
14277
14278 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
14279 {
14280 bp->rss_cap &= ~BNXT_RSS_CAP_UDP_RSS_CAP;
14281 bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
14282 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
14283 VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
14284 VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
14285 if (bp->rss_cap & BNXT_RSS_CAP_RSS_HASH_TYPE_DELTA)
14286 bp->rss_hash_delta = bp->rss_hash_cfg;
14287 if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
14288 bp->rss_cap |= BNXT_RSS_CAP_UDP_RSS_CAP;
14289 bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
14290 VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
14291 }
14292 }
14293
14294 static void bnxt_set_dflt_rfs(struct bnxt *bp)
14295 {
14296 struct net_device *dev = bp->dev;
14297
14298 dev->hw_features &= ~NETIF_F_NTUPLE;
14299 dev->features &= ~NETIF_F_NTUPLE;
14300 bp->flags &= ~BNXT_FLAG_RFS;
14301 if (bnxt_rfs_supported(bp)) {
14302 dev->hw_features |= NETIF_F_NTUPLE;
14303 if (bnxt_rfs_capable(bp, false)) {
14304 bp->flags |= BNXT_FLAG_RFS;
14305 dev->features |= NETIF_F_NTUPLE;
14306 }
14307 }
14308 }
14309
14310 static void bnxt_fw_init_one_p3(struct bnxt *bp)
14311 {
14312 struct pci_dev *pdev = bp->pdev;
14313
14314 bnxt_set_dflt_rss_hash_type(bp);
14315 bnxt_set_dflt_rfs(bp);
14316
14317 bnxt_get_wol_settings(bp);
14318 if (bp->flags & BNXT_FLAG_WOL_CAP)
14319 device_set_wakeup_enable(&pdev->dev, bp->wol);
14320 else
14321 device_set_wakeup_capable(&pdev->dev, false);
14322
14323 bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
14324 bnxt_hwrm_coal_params_qcaps(bp);
14325 }
14326
14327 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
14328
14329 int bnxt_fw_init_one(struct bnxt *bp)
14330 {
14331 int rc;
14332
14333 rc = bnxt_fw_init_one_p1(bp);
14334 if (rc) {
14335 netdev_err(bp->dev, "Firmware init phase 1 failed\n");
14336 return rc;
14337 }
14338 rc = bnxt_fw_init_one_p2(bp);
14339 if (rc) {
14340 netdev_err(bp->dev, "Firmware init phase 2 failed\n");
14341 return rc;
14342 }
14343 rc = bnxt_probe_phy(bp, false);
14344 if (rc)
14345 return rc;
14346 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
14347 if (rc)
14348 return rc;
14349
14350 bnxt_fw_init_one_p3(bp);
14351 return 0;
14352 }
14353
14354 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
14355 {
14356 struct bnxt_fw_health *fw_health = bp->fw_health;
14357 u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
14358 u32 val = fw_health->fw_reset_seq_vals[reg_idx];
14359 u32 reg_type, reg_off, delay_msecs;
14360
14361 delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
14362 reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
14363 reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
14364 switch (reg_type) {
14365 case BNXT_FW_HEALTH_REG_TYPE_CFG:
14366 pci_write_config_dword(bp->pdev, reg_off, val);
14367 break;
14368 case BNXT_FW_HEALTH_REG_TYPE_GRC:
14369 writel(reg_off & BNXT_GRC_BASE_MASK,
14370 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
14371 reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
14372 fallthrough;
14373 case BNXT_FW_HEALTH_REG_TYPE_BAR0:
14374 writel(val, bp->bar0 + reg_off);
14375 break;
14376 case BNXT_FW_HEALTH_REG_TYPE_BAR1:
14377 writel(val, bp->bar1 + reg_off);
14378 break;
14379 }
14380 if (delay_msecs) {
14381 pci_read_config_dword(bp->pdev, 0, &val);
14382 msleep(delay_msecs);
14383 }
14384 }
14385
14386 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
14387 {
14388 struct hwrm_func_qcfg_output *resp;
14389 struct hwrm_func_qcfg_input *req;
14390 bool result = true; /* firmware will enforce if unknown */
14391
14392 if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
14393 return result;
14394
14395 if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
14396 return result;
14397
14398 req->fid = cpu_to_le16(0xffff);
14399 resp = hwrm_req_hold(bp, req);
14400 if (!hwrm_req_send(bp, req))
14401 result = !!(le16_to_cpu(resp->flags) &
14402 FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
14403 hwrm_req_drop(bp, req);
14404 return result;
14405 }
14406
14407 static void bnxt_reset_all(struct bnxt *bp)
14408 {
14409 struct bnxt_fw_health *fw_health = bp->fw_health;
14410 int i, rc;
14411
14412 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
14413 bnxt_fw_reset_via_optee(bp);
14414 bp->fw_reset_timestamp = jiffies;
14415 return;
14416 }
14417
14418 if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
14419 for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
14420 bnxt_fw_reset_writel(bp, i);
14421 } else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
14422 struct hwrm_fw_reset_input *req;
14423
14424 rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
14425 if (!rc) {
14426 req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
14427 req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
14428 req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
14429 req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
14430 rc = hwrm_req_send(bp, req);
14431 }
14432 if (rc != -ENODEV)
14433 netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
14434 }
14435 bp->fw_reset_timestamp = jiffies;
14436 }
14437
14438 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
14439 {
14440 return time_after(jiffies, bp->fw_reset_timestamp +
14441 (bp->fw_reset_max_dsecs * HZ / 10));
14442 }
14443
14444 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
14445 {
14446 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14447 if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
14448 bnxt_dl_health_fw_status_update(bp, false);
14449 bp->fw_reset_state = 0;
14450 dev_close(bp->dev);
14451 }
14452
14453 static void bnxt_fw_reset_task(struct work_struct *work)
14454 {
14455 struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
14456 int rc = 0;
14457
14458 if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
14459 netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
14460 return;
14461 }
14462
14463 switch (bp->fw_reset_state) {
14464 case BNXT_FW_RESET_STATE_POLL_VF: {
14465 int n = bnxt_get_registered_vfs(bp);
14466 int tmo;
14467
14468 if (n < 0) {
14469 netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
14470 n, jiffies_to_msecs(jiffies -
14471 bp->fw_reset_timestamp));
14472 goto fw_reset_abort;
14473 } else if (n > 0) {
14474 if (bnxt_fw_reset_timeout(bp)) {
14475 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14476 bp->fw_reset_state = 0;
14477 netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
14478 n);
14479 goto ulp_start;
14480 }
14481 bnxt_queue_fw_reset_work(bp, HZ / 10);
14482 return;
14483 }
14484 bp->fw_reset_timestamp = jiffies;
14485 rtnl_lock();
14486 if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
14487 bnxt_fw_reset_abort(bp, rc);
14488 rtnl_unlock();
14489 goto ulp_start;
14490 }
14491 bnxt_fw_reset_close(bp);
14492 if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
14493 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
14494 tmo = HZ / 10;
14495 } else {
14496 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14497 tmo = bp->fw_reset_min_dsecs * HZ / 10;
14498 }
14499 rtnl_unlock();
14500 bnxt_queue_fw_reset_work(bp, tmo);
14501 return;
14502 }
14503 case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
14504 u32 val;
14505
14506 val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
14507 if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
14508 !bnxt_fw_reset_timeout(bp)) {
14509 bnxt_queue_fw_reset_work(bp, HZ / 5);
14510 return;
14511 }
14512
14513 if (!bp->fw_health->primary) {
14514 u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
14515
14516 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14517 bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
14518 return;
14519 }
14520 bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
14521 }
14522 fallthrough;
14523 case BNXT_FW_RESET_STATE_RESET_FW:
14524 bnxt_reset_all(bp);
14525 bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
14526 bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
14527 return;
14528 case BNXT_FW_RESET_STATE_ENABLE_DEV:
14529 bnxt_inv_fw_health_reg(bp);
14530 if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
14531 !bp->fw_reset_min_dsecs) {
14532 u16 val;
14533
14534 pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
14535 if (val == 0xffff) {
14536 if (bnxt_fw_reset_timeout(bp)) {
14537 netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
14538 rc = -ETIMEDOUT;
14539 goto fw_reset_abort;
14540 }
14541 bnxt_queue_fw_reset_work(bp, HZ / 1000);
14542 return;
14543 }
14544 }
14545 clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
14546 clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
14547 if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
14548 !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
14549 bnxt_dl_remote_reload(bp);
14550 if (pci_enable_device(bp->pdev)) {
14551 netdev_err(bp->dev, "Cannot re-enable PCI device\n");
14552 rc = -ENODEV;
14553 goto fw_reset_abort;
14554 }
14555 pci_set_master(bp->pdev);
14556 bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
14557 fallthrough;
14558 case BNXT_FW_RESET_STATE_POLL_FW:
14559 bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
14560 rc = bnxt_hwrm_poll(bp);
14561 if (rc) {
14562 if (bnxt_fw_reset_timeout(bp)) {
14563 netdev_err(bp->dev, "Firmware reset aborted\n");
14564 goto fw_reset_abort_status;
14565 }
14566 bnxt_queue_fw_reset_work(bp, HZ / 5);
14567 return;
14568 }
14569 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
14570 bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
14571 fallthrough;
14572 case BNXT_FW_RESET_STATE_OPENING:
14573 while (!rtnl_trylock()) {
14574 bnxt_queue_fw_reset_work(bp, HZ / 10);
14575 return;
14576 }
14577 rc = bnxt_open(bp->dev);
14578 if (rc) {
14579 netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
14580 bnxt_fw_reset_abort(bp, rc);
14581 rtnl_unlock();
14582 goto ulp_start;
14583 }
14584
14585 if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
14586 bp->fw_health->enabled) {
14587 bp->fw_health->last_fw_reset_cnt =
14588 bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
14589 }
14590 bp->fw_reset_state = 0;
14591 /* Make sure fw_reset_state is 0 before clearing the flag */
14592 smp_mb__before_atomic();
14593 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14594 bnxt_ptp_reapply_pps(bp);
14595 clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
14596 if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
14597 bnxt_dl_health_fw_recovery_done(bp);
14598 bnxt_dl_health_fw_status_update(bp, true);
14599 }
14600 rtnl_unlock();
14601 bnxt_ulp_start(bp, 0);
14602 bnxt_reenable_sriov(bp);
14603 rtnl_lock();
14604 bnxt_vf_reps_alloc(bp);
14605 bnxt_vf_reps_open(bp);
14606 rtnl_unlock();
14607 break;
14608 }
14609 return;
14610
14611 fw_reset_abort_status:
14612 if (bp->fw_health->status_reliable ||
14613 (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
14614 u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
14615
14616 netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
14617 }
14618 fw_reset_abort:
14619 rtnl_lock();
14620 bnxt_fw_reset_abort(bp, rc);
14621 rtnl_unlock();
14622 ulp_start:
14623 bnxt_ulp_start(bp, rc);
14624 }
14625
14626 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
14627 {
14628 int rc;
14629 struct bnxt *bp = netdev_priv(dev);
14630
14631 SET_NETDEV_DEV(dev, &pdev->dev);
14632
14633 /* enable device (incl. PCI PM wakeup), and bus-mastering */
14634 rc = pci_enable_device(pdev);
14635 if (rc) {
14636 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
14637 goto init_err;
14638 }
14639
14640 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
14641 dev_err(&pdev->dev,
14642 "Cannot find PCI device base address, aborting\n");
14643 rc = -ENODEV;
14644 goto init_err_disable;
14645 }
14646
14647 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
14648 if (rc) {
14649 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
14650 goto init_err_disable;
14651 }
14652
14653 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
14654 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
14655 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
14656 rc = -EIO;
14657 goto init_err_release;
14658 }
14659
14660 pci_set_master(pdev);
14661
14662 bp->dev = dev;
14663 bp->pdev = pdev;
14664
14665 /* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
14666 * determines the BAR size.
14667 */
14668 bp->bar0 = pci_ioremap_bar(pdev, 0);
14669 if (!bp->bar0) {
14670 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
14671 rc = -ENOMEM;
14672 goto init_err_release;
14673 }
14674
14675 bp->bar2 = pci_ioremap_bar(pdev, 4);
14676 if (!bp->bar2) {
14677 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
14678 rc = -ENOMEM;
14679 goto init_err_release;
14680 }
14681
14682 INIT_WORK(&bp->sp_task, bnxt_sp_task);
14683 INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
14684
14685 spin_lock_init(&bp->ntp_fltr_lock);
14686 #if BITS_PER_LONG == 32
14687 spin_lock_init(&bp->db_lock);
14688 #endif
14689
14690 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
14691 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
14692
14693 timer_setup(&bp->timer, bnxt_timer, 0);
14694 bp->current_interval = BNXT_TIMER_INTERVAL;
14695
14696 bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
14697 bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
14698
14699 clear_bit(BNXT_STATE_OPEN, &bp->state);
14700 return 0;
14701
14702 init_err_release:
14703 bnxt_unmap_bars(bp, pdev);
14704 pci_release_regions(pdev);
14705
14706 init_err_disable:
14707 pci_disable_device(pdev);
14708
14709 init_err:
14710 return rc;
14711 }
14712
14713 /* rtnl_lock held */
14714 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
14715 {
14716 struct sockaddr *addr = p;
14717 struct bnxt *bp = netdev_priv(dev);
14718 int rc = 0;
14719
14720 if (!is_valid_ether_addr(addr->sa_data))
14721 return -EADDRNOTAVAIL;
14722
14723 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
14724 return 0;
14725
14726 rc = bnxt_approve_mac(bp, addr->sa_data, true);
14727 if (rc)
14728 return rc;
14729
14730 eth_hw_addr_set(dev, addr->sa_data);
14731 bnxt_clear_usr_fltrs(bp, true);
14732 if (netif_running(dev)) {
14733 bnxt_close_nic(bp, false, false);
14734 rc = bnxt_open_nic(bp, false, false);
14735 }
14736
14737 return rc;
14738 }
14739
14740 /* rtnl_lock held */
14741 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
14742 {
14743 struct bnxt *bp = netdev_priv(dev);
14744
14745 if (netif_running(dev))
14746 bnxt_close_nic(bp, true, false);
14747
14748 WRITE_ONCE(dev->mtu, new_mtu);
14749
14750 /* MTU change may change the AGG ring settings if an XDP multi-buffer
14751 * program is attached. We need to set the AGG rings settings and
14752 * rx_skb_func accordingly.
14753 */
14754 if (READ_ONCE(bp->xdp_prog))
14755 bnxt_set_rx_skb_mode(bp, true);
14756
14757 bnxt_set_ring_params(bp);
14758
14759 if (netif_running(dev))
14760 return bnxt_open_nic(bp, true, false);
14761
14762 return 0;
14763 }
14764
14765 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
14766 {
14767 struct bnxt *bp = netdev_priv(dev);
14768 bool sh = false;
14769 int rc, tx_cp;
14770
14771 if (tc > bp->max_tc) {
14772 netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
14773 tc, bp->max_tc);
14774 return -EINVAL;
14775 }
14776
14777 if (bp->num_tc == tc)
14778 return 0;
14779
14780 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
14781 sh = true;
14782
14783 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
14784 sh, tc, bp->tx_nr_rings_xdp);
14785 if (rc)
14786 return rc;
14787
14788 /* Needs to close the device and do hw resource re-allocations */
14789 if (netif_running(bp->dev))
14790 bnxt_close_nic(bp, true, false);
14791
14792 if (tc) {
14793 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
14794 netdev_set_num_tc(dev, tc);
14795 bp->num_tc = tc;
14796 } else {
14797 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
14798 netdev_reset_tc(dev);
14799 bp->num_tc = 0;
14800 }
14801 bp->tx_nr_rings += bp->tx_nr_rings_xdp;
14802 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);
14803 bp->cp_nr_rings = sh ? max_t(int, tx_cp, bp->rx_nr_rings) :
14804 tx_cp + bp->rx_nr_rings;
14805
14806 if (netif_running(bp->dev))
14807 return bnxt_open_nic(bp, true, false);
14808
14809 return 0;
14810 }
14811
14812 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
14813 void *cb_priv)
14814 {
14815 struct bnxt *bp = cb_priv;
14816
14817 if (!bnxt_tc_flower_enabled(bp) ||
14818 !tc_cls_can_offload_and_chain0(bp->dev, type_data))
14819 return -EOPNOTSUPP;
14820
14821 switch (type) {
14822 case TC_SETUP_CLSFLOWER:
14823 return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
14824 default:
14825 return -EOPNOTSUPP;
14826 }
14827 }
14828
14829 LIST_HEAD(bnxt_block_cb_list);
14830
14831 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
14832 void *type_data)
14833 {
14834 struct bnxt *bp = netdev_priv(dev);
14835
14836 switch (type) {
14837 case TC_SETUP_BLOCK:
14838 return flow_block_cb_setup_simple(type_data,
14839 &bnxt_block_cb_list,
14840 bnxt_setup_tc_block_cb,
14841 bp, bp, true);
14842 case TC_SETUP_QDISC_MQPRIO: {
14843 struct tc_mqprio_qopt *mqprio = type_data;
14844
14845 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
14846
14847 return bnxt_setup_mq_tc(dev, mqprio->num_tc);
14848 }
14849 default:
14850 return -EOPNOTSUPP;
14851 }
14852 }
14853
14854 u32 bnxt_get_ntp_filter_idx(struct bnxt *bp, struct flow_keys *fkeys,
14855 const struct sk_buff *skb)
14856 {
14857 struct bnxt_vnic_info *vnic;
14858
14859 if (skb)
14860 return skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
14861
14862 vnic = &bp->vnic_info[BNXT_VNIC_DEFAULT];
14863 return bnxt_toeplitz(bp, fkeys, (void *)vnic->rss_hash_key);
14864 }
14865
14866 int bnxt_insert_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr,
14867 u32 idx)
14868 {
14869 struct hlist_head *head;
14870 int bit_id;
14871
14872 spin_lock_bh(&bp->ntp_fltr_lock);
14873 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap, bp->max_fltr, 0);
14874 if (bit_id < 0) {
14875 spin_unlock_bh(&bp->ntp_fltr_lock);
14876 return -ENOMEM;
14877 }
14878
14879 fltr->base.sw_id = (u16)bit_id;
14880 fltr->base.type = BNXT_FLTR_TYPE_NTUPLE;
14881 fltr->base.flags |= BNXT_ACT_RING_DST;
14882 head = &bp->ntp_fltr_hash_tbl[idx];
14883 hlist_add_head_rcu(&fltr->base.hash, head);
14884 set_bit(BNXT_FLTR_INSERTED, &fltr->base.state);
14885 bnxt_insert_usr_fltr(bp, &fltr->base);
14886 bp->ntp_fltr_count++;
14887 spin_unlock_bh(&bp->ntp_fltr_lock);
14888 return 0;
14889 }
14890
14891 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
14892 struct bnxt_ntuple_filter *f2)
14893 {
14894 struct bnxt_flow_masks *masks1 = &f1->fmasks;
14895 struct bnxt_flow_masks *masks2 = &f2->fmasks;
14896 struct flow_keys *keys1 = &f1->fkeys;
14897 struct flow_keys *keys2 = &f2->fkeys;
14898
14899 if (keys1->basic.n_proto != keys2->basic.n_proto ||
14900 keys1->basic.ip_proto != keys2->basic.ip_proto)
14901 return false;
14902
14903 if (keys1->basic.n_proto == htons(ETH_P_IP)) {
14904 if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
14905 masks1->addrs.v4addrs.src != masks2->addrs.v4addrs.src ||
14906 keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst ||
14907 masks1->addrs.v4addrs.dst != masks2->addrs.v4addrs.dst)
14908 return false;
14909 } else {
14910 if (!ipv6_addr_equal(&keys1->addrs.v6addrs.src,
14911 &keys2->addrs.v6addrs.src) ||
14912 !ipv6_addr_equal(&masks1->addrs.v6addrs.src,
14913 &masks2->addrs.v6addrs.src) ||
14914 !ipv6_addr_equal(&keys1->addrs.v6addrs.dst,
14915 &keys2->addrs.v6addrs.dst) ||
14916 !ipv6_addr_equal(&masks1->addrs.v6addrs.dst,
14917 &masks2->addrs.v6addrs.dst))
14918 return false;
14919 }
14920
14921 return keys1->ports.src == keys2->ports.src &&
14922 masks1->ports.src == masks2->ports.src &&
14923 keys1->ports.dst == keys2->ports.dst &&
14924 masks1->ports.dst == masks2->ports.dst &&
14925 keys1->control.flags == keys2->control.flags &&
14926 f1->l2_fltr == f2->l2_fltr;
14927 }
14928
14929 struct bnxt_ntuple_filter *
14930 bnxt_lookup_ntp_filter_from_idx(struct bnxt *bp,
14931 struct bnxt_ntuple_filter *fltr, u32 idx)
14932 {
14933 struct bnxt_ntuple_filter *f;
14934 struct hlist_head *head;
14935
14936 head = &bp->ntp_fltr_hash_tbl[idx];
14937 hlist_for_each_entry_rcu(f, head, base.hash) {
14938 if (bnxt_fltr_match(f, fltr))
14939 return f;
14940 }
14941 return NULL;
14942 }
14943
14944 #ifdef CONFIG_RFS_ACCEL
14945 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
14946 u16 rxq_index, u32 flow_id)
14947 {
14948 struct bnxt *bp = netdev_priv(dev);
14949 struct bnxt_ntuple_filter *fltr, *new_fltr;
14950 struct flow_keys *fkeys;
14951 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
14952 struct bnxt_l2_filter *l2_fltr;
14953 int rc = 0, idx;
14954 u32 flags;
14955
14956 if (ether_addr_equal(dev->dev_addr, eth->h_dest)) {
14957 l2_fltr = bp->vnic_info[BNXT_VNIC_DEFAULT].l2_filters[0];
14958 atomic_inc(&l2_fltr->refcnt);
14959 } else {
14960 struct bnxt_l2_key key;
14961
14962 ether_addr_copy(key.dst_mac_addr, eth->h_dest);
14963 key.vlan = 0;
14964 l2_fltr = bnxt_lookup_l2_filter_from_key(bp, &key);
14965 if (!l2_fltr)
14966 return -EINVAL;
14967 if (l2_fltr->base.flags & BNXT_ACT_FUNC_DST) {
14968 bnxt_del_l2_filter(bp, l2_fltr);
14969 return -EINVAL;
14970 }
14971 }
14972 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
14973 if (!new_fltr) {
14974 bnxt_del_l2_filter(bp, l2_fltr);
14975 return -ENOMEM;
14976 }
14977
14978 fkeys = &new_fltr->fkeys;
14979 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
14980 rc = -EPROTONOSUPPORT;
14981 goto err_free;
14982 }
14983
14984 if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
14985 fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
14986 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
14987 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
14988 rc = -EPROTONOSUPPORT;
14989 goto err_free;
14990 }
14991 new_fltr->fmasks = BNXT_FLOW_IPV4_MASK_ALL;
14992 if (fkeys->basic.n_proto == htons(ETH_P_IPV6)) {
14993 if (bp->hwrm_spec_code < 0x10601) {
14994 rc = -EPROTONOSUPPORT;
14995 goto err_free;
14996 }
14997 new_fltr->fmasks = BNXT_FLOW_IPV6_MASK_ALL;
14998 }
14999 flags = fkeys->control.flags;
15000 if (((flags & FLOW_DIS_ENCAPSULATION) &&
15001 bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
15002 rc = -EPROTONOSUPPORT;
15003 goto err_free;
15004 }
15005 new_fltr->l2_fltr = l2_fltr;
15006
15007 idx = bnxt_get_ntp_filter_idx(bp, fkeys, skb);
15008 rcu_read_lock();
15009 fltr = bnxt_lookup_ntp_filter_from_idx(bp, new_fltr, idx);
15010 if (fltr) {
15011 rc = fltr->base.sw_id;
15012 rcu_read_unlock();
15013 goto err_free;
15014 }
15015 rcu_read_unlock();
15016
15017 new_fltr->flow_id = flow_id;
15018 new_fltr->base.rxq = rxq_index;
15019 rc = bnxt_insert_ntp_filter(bp, new_fltr, idx);
15020 if (!rc) {
15021 bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
15022 return new_fltr->base.sw_id;
15023 }
15024
15025 err_free:
15026 bnxt_del_l2_filter(bp, l2_fltr);
15027 kfree(new_fltr);
15028 return rc;
15029 }
15030 #endif
15031
15032 void bnxt_del_ntp_filter(struct bnxt *bp, struct bnxt_ntuple_filter *fltr)
15033 {
15034 spin_lock_bh(&bp->ntp_fltr_lock);
15035 if (!test_and_clear_bit(BNXT_FLTR_INSERTED, &fltr->base.state)) {
15036 spin_unlock_bh(&bp->ntp_fltr_lock);
15037 return;
15038 }
15039 hlist_del_rcu(&fltr->base.hash);
15040 bnxt_del_one_usr_fltr(bp, &fltr->base);
15041 bp->ntp_fltr_count--;
15042 spin_unlock_bh(&bp->ntp_fltr_lock);
15043 bnxt_del_l2_filter(bp, fltr->l2_fltr);
15044 clear_bit(fltr->base.sw_id, bp->ntp_fltr_bmap);
15045 kfree_rcu(fltr, base.rcu);
15046 }
15047
15048 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
15049 {
15050 #ifdef CONFIG_RFS_ACCEL
15051 int i;
15052
15053 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
15054 struct hlist_head *head;
15055 struct hlist_node *tmp;
15056 struct bnxt_ntuple_filter *fltr;
15057 int rc;
15058
15059 head = &bp->ntp_fltr_hash_tbl[i];
15060 hlist_for_each_entry_safe(fltr, tmp, head, base.hash) {
15061 bool del = false;
15062
15063 if (test_bit(BNXT_FLTR_VALID, &fltr->base.state)) {
15064 if (fltr->base.flags & BNXT_ACT_NO_AGING)
15065 continue;
15066 if (rps_may_expire_flow(bp->dev, fltr->base.rxq,
15067 fltr->flow_id,
15068 fltr->base.sw_id)) {
15069 bnxt_hwrm_cfa_ntuple_filter_free(bp,
15070 fltr);
15071 del = true;
15072 }
15073 } else {
15074 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
15075 fltr);
15076 if (rc)
15077 del = true;
15078 else
15079 set_bit(BNXT_FLTR_VALID, &fltr->base.state);
15080 }
15081
15082 if (del)
15083 bnxt_del_ntp_filter(bp, fltr);
15084 }
15085 }
15086 #endif
15087 }
15088
15089 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
15090 unsigned int entry, struct udp_tunnel_info *ti)
15091 {
15092 struct bnxt *bp = netdev_priv(netdev);
15093 unsigned int cmd;
15094
15095 if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
15096 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
15097 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
15098 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE;
15099 else
15100 cmd = TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE;
15101
15102 return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
15103 }
15104
15105 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
15106 unsigned int entry, struct udp_tunnel_info *ti)
15107 {
15108 struct bnxt *bp = netdev_priv(netdev);
15109 unsigned int cmd;
15110
15111 if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
15112 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
15113 else if (ti->type == UDP_TUNNEL_TYPE_GENEVE)
15114 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
15115 else
15116 cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN_GPE;
15117
15118 return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
15119 }
15120
15121 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
15122 .set_port = bnxt_udp_tunnel_set_port,
15123 .unset_port = bnxt_udp_tunnel_unset_port,
15124 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
15125 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
15126 .tables = {
15127 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
15128 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
15129 },
15130 }, bnxt_udp_tunnels_p7 = {
15131 .set_port = bnxt_udp_tunnel_set_port,
15132 .unset_port = bnxt_udp_tunnel_unset_port,
15133 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
15134 UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
15135 .tables = {
15136 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
15137 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
15138 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN_GPE, },
15139 },
15140 };
15141
15142 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
15143 struct net_device *dev, u32 filter_mask,
15144 int nlflags)
15145 {
15146 struct bnxt *bp = netdev_priv(dev);
15147
15148 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
15149 nlflags, filter_mask, NULL);
15150 }
15151
15152 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
15153 u16 flags, struct netlink_ext_ack *extack)
15154 {
15155 struct bnxt *bp = netdev_priv(dev);
15156 struct nlattr *attr, *br_spec;
15157 int rem, rc = 0;
15158
15159 if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
15160 return -EOPNOTSUPP;
15161
15162 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
15163 if (!br_spec)
15164 return -EINVAL;
15165
15166 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
15167 u16 mode;
15168
15169 mode = nla_get_u16(attr);
15170 if (mode == bp->br_mode)
15171 break;
15172
15173 rc = bnxt_hwrm_set_br_mode(bp, mode);
15174 if (!rc)
15175 bp->br_mode = mode;
15176 break;
15177 }
15178 return rc;
15179 }
15180
15181 int bnxt_get_port_parent_id(struct net_device *dev,
15182 struct netdev_phys_item_id *ppid)
15183 {
15184 struct bnxt *bp = netdev_priv(dev);
15185
15186 if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
15187 return -EOPNOTSUPP;
15188
15189 /* The PF and it's VF-reps only support the switchdev framework */
15190 if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
15191 return -EOPNOTSUPP;
15192
15193 ppid->id_len = sizeof(bp->dsn);
15194 memcpy(ppid->id, bp->dsn, ppid->id_len);
15195
15196 return 0;
15197 }
15198
15199 static const struct net_device_ops bnxt_netdev_ops = {
15200 .ndo_open = bnxt_open,
15201 .ndo_start_xmit = bnxt_start_xmit,
15202 .ndo_stop = bnxt_close,
15203 .ndo_get_stats64 = bnxt_get_stats64,
15204 .ndo_set_rx_mode = bnxt_set_rx_mode,
15205 .ndo_eth_ioctl = bnxt_ioctl,
15206 .ndo_validate_addr = eth_validate_addr,
15207 .ndo_set_mac_address = bnxt_change_mac_addr,
15208 .ndo_change_mtu = bnxt_change_mtu,
15209 .ndo_fix_features = bnxt_fix_features,
15210 .ndo_set_features = bnxt_set_features,
15211 .ndo_features_check = bnxt_features_check,
15212 .ndo_tx_timeout = bnxt_tx_timeout,
15213 #ifdef CONFIG_BNXT_SRIOV
15214 .ndo_get_vf_config = bnxt_get_vf_config,
15215 .ndo_set_vf_mac = bnxt_set_vf_mac,
15216 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
15217 .ndo_set_vf_rate = bnxt_set_vf_bw,
15218 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
15219 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
15220 .ndo_set_vf_trust = bnxt_set_vf_trust,
15221 #endif
15222 .ndo_setup_tc = bnxt_setup_tc,
15223 #ifdef CONFIG_RFS_ACCEL
15224 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
15225 #endif
15226 .ndo_bpf = bnxt_xdp,
15227 .ndo_xdp_xmit = bnxt_xdp_xmit,
15228 .ndo_bridge_getlink = bnxt_bridge_getlink,
15229 .ndo_bridge_setlink = bnxt_bridge_setlink,
15230 };
15231
15232 static void bnxt_get_queue_stats_rx(struct net_device *dev, int i,
15233 struct netdev_queue_stats_rx *stats)
15234 {
15235 struct bnxt *bp = netdev_priv(dev);
15236 struct bnxt_cp_ring_info *cpr;
15237 u64 *sw;
15238
15239 cpr = &bp->bnapi[i]->cp_ring;
15240 sw = cpr->stats.sw_stats;
15241
15242 stats->packets = 0;
15243 stats->packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
15244 stats->packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
15245 stats->packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
15246
15247 stats->bytes = 0;
15248 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
15249 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
15250 stats->bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
15251
15252 stats->alloc_fail = cpr->sw_stats->rx.rx_oom_discards;
15253 }
15254
15255 static void bnxt_get_queue_stats_tx(struct net_device *dev, int i,
15256 struct netdev_queue_stats_tx *stats)
15257 {
15258 struct bnxt *bp = netdev_priv(dev);
15259 struct bnxt_napi *bnapi;
15260 u64 *sw;
15261
15262 bnapi = bp->tx_ring[bp->tx_ring_map[i]].bnapi;
15263 sw = bnapi->cp_ring.stats.sw_stats;
15264
15265 stats->packets = 0;
15266 stats->packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
15267 stats->packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
15268 stats->packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
15269
15270 stats->bytes = 0;
15271 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
15272 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
15273 stats->bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
15274 }
15275
15276 static void bnxt_get_base_stats(struct net_device *dev,
15277 struct netdev_queue_stats_rx *rx,
15278 struct netdev_queue_stats_tx *tx)
15279 {
15280 struct bnxt *bp = netdev_priv(dev);
15281
15282 rx->packets = bp->net_stats_prev.rx_packets;
15283 rx->bytes = bp->net_stats_prev.rx_bytes;
15284 rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards;
15285
15286 tx->packets = bp->net_stats_prev.tx_packets;
15287 tx->bytes = bp->net_stats_prev.tx_bytes;
15288 }
15289
15290 static const struct netdev_stat_ops bnxt_stat_ops = {
15291 .get_queue_stats_rx = bnxt_get_queue_stats_rx,
15292 .get_queue_stats_tx = bnxt_get_queue_stats_tx,
15293 .get_base_stats = bnxt_get_base_stats,
15294 };
15295
15296 static int bnxt_alloc_rx_agg_bmap(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
15297 {
15298 u16 mem_size;
15299
15300 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
15301 mem_size = rxr->rx_agg_bmap_size / 8;
15302 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
15303 if (!rxr->rx_agg_bmap)
15304 return -ENOMEM;
15305
15306 return 0;
15307 }
15308
15309 static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx)
15310 {
15311 struct bnxt_rx_ring_info *rxr, *clone;
15312 struct bnxt *bp = netdev_priv(dev);
15313 struct bnxt_ring_struct *ring;
15314 int rc;
15315
15316 rxr = &bp->rx_ring[idx];
15317 clone = qmem;
15318 memcpy(clone, rxr, sizeof(*rxr));
15319 bnxt_init_rx_ring_struct(bp, clone);
15320 bnxt_reset_rx_ring_struct(bp, clone);
15321
15322 clone->rx_prod = 0;
15323 clone->rx_agg_prod = 0;
15324 clone->rx_sw_agg_prod = 0;
15325 clone->rx_next_cons = 0;
15326
15327 rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid);
15328 if (rc)
15329 return rc;
15330
15331 rc = xdp_rxq_info_reg(&clone->xdp_rxq, bp->dev, idx, 0);
15332 if (rc < 0)
15333 goto err_page_pool_destroy;
15334
15335 rc = xdp_rxq_info_reg_mem_model(&clone->xdp_rxq,
15336 MEM_TYPE_PAGE_POOL,
15337 clone->page_pool);
15338 if (rc)
15339 goto err_rxq_info_unreg;
15340
15341 ring = &clone->rx_ring_struct;
15342 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
15343 if (rc)
15344 goto err_free_rx_ring;
15345
15346 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
15347 ring = &clone->rx_agg_ring_struct;
15348 rc = bnxt_alloc_ring(bp, &ring->ring_mem);
15349 if (rc)
15350 goto err_free_rx_agg_ring;
15351
15352 rc = bnxt_alloc_rx_agg_bmap(bp, clone);
15353 if (rc)
15354 goto err_free_rx_agg_ring;
15355 }
15356
15357 bnxt_init_one_rx_ring_rxbd(bp, clone);
15358 bnxt_init_one_rx_agg_ring_rxbd(bp, clone);
15359
15360 bnxt_alloc_one_rx_ring_skb(bp, clone, idx);
15361 if (bp->flags & BNXT_FLAG_AGG_RINGS)
15362 bnxt_alloc_one_rx_ring_page(bp, clone, idx);
15363
15364 return 0;
15365
15366 err_free_rx_agg_ring:
15367 bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem);
15368 err_free_rx_ring:
15369 bnxt_free_ring(bp, &clone->rx_ring_struct.ring_mem);
15370 err_rxq_info_unreg:
15371 xdp_rxq_info_unreg(&clone->xdp_rxq);
15372 err_page_pool_destroy:
15373 clone->page_pool->p.napi = NULL;
15374 page_pool_destroy(clone->page_pool);
15375 clone->page_pool = NULL;
15376 return rc;
15377 }
15378
15379 static void bnxt_queue_mem_free(struct net_device *dev, void *qmem)
15380 {
15381 struct bnxt_rx_ring_info *rxr = qmem;
15382 struct bnxt *bp = netdev_priv(dev);
15383 struct bnxt_ring_struct *ring;
15384
15385 bnxt_free_one_rx_ring(bp, rxr);
15386 bnxt_free_one_rx_agg_ring(bp, rxr);
15387
15388 xdp_rxq_info_unreg(&rxr->xdp_rxq);
15389
15390 page_pool_destroy(rxr->page_pool);
15391 rxr->page_pool = NULL;
15392
15393 ring = &rxr->rx_ring_struct;
15394 bnxt_free_ring(bp, &ring->ring_mem);
15395
15396 ring = &rxr->rx_agg_ring_struct;
15397 bnxt_free_ring(bp, &ring->ring_mem);
15398
15399 kfree(rxr->rx_agg_bmap);
15400 rxr->rx_agg_bmap = NULL;
15401 }
15402
15403 static void bnxt_copy_rx_ring(struct bnxt *bp,
15404 struct bnxt_rx_ring_info *dst,
15405 struct bnxt_rx_ring_info *src)
15406 {
15407 struct bnxt_ring_mem_info *dst_rmem, *src_rmem;
15408 struct bnxt_ring_struct *dst_ring, *src_ring;
15409 int i;
15410
15411 dst_ring = &dst->rx_ring_struct;
15412 dst_rmem = &dst_ring->ring_mem;
15413 src_ring = &src->rx_ring_struct;
15414 src_rmem = &src_ring->ring_mem;
15415
15416 WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages);
15417 WARN_ON(dst_rmem->page_size != src_rmem->page_size);
15418 WARN_ON(dst_rmem->flags != src_rmem->flags);
15419 WARN_ON(dst_rmem->depth != src_rmem->depth);
15420 WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size);
15421 WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem);
15422
15423 dst_rmem->pg_tbl = src_rmem->pg_tbl;
15424 dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map;
15425 *dst_rmem->vmem = *src_rmem->vmem;
15426 for (i = 0; i < dst_rmem->nr_pages; i++) {
15427 dst_rmem->pg_arr[i] = src_rmem->pg_arr[i];
15428 dst_rmem->dma_arr[i] = src_rmem->dma_arr[i];
15429 }
15430
15431 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
15432 return;
15433
15434 dst_ring = &dst->rx_agg_ring_struct;
15435 dst_rmem = &dst_ring->ring_mem;
15436 src_ring = &src->rx_agg_ring_struct;
15437 src_rmem = &src_ring->ring_mem;
15438
15439 WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages);
15440 WARN_ON(dst_rmem->page_size != src_rmem->page_size);
15441 WARN_ON(dst_rmem->flags != src_rmem->flags);
15442 WARN_ON(dst_rmem->depth != src_rmem->depth);
15443 WARN_ON(dst_rmem->vmem_size != src_rmem->vmem_size);
15444 WARN_ON(dst_rmem->ctx_mem != src_rmem->ctx_mem);
15445 WARN_ON(dst->rx_agg_bmap_size != src->rx_agg_bmap_size);
15446
15447 dst_rmem->pg_tbl = src_rmem->pg_tbl;
15448 dst_rmem->pg_tbl_map = src_rmem->pg_tbl_map;
15449 *dst_rmem->vmem = *src_rmem->vmem;
15450 for (i = 0; i < dst_rmem->nr_pages; i++) {
15451 dst_rmem->pg_arr[i] = src_rmem->pg_arr[i];
15452 dst_rmem->dma_arr[i] = src_rmem->dma_arr[i];
15453 }
15454
15455 dst->rx_agg_bmap = src->rx_agg_bmap;
15456 }
15457
15458 static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx)
15459 {
15460 struct bnxt *bp = netdev_priv(dev);
15461 struct bnxt_rx_ring_info *rxr, *clone;
15462 struct bnxt_cp_ring_info *cpr;
15463 struct bnxt_vnic_info *vnic;
15464 int i, rc;
15465
15466 rxr = &bp->rx_ring[idx];
15467 clone = qmem;
15468
15469 rxr->rx_prod = clone->rx_prod;
15470 rxr->rx_agg_prod = clone->rx_agg_prod;
15471 rxr->rx_sw_agg_prod = clone->rx_sw_agg_prod;
15472 rxr->rx_next_cons = clone->rx_next_cons;
15473 rxr->page_pool = clone->page_pool;
15474 rxr->xdp_rxq = clone->xdp_rxq;
15475
15476 bnxt_copy_rx_ring(bp, rxr, clone);
15477
15478 rc = bnxt_hwrm_rx_ring_alloc(bp, rxr);
15479 if (rc)
15480 return rc;
15481 rc = bnxt_hwrm_rx_agg_ring_alloc(bp, rxr);
15482 if (rc)
15483 goto err_free_hwrm_rx_ring;
15484
15485 bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
15486 if (bp->flags & BNXT_FLAG_AGG_RINGS)
15487 bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
15488
15489 cpr = &rxr->bnapi->cp_ring;
15490 cpr->sw_stats->rx.rx_resets++;
15491
15492 for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) {
15493 vnic = &bp->vnic_info[i];
15494
15495 rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic, true);
15496 if (rc) {
15497 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
15498 vnic->vnic_id, rc);
15499 return rc;
15500 }
15501 vnic->mru = bp->dev->mtu + ETH_HLEN + VLAN_HLEN;
15502 bnxt_hwrm_vnic_update(bp, vnic,
15503 VNIC_UPDATE_REQ_ENABLES_MRU_VALID);
15504 }
15505
15506 return 0;
15507
15508 err_free_hwrm_rx_ring:
15509 bnxt_hwrm_rx_ring_free(bp, rxr, false);
15510 return rc;
15511 }
15512
15513 static int bnxt_queue_stop(struct net_device *dev, void *qmem, int idx)
15514 {
15515 struct bnxt *bp = netdev_priv(dev);
15516 struct bnxt_rx_ring_info *rxr;
15517 struct bnxt_vnic_info *vnic;
15518 int i;
15519
15520 for (i = 0; i <= BNXT_VNIC_NTUPLE; i++) {
15521 vnic = &bp->vnic_info[i];
15522 vnic->mru = 0;
15523 bnxt_hwrm_vnic_update(bp, vnic,
15524 VNIC_UPDATE_REQ_ENABLES_MRU_VALID);
15525 }
15526
15527 rxr = &bp->rx_ring[idx];
15528 bnxt_hwrm_rx_ring_free(bp, rxr, false);
15529 bnxt_hwrm_rx_agg_ring_free(bp, rxr, false);
15530 rxr->rx_next_cons = 0;
15531 page_pool_disable_direct_recycling(rxr->page_pool);
15532
15533 memcpy(qmem, rxr, sizeof(*rxr));
15534 bnxt_init_rx_ring_struct(bp, qmem);
15535
15536 return 0;
15537 }
15538
15539 static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops = {
15540 .ndo_queue_mem_size = sizeof(struct bnxt_rx_ring_info),
15541 .ndo_queue_mem_alloc = bnxt_queue_mem_alloc,
15542 .ndo_queue_mem_free = bnxt_queue_mem_free,
15543 .ndo_queue_start = bnxt_queue_start,
15544 .ndo_queue_stop = bnxt_queue_stop,
15545 };
15546
15547 static void bnxt_remove_one(struct pci_dev *pdev)
15548 {
15549 struct net_device *dev = pci_get_drvdata(pdev);
15550 struct bnxt *bp = netdev_priv(dev);
15551
15552 if (BNXT_PF(bp))
15553 bnxt_sriov_disable(bp);
15554
15555 bnxt_rdma_aux_device_del(bp);
15556
15557 bnxt_ptp_clear(bp);
15558 unregister_netdev(dev);
15559
15560 bnxt_rdma_aux_device_uninit(bp);
15561
15562 bnxt_free_l2_filters(bp, true);
15563 bnxt_free_ntp_fltrs(bp, true);
15564 WARN_ON(bp->num_rss_ctx);
15565 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
15566 /* Flush any pending tasks */
15567 cancel_work_sync(&bp->sp_task);
15568 cancel_delayed_work_sync(&bp->fw_reset_task);
15569 bp->sp_event = 0;
15570
15571 bnxt_dl_fw_reporters_destroy(bp);
15572 bnxt_dl_unregister(bp);
15573 bnxt_shutdown_tc(bp);
15574
15575 bnxt_clear_int_mode(bp);
15576 bnxt_hwrm_func_drv_unrgtr(bp);
15577 bnxt_free_hwrm_resources(bp);
15578 bnxt_hwmon_uninit(bp);
15579 bnxt_ethtool_free(bp);
15580 bnxt_dcb_free(bp);
15581 kfree(bp->ptp_cfg);
15582 bp->ptp_cfg = NULL;
15583 kfree(bp->fw_health);
15584 bp->fw_health = NULL;
15585 bnxt_cleanup_pci(bp);
15586 bnxt_free_ctx_mem(bp, true);
15587 bnxt_free_crash_dump_mem(bp);
15588 kfree(bp->rss_indir_tbl);
15589 bp->rss_indir_tbl = NULL;
15590 bnxt_free_port_stats(bp);
15591 free_netdev(dev);
15592 }
15593
15594 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
15595 {
15596 int rc = 0;
15597 struct bnxt_link_info *link_info = &bp->link_info;
15598
15599 bp->phy_flags = 0;
15600 rc = bnxt_hwrm_phy_qcaps(bp);
15601 if (rc) {
15602 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
15603 rc);
15604 return rc;
15605 }
15606 if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
15607 bp->dev->priv_flags |= IFF_SUPP_NOFCS;
15608 else
15609 bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
15610 if (!fw_dflt)
15611 return 0;
15612
15613 mutex_lock(&bp->link_lock);
15614 rc = bnxt_update_link(bp, false);
15615 if (rc) {
15616 mutex_unlock(&bp->link_lock);
15617 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
15618 rc);
15619 return rc;
15620 }
15621
15622 /* Older firmware does not have supported_auto_speeds, so assume
15623 * that all supported speeds can be autonegotiated.
15624 */
15625 if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
15626 link_info->support_auto_speeds = link_info->support_speeds;
15627
15628 bnxt_init_ethtool_link_settings(bp);
15629 mutex_unlock(&bp->link_lock);
15630 return 0;
15631 }
15632
15633 static int bnxt_get_max_irq(struct pci_dev *pdev)
15634 {
15635 u16 ctrl;
15636
15637 if (!pdev->msix_cap)
15638 return 1;
15639
15640 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
15641 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
15642 }
15643
15644 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
15645 int *max_cp)
15646 {
15647 struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
15648 int max_ring_grps = 0, max_irq;
15649
15650 *max_tx = hw_resc->max_tx_rings;
15651 *max_rx = hw_resc->max_rx_rings;
15652 *max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
15653 max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
15654 bnxt_get_ulp_msix_num_in_use(bp),
15655 hw_resc->max_stat_ctxs -
15656 bnxt_get_ulp_stat_ctxs_in_use(bp));
15657 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS))
15658 *max_cp = min_t(int, *max_cp, max_irq);
15659 max_ring_grps = hw_resc->max_hw_ring_grps;
15660 if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
15661 *max_cp -= 1;
15662 *max_rx -= 2;
15663 }
15664 if (bp->flags & BNXT_FLAG_AGG_RINGS)
15665 *max_rx >>= 1;
15666 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {
15667 int rc;
15668
15669 rc = __bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
15670 if (rc) {
15671 *max_rx = 0;
15672 *max_tx = 0;
15673 }
15674 /* On P5 chips, max_cp output param should be available NQs */
15675 *max_cp = max_irq;
15676 }
15677 *max_rx = min_t(int, *max_rx, max_ring_grps);
15678 }
15679
15680 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
15681 {
15682 int rx, tx, cp;
15683
15684 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
15685 *max_rx = rx;
15686 *max_tx = tx;
15687 if (!rx || !tx || !cp)
15688 return -ENOMEM;
15689
15690 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
15691 }
15692
15693 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
15694 bool shared)
15695 {
15696 int rc;
15697
15698 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
15699 if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
15700 /* Not enough rings, try disabling agg rings. */
15701 bp->flags &= ~BNXT_FLAG_AGG_RINGS;
15702 rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
15703 if (rc) {
15704 /* set BNXT_FLAG_AGG_RINGS back for consistency */
15705 bp->flags |= BNXT_FLAG_AGG_RINGS;
15706 return rc;
15707 }
15708 bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
15709 bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
15710 bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
15711 bnxt_set_ring_params(bp);
15712 }
15713
15714 if (bp->flags & BNXT_FLAG_ROCE_CAP) {
15715 int max_cp, max_stat, max_irq;
15716
15717 /* Reserve minimum resources for RoCE */
15718 max_cp = bnxt_get_max_func_cp_rings(bp);
15719 max_stat = bnxt_get_max_func_stat_ctxs(bp);
15720 max_irq = bnxt_get_max_func_irqs(bp);
15721 if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
15722 max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
15723 max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
15724 return 0;
15725
15726 max_cp -= BNXT_MIN_ROCE_CP_RINGS;
15727 max_irq -= BNXT_MIN_ROCE_CP_RINGS;
15728 max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
15729 max_cp = min_t(int, max_cp, max_irq);
15730 max_cp = min_t(int, max_cp, max_stat);
15731 rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
15732 if (rc)
15733 rc = 0;
15734 }
15735 return rc;
15736 }
15737
15738 /* In initial default shared ring setting, each shared ring must have a
15739 * RX/TX ring pair.
15740 */
15741 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
15742 {
15743 bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
15744 bp->rx_nr_rings = bp->cp_nr_rings;
15745 bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
15746 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
15747 }
15748
15749 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
15750 {
15751 int dflt_rings, max_rx_rings, max_tx_rings, rc;
15752 int avail_msix;
15753
15754 if (!bnxt_can_reserve_rings(bp))
15755 return 0;
15756
15757 if (sh)
15758 bp->flags |= BNXT_FLAG_SHARED_RINGS;
15759 dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
15760 /* Reduce default rings on multi-port cards so that total default
15761 * rings do not exceed CPU count.
15762 */
15763 if (bp->port_count > 1) {
15764 int max_rings =
15765 max_t(int, num_online_cpus() / bp->port_count, 1);
15766
15767 dflt_rings = min_t(int, dflt_rings, max_rings);
15768 }
15769 rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
15770 if (rc)
15771 return rc;
15772 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
15773 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
15774 if (sh)
15775 bnxt_trim_dflt_sh_rings(bp);
15776 else
15777 bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
15778 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
15779
15780 avail_msix = bnxt_get_max_func_irqs(bp) - bp->cp_nr_rings;
15781 if (avail_msix >= BNXT_MIN_ROCE_CP_RINGS) {
15782 int ulp_num_msix = min(avail_msix, bp->ulp_num_msix_want);
15783
15784 bnxt_set_ulp_msix_num(bp, ulp_num_msix);
15785 bnxt_set_dflt_ulp_stat_ctxs(bp);
15786 }
15787
15788 rc = __bnxt_reserve_rings(bp);
15789 if (rc && rc != -ENODEV)
15790 netdev_warn(bp->dev, "Unable to reserve tx rings\n");
15791 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15792 if (sh)
15793 bnxt_trim_dflt_sh_rings(bp);
15794
15795 /* Rings may have been trimmed, re-reserve the trimmed rings. */
15796 if (bnxt_need_reserve_rings(bp)) {
15797 rc = __bnxt_reserve_rings(bp);
15798 if (rc && rc != -ENODEV)
15799 netdev_warn(bp->dev, "2nd rings reservation failed.\n");
15800 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15801 }
15802 if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
15803 bp->rx_nr_rings++;
15804 bp->cp_nr_rings++;
15805 }
15806 if (rc) {
15807 bp->tx_nr_rings = 0;
15808 bp->rx_nr_rings = 0;
15809 }
15810 return rc;
15811 }
15812
15813 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
15814 {
15815 int rc;
15816
15817 if (bp->tx_nr_rings)
15818 return 0;
15819
15820 bnxt_ulp_irq_stop(bp);
15821 bnxt_clear_int_mode(bp);
15822 rc = bnxt_set_dflt_rings(bp, true);
15823 if (rc) {
15824 if (BNXT_VF(bp) && rc == -ENODEV)
15825 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
15826 else
15827 netdev_err(bp->dev, "Not enough rings available.\n");
15828 goto init_dflt_ring_err;
15829 }
15830 rc = bnxt_init_int_mode(bp);
15831 if (rc)
15832 goto init_dflt_ring_err;
15833
15834 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
15835
15836 bnxt_set_dflt_rfs(bp);
15837
15838 init_dflt_ring_err:
15839 bnxt_ulp_irq_restart(bp, rc);
15840 return rc;
15841 }
15842
15843 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
15844 {
15845 int rc;
15846
15847 ASSERT_RTNL();
15848 bnxt_hwrm_func_qcaps(bp);
15849
15850 if (netif_running(bp->dev))
15851 __bnxt_close_nic(bp, true, false);
15852
15853 bnxt_ulp_irq_stop(bp);
15854 bnxt_clear_int_mode(bp);
15855 rc = bnxt_init_int_mode(bp);
15856 bnxt_ulp_irq_restart(bp, rc);
15857
15858 if (netif_running(bp->dev)) {
15859 if (rc)
15860 dev_close(bp->dev);
15861 else
15862 rc = bnxt_open_nic(bp, true, false);
15863 }
15864
15865 return rc;
15866 }
15867
15868 static int bnxt_init_mac_addr(struct bnxt *bp)
15869 {
15870 int rc = 0;
15871
15872 if (BNXT_PF(bp)) {
15873 eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
15874 } else {
15875 #ifdef CONFIG_BNXT_SRIOV
15876 struct bnxt_vf_info *vf = &bp->vf;
15877 bool strict_approval = true;
15878
15879 if (is_valid_ether_addr(vf->mac_addr)) {
15880 /* overwrite netdev dev_addr with admin VF MAC */
15881 eth_hw_addr_set(bp->dev, vf->mac_addr);
15882 /* Older PF driver or firmware may not approve this
15883 * correctly.
15884 */
15885 strict_approval = false;
15886 } else {
15887 eth_hw_addr_random(bp->dev);
15888 }
15889 rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
15890 #endif
15891 }
15892 return rc;
15893 }
15894
15895 static void bnxt_vpd_read_info(struct bnxt *bp)
15896 {
15897 struct pci_dev *pdev = bp->pdev;
15898 unsigned int vpd_size, kw_len;
15899 int pos, size;
15900 u8 *vpd_data;
15901
15902 vpd_data = pci_vpd_alloc(pdev, &vpd_size);
15903 if (IS_ERR(vpd_data)) {
15904 pci_warn(pdev, "Unable to read VPD\n");
15905 return;
15906 }
15907
15908 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
15909 PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
15910 if (pos < 0)
15911 goto read_sn;
15912
15913 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
15914 memcpy(bp->board_partno, &vpd_data[pos], size);
15915
15916 read_sn:
15917 pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
15918 PCI_VPD_RO_KEYWORD_SERIALNO,
15919 &kw_len);
15920 if (pos < 0)
15921 goto exit;
15922
15923 size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
15924 memcpy(bp->board_serialno, &vpd_data[pos], size);
15925 exit:
15926 kfree(vpd_data);
15927 }
15928
15929 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
15930 {
15931 struct pci_dev *pdev = bp->pdev;
15932 u64 qword;
15933
15934 qword = pci_get_dsn(pdev);
15935 if (!qword) {
15936 netdev_info(bp->dev, "Unable to read adapter's DSN\n");
15937 return -EOPNOTSUPP;
15938 }
15939
15940 put_unaligned_le64(qword, dsn);
15941
15942 bp->flags |= BNXT_FLAG_DSN_VALID;
15943 return 0;
15944 }
15945
15946 static int bnxt_map_db_bar(struct bnxt *bp)
15947 {
15948 if (!bp->db_size)
15949 return -ENODEV;
15950 bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
15951 if (!bp->bar1)
15952 return -ENOMEM;
15953 return 0;
15954 }
15955
15956 void bnxt_print_device_info(struct bnxt *bp)
15957 {
15958 netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
15959 board_info[bp->board_idx].name,
15960 (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
15961
15962 pcie_print_link_status(bp->pdev);
15963 }
15964
15965 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
15966 {
15967 struct bnxt_hw_resc *hw_resc;
15968 struct net_device *dev;
15969 struct bnxt *bp;
15970 int rc, max_irqs;
15971
15972 if (pci_is_bridge(pdev))
15973 return -ENODEV;
15974
15975 if (!pdev->msix_cap) {
15976 dev_err(&pdev->dev, "MSIX capability not found, aborting\n");
15977 return -ENODEV;
15978 }
15979
15980 /* Clear any pending DMA transactions from crash kernel
15981 * while loading driver in capture kernel.
15982 */
15983 if (is_kdump_kernel()) {
15984 pci_clear_master(pdev);
15985 pcie_flr(pdev);
15986 }
15987
15988 max_irqs = bnxt_get_max_irq(pdev);
15989 dev = alloc_etherdev_mqs(sizeof(*bp), max_irqs * BNXT_MAX_QUEUE,
15990 max_irqs);
15991 if (!dev)
15992 return -ENOMEM;
15993
15994 bp = netdev_priv(dev);
15995 bp->board_idx = ent->driver_data;
15996 bp->msg_enable = BNXT_DEF_MSG_ENABLE;
15997 bnxt_set_max_func_irqs(bp, max_irqs);
15998
15999 if (bnxt_vf_pciid(bp->board_idx))
16000 bp->flags |= BNXT_FLAG_VF;
16001
16002 /* No devlink port registration in case of a VF */
16003 if (BNXT_PF(bp))
16004 SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
16005
16006 rc = bnxt_init_board(pdev, dev);
16007 if (rc < 0)
16008 goto init_err_free;
16009
16010 dev->netdev_ops = &bnxt_netdev_ops;
16011 dev->stat_ops = &bnxt_stat_ops;
16012 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
16013 dev->ethtool_ops = &bnxt_ethtool_ops;
16014 pci_set_drvdata(pdev, dev);
16015
16016 rc = bnxt_alloc_hwrm_resources(bp);
16017 if (rc)
16018 goto init_err_pci_clean;
16019
16020 mutex_init(&bp->hwrm_cmd_lock);
16021 mutex_init(&bp->link_lock);
16022
16023 rc = bnxt_fw_init_one_p1(bp);
16024 if (rc)
16025 goto init_err_pci_clean;
16026
16027 if (BNXT_PF(bp))
16028 bnxt_vpd_read_info(bp);
16029
16030 if (BNXT_CHIP_P5_PLUS(bp)) {
16031 bp->flags |= BNXT_FLAG_CHIP_P5_PLUS;
16032 if (BNXT_CHIP_P7(bp))
16033 bp->flags |= BNXT_FLAG_CHIP_P7;
16034 }
16035
16036 rc = bnxt_alloc_rss_indir_tbl(bp);
16037 if (rc)
16038 goto init_err_pci_clean;
16039
16040 rc = bnxt_fw_init_one_p2(bp);
16041 if (rc)
16042 goto init_err_pci_clean;
16043
16044 rc = bnxt_map_db_bar(bp);
16045 if (rc) {
16046 dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
16047 rc);
16048 goto init_err_pci_clean;
16049 }
16050
16051 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
16052 NETIF_F_TSO | NETIF_F_TSO6 |
16053 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
16054 NETIF_F_GSO_IPXIP4 |
16055 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
16056 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
16057 NETIF_F_RXCSUM | NETIF_F_GRO;
16058 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
16059 dev->hw_features |= NETIF_F_GSO_UDP_L4;
16060
16061 if (BNXT_SUPPORTS_TPA(bp))
16062 dev->hw_features |= NETIF_F_LRO;
16063
16064 dev->hw_enc_features =
16065 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
16066 NETIF_F_TSO | NETIF_F_TSO6 |
16067 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
16068 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
16069 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
16070 if (bp->flags & BNXT_FLAG_UDP_GSO_CAP)
16071 dev->hw_enc_features |= NETIF_F_GSO_UDP_L4;
16072 if (bp->flags & BNXT_FLAG_CHIP_P7)
16073 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels_p7;
16074 else
16075 dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
16076
16077 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
16078 NETIF_F_GSO_GRE_CSUM;
16079 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
16080 if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
16081 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
16082 if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
16083 dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
16084 if (BNXT_SUPPORTS_TPA(bp))
16085 dev->hw_features |= NETIF_F_GRO_HW;
16086 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
16087 if (dev->features & NETIF_F_GRO_HW)
16088 dev->features &= ~NETIF_F_LRO;
16089 dev->priv_flags |= IFF_UNICAST_FLT;
16090
16091 netif_set_tso_max_size(dev, GSO_MAX_SIZE);
16092 if (bp->tso_max_segs)
16093 netif_set_tso_max_segs(dev, bp->tso_max_segs);
16094
16095 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
16096 NETDEV_XDP_ACT_RX_SG;
16097
16098 #ifdef CONFIG_BNXT_SRIOV
16099 init_waitqueue_head(&bp->sriov_cfg_wait);
16100 #endif
16101 if (BNXT_SUPPORTS_TPA(bp)) {
16102 bp->gro_func = bnxt_gro_func_5730x;
16103 if (BNXT_CHIP_P4(bp))
16104 bp->gro_func = bnxt_gro_func_5731x;
16105 else if (BNXT_CHIP_P5_PLUS(bp))
16106 bp->gro_func = bnxt_gro_func_5750x;
16107 }
16108 if (!BNXT_CHIP_P4_PLUS(bp))
16109 bp->flags |= BNXT_FLAG_DOUBLE_DB;
16110
16111 rc = bnxt_init_mac_addr(bp);
16112 if (rc) {
16113 dev_err(&pdev->dev, "Unable to initialize mac address.\n");
16114 rc = -EADDRNOTAVAIL;
16115 goto init_err_pci_clean;
16116 }
16117
16118 if (BNXT_PF(bp)) {
16119 /* Read the adapter's DSN to use as the eswitch switch_id */
16120 rc = bnxt_pcie_dsn_get(bp, bp->dsn);
16121 }
16122
16123 /* MTU range: 60 - FW defined max */
16124 dev->min_mtu = ETH_ZLEN;
16125 dev->max_mtu = bp->max_mtu;
16126
16127 rc = bnxt_probe_phy(bp, true);
16128 if (rc)
16129 goto init_err_pci_clean;
16130
16131 hw_resc = &bp->hw_resc;
16132 bp->max_fltr = hw_resc->max_rx_em_flows + hw_resc->max_rx_wm_flows +
16133 BNXT_L2_FLTR_MAX_FLTR;
16134 /* Older firmware may not report these filters properly */
16135 if (bp->max_fltr < BNXT_MAX_FLTR)
16136 bp->max_fltr = BNXT_MAX_FLTR;
16137 bnxt_init_l2_fltr_tbl(bp);
16138 bnxt_set_rx_skb_mode(bp, false);
16139 bnxt_set_tpa_flags(bp);
16140 bnxt_set_ring_params(bp);
16141 bnxt_rdma_aux_device_init(bp);
16142 rc = bnxt_set_dflt_rings(bp, true);
16143 if (rc) {
16144 if (BNXT_VF(bp) && rc == -ENODEV) {
16145 netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
16146 } else {
16147 netdev_err(bp->dev, "Not enough rings available.\n");
16148 rc = -ENOMEM;
16149 }
16150 goto init_err_pci_clean;
16151 }
16152
16153 bnxt_fw_init_one_p3(bp);
16154
16155 bnxt_init_dflt_coal(bp);
16156
16157 if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
16158 bp->flags |= BNXT_FLAG_STRIP_VLAN;
16159
16160 rc = bnxt_init_int_mode(bp);
16161 if (rc)
16162 goto init_err_pci_clean;
16163
16164 /* No TC has been set yet and rings may have been trimmed due to
16165 * limited MSIX, so we re-initialize the TX rings per TC.
16166 */
16167 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
16168
16169 if (BNXT_PF(bp)) {
16170 if (!bnxt_pf_wq) {
16171 bnxt_pf_wq =
16172 create_singlethread_workqueue("bnxt_pf_wq");
16173 if (!bnxt_pf_wq) {
16174 dev_err(&pdev->dev, "Unable to create workqueue.\n");
16175 rc = -ENOMEM;
16176 goto init_err_pci_clean;
16177 }
16178 }
16179 rc = bnxt_init_tc(bp);
16180 if (rc)
16181 netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
16182 rc);
16183 }
16184
16185 bnxt_inv_fw_health_reg(bp);
16186 rc = bnxt_dl_register(bp);
16187 if (rc)
16188 goto init_err_dl;
16189
16190 INIT_LIST_HEAD(&bp->usr_fltr_list);
16191
16192 if (BNXT_SUPPORTS_NTUPLE_VNIC(bp))
16193 bp->rss_cap |= BNXT_RSS_CAP_MULTI_RSS_CTX;
16194 if (BNXT_SUPPORTS_QUEUE_API(bp))
16195 dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops;
16196
16197 rc = register_netdev(dev);
16198 if (rc)
16199 goto init_err_cleanup;
16200
16201 bnxt_dl_fw_reporters_create(bp);
16202
16203 bnxt_rdma_aux_device_add(bp);
16204
16205 bnxt_print_device_info(bp);
16206
16207 pci_save_state(pdev);
16208
16209 return 0;
16210 init_err_cleanup:
16211 bnxt_rdma_aux_device_uninit(bp);
16212 bnxt_dl_unregister(bp);
16213 init_err_dl:
16214 bnxt_shutdown_tc(bp);
16215 bnxt_clear_int_mode(bp);
16216
16217 init_err_pci_clean:
16218 bnxt_hwrm_func_drv_unrgtr(bp);
16219 bnxt_free_hwrm_resources(bp);
16220 bnxt_hwmon_uninit(bp);
16221 bnxt_ethtool_free(bp);
16222 bnxt_ptp_clear(bp);
16223 kfree(bp->ptp_cfg);
16224 bp->ptp_cfg = NULL;
16225 kfree(bp->fw_health);
16226 bp->fw_health = NULL;
16227 bnxt_cleanup_pci(bp);
16228 bnxt_free_ctx_mem(bp, true);
16229 bnxt_free_crash_dump_mem(bp);
16230 kfree(bp->rss_indir_tbl);
16231 bp->rss_indir_tbl = NULL;
16232
16233 init_err_free:
16234 free_netdev(dev);
16235 return rc;
16236 }
16237
16238 static void bnxt_shutdown(struct pci_dev *pdev)
16239 {
16240 struct net_device *dev = pci_get_drvdata(pdev);
16241 struct bnxt *bp;
16242
16243 if (!dev)
16244 return;
16245
16246 rtnl_lock();
16247 bp = netdev_priv(dev);
16248 if (!bp)
16249 goto shutdown_exit;
16250
16251 if (netif_running(dev))
16252 dev_close(dev);
16253
16254 bnxt_ptp_clear(bp);
16255 bnxt_clear_int_mode(bp);
16256 pci_disable_device(pdev);
16257
16258 if (system_state == SYSTEM_POWER_OFF) {
16259 pci_wake_from_d3(pdev, bp->wol);
16260 pci_set_power_state(pdev, PCI_D3hot);
16261 }
16262
16263 shutdown_exit:
16264 rtnl_unlock();
16265 }
16266
16267 #ifdef CONFIG_PM_SLEEP
16268 static int bnxt_suspend(struct device *device)
16269 {
16270 struct net_device *dev = dev_get_drvdata(device);
16271 struct bnxt *bp = netdev_priv(dev);
16272 int rc = 0;
16273
16274 bnxt_ulp_stop(bp);
16275
16276 rtnl_lock();
16277 if (netif_running(dev)) {
16278 netif_device_detach(dev);
16279 rc = bnxt_close(dev);
16280 }
16281 bnxt_hwrm_func_drv_unrgtr(bp);
16282 bnxt_ptp_clear(bp);
16283 pci_disable_device(bp->pdev);
16284 bnxt_free_ctx_mem(bp, false);
16285 rtnl_unlock();
16286 return rc;
16287 }
16288
16289 static int bnxt_resume(struct device *device)
16290 {
16291 struct net_device *dev = dev_get_drvdata(device);
16292 struct bnxt *bp = netdev_priv(dev);
16293 int rc = 0;
16294
16295 rtnl_lock();
16296 rc = pci_enable_device(bp->pdev);
16297 if (rc) {
16298 netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
16299 rc);
16300 goto resume_exit;
16301 }
16302 pci_set_master(bp->pdev);
16303 if (bnxt_hwrm_ver_get(bp)) {
16304 rc = -ENODEV;
16305 goto resume_exit;
16306 }
16307 rc = bnxt_hwrm_func_reset(bp);
16308 if (rc) {
16309 rc = -EBUSY;
16310 goto resume_exit;
16311 }
16312
16313 rc = bnxt_hwrm_func_qcaps(bp);
16314 if (rc)
16315 goto resume_exit;
16316
16317 bnxt_clear_reservations(bp, true);
16318
16319 if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
16320 rc = -ENODEV;
16321 goto resume_exit;
16322 }
16323 if (bp->fw_crash_mem)
16324 bnxt_hwrm_crash_dump_mem_cfg(bp);
16325
16326 if (bnxt_ptp_init(bp)) {
16327 kfree(bp->ptp_cfg);
16328 bp->ptp_cfg = NULL;
16329 }
16330 bnxt_get_wol_settings(bp);
16331 if (netif_running(dev)) {
16332 rc = bnxt_open(dev);
16333 if (!rc)
16334 netif_device_attach(dev);
16335 }
16336
16337 resume_exit:
16338 rtnl_unlock();
16339 bnxt_ulp_start(bp, rc);
16340 if (!rc)
16341 bnxt_reenable_sriov(bp);
16342 return rc;
16343 }
16344
16345 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
16346 #define BNXT_PM_OPS (&bnxt_pm_ops)
16347
16348 #else
16349
16350 #define BNXT_PM_OPS NULL
16351
16352 #endif /* CONFIG_PM_SLEEP */
16353
16354 /**
16355 * bnxt_io_error_detected - called when PCI error is detected
16356 * @pdev: Pointer to PCI device
16357 * @state: The current pci connection state
16358 *
16359 * This function is called after a PCI bus error affecting
16360 * this device has been detected.
16361 */
16362 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
16363 pci_channel_state_t state)
16364 {
16365 struct net_device *netdev = pci_get_drvdata(pdev);
16366 struct bnxt *bp = netdev_priv(netdev);
16367 bool abort = false;
16368
16369 netdev_info(netdev, "PCI I/O error detected\n");
16370
16371 bnxt_ulp_stop(bp);
16372
16373 rtnl_lock();
16374 netif_device_detach(netdev);
16375
16376 if (test_and_set_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
16377 netdev_err(bp->dev, "Firmware reset already in progress\n");
16378 abort = true;
16379 }
16380
16381 if (abort || state == pci_channel_io_perm_failure) {
16382 rtnl_unlock();
16383 return PCI_ERS_RESULT_DISCONNECT;
16384 }
16385
16386 /* Link is not reliable anymore if state is pci_channel_io_frozen
16387 * so we disable bus master to prevent any potential bad DMAs before
16388 * freeing kernel memory.
16389 */
16390 if (state == pci_channel_io_frozen) {
16391 set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
16392 bnxt_fw_fatal_close(bp);
16393 }
16394
16395 if (netif_running(netdev))
16396 __bnxt_close_nic(bp, true, true);
16397
16398 if (pci_is_enabled(pdev))
16399 pci_disable_device(pdev);
16400 bnxt_free_ctx_mem(bp, false);
16401 rtnl_unlock();
16402
16403 /* Request a slot slot reset. */
16404 return PCI_ERS_RESULT_NEED_RESET;
16405 }
16406
16407 /**
16408 * bnxt_io_slot_reset - called after the pci bus has been reset.
16409 * @pdev: Pointer to PCI device
16410 *
16411 * Restart the card from scratch, as if from a cold-boot.
16412 * At this point, the card has experienced a hard reset,
16413 * followed by fixups by BIOS, and has its config space
16414 * set up identically to what it was at cold boot.
16415 */
16416 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
16417 {
16418 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
16419 struct net_device *netdev = pci_get_drvdata(pdev);
16420 struct bnxt *bp = netdev_priv(netdev);
16421 int retry = 0;
16422 int err = 0;
16423 int off;
16424
16425 netdev_info(bp->dev, "PCI Slot Reset\n");
16426
16427 if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) &&
16428 test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state))
16429 msleep(900);
16430
16431 rtnl_lock();
16432
16433 if (pci_enable_device(pdev)) {
16434 dev_err(&pdev->dev,
16435 "Cannot re-enable PCI device after reset.\n");
16436 } else {
16437 pci_set_master(pdev);
16438 /* Upon fatal error, our device internal logic that latches to
16439 * BAR value is getting reset and will restore only upon
16440 * rewriting the BARs.
16441 *
16442 * As pci_restore_state() does not re-write the BARs if the
16443 * value is same as saved value earlier, driver needs to
16444 * write the BARs to 0 to force restore, in case of fatal error.
16445 */
16446 if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
16447 &bp->state)) {
16448 for (off = PCI_BASE_ADDRESS_0;
16449 off <= PCI_BASE_ADDRESS_5; off += 4)
16450 pci_write_config_dword(bp->pdev, off, 0);
16451 }
16452 pci_restore_state(pdev);
16453 pci_save_state(pdev);
16454
16455 bnxt_inv_fw_health_reg(bp);
16456 bnxt_try_map_fw_health_reg(bp);
16457
16458 /* In some PCIe AER scenarios, firmware may take up to
16459 * 10 seconds to become ready in the worst case.
16460 */
16461 do {
16462 err = bnxt_try_recover_fw(bp);
16463 if (!err)
16464 break;
16465 retry++;
16466 } while (retry < BNXT_FW_SLOT_RESET_RETRY);
16467
16468 if (err) {
16469 dev_err(&pdev->dev, "Firmware not ready\n");
16470 goto reset_exit;
16471 }
16472
16473 err = bnxt_hwrm_func_reset(bp);
16474 if (!err)
16475 result = PCI_ERS_RESULT_RECOVERED;
16476
16477 bnxt_ulp_irq_stop(bp);
16478 bnxt_clear_int_mode(bp);
16479 err = bnxt_init_int_mode(bp);
16480 bnxt_ulp_irq_restart(bp, err);
16481 }
16482
16483 reset_exit:
16484 clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
16485 bnxt_clear_reservations(bp, true);
16486 rtnl_unlock();
16487
16488 return result;
16489 }
16490
16491 /**
16492 * bnxt_io_resume - called when traffic can start flowing again.
16493 * @pdev: Pointer to PCI device
16494 *
16495 * This callback is called when the error recovery driver tells
16496 * us that its OK to resume normal operation.
16497 */
16498 static void bnxt_io_resume(struct pci_dev *pdev)
16499 {
16500 struct net_device *netdev = pci_get_drvdata(pdev);
16501 struct bnxt *bp = netdev_priv(netdev);
16502 int err;
16503
16504 netdev_info(bp->dev, "PCI Slot Resume\n");
16505 rtnl_lock();
16506
16507 err = bnxt_hwrm_func_qcaps(bp);
16508 if (!err) {
16509 if (netif_running(netdev))
16510 err = bnxt_open(netdev);
16511 else
16512 err = bnxt_reserve_rings(bp, true);
16513 }
16514
16515 if (!err)
16516 netif_device_attach(netdev);
16517
16518 rtnl_unlock();
16519 bnxt_ulp_start(bp, err);
16520 if (!err)
16521 bnxt_reenable_sriov(bp);
16522 }
16523
16524 static const struct pci_error_handlers bnxt_err_handler = {
16525 .error_detected = bnxt_io_error_detected,
16526 .slot_reset = bnxt_io_slot_reset,
16527 .resume = bnxt_io_resume
16528 };
16529
16530 static struct pci_driver bnxt_pci_driver = {
16531 .name = DRV_MODULE_NAME,
16532 .id_table = bnxt_pci_tbl,
16533 .probe = bnxt_init_one,
16534 .remove = bnxt_remove_one,
16535 .shutdown = bnxt_shutdown,
16536 .driver.pm = BNXT_PM_OPS,
16537 .err_handler = &bnxt_err_handler,
16538 #if defined(CONFIG_BNXT_SRIOV)
16539 .sriov_configure = bnxt_sriov_configure,
16540 #endif
16541 };
16542
16543 static int __init bnxt_init(void)
16544 {
16545 int err;
16546
16547 bnxt_debug_init();
16548 err = pci_register_driver(&bnxt_pci_driver);
16549 if (err) {
16550 bnxt_debug_exit();
16551 return err;
16552 }
16553
16554 return 0;
16555 }
16556
16557 static void __exit bnxt_exit(void)
16558 {
16559 pci_unregister_driver(&bnxt_pci_driver);
16560 if (bnxt_pf_wq)
16561 destroy_workqueue(bnxt_pf_wq);
16562 bnxt_debug_exit();
16563 }
16564
16565 module_init(bnxt_init);
16566 module_exit(bnxt_exit);
Kernel DRM miscellaneous fixes and cross-tree changes