?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/mm/pat: Don't report PAT on CPUs that don't support it
[linux/fpc-iii.git] / drivers / net / ethernet / cavium / thunder / nicvf_main.c
blob24017588f5317107142897ca77aa0668046532d5
1 /*
2 * Copyright (C) 2015 Cavium, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of version 2 of the GNU General Public License
6 * as published by the Free Software Foundation.
7 */
8
9 #include <linux/module.h>
10 #include <linux/interrupt.h>
11 #include <linux/pci.h>
12 #include <linux/netdevice.h>
13 #include <linux/if_vlan.h>
14 #include <linux/etherdevice.h>
15 #include <linux/ethtool.h>
16 #include <linux/log2.h>
17 #include <linux/prefetch.h>
18 #include <linux/irq.h>
19 #include <linux/iommu.h>
20
21 #include "nic_reg.h"
22 #include "nic.h"
23 #include "nicvf_queues.h"
24 #include "thunder_bgx.h"
25
26 #define DRV_NAME "thunder-nicvf"
27 #define DRV_VERSION "1.0"
28
29 /* Supported devices */
30 static const struct pci_device_id nicvf_id_table[] = {
31 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
32 PCI_DEVICE_ID_THUNDER_NIC_VF,
33 PCI_VENDOR_ID_CAVIUM,
34 PCI_SUBSYS_DEVID_88XX_NIC_VF) },
35 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
36 PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF,
37 PCI_VENDOR_ID_CAVIUM,
38 PCI_SUBSYS_DEVID_88XX_PASS1_NIC_VF) },
39 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
40 PCI_DEVICE_ID_THUNDER_NIC_VF,
41 PCI_VENDOR_ID_CAVIUM,
42 PCI_SUBSYS_DEVID_81XX_NIC_VF) },
43 { PCI_DEVICE_SUB(PCI_VENDOR_ID_CAVIUM,
44 PCI_DEVICE_ID_THUNDER_NIC_VF,
45 PCI_VENDOR_ID_CAVIUM,
46 PCI_SUBSYS_DEVID_83XX_NIC_VF) },
47 { 0, } /* end of table */
48 };
49
50 MODULE_AUTHOR("Sunil Goutham");
51 MODULE_DESCRIPTION("Cavium Thunder NIC Virtual Function Driver");
52 MODULE_LICENSE("GPL v2");
53 MODULE_VERSION(DRV_VERSION);
54 MODULE_DEVICE_TABLE(pci, nicvf_id_table);
55
56 static int debug = 0x00;
57 module_param(debug, int, 0644);
58 MODULE_PARM_DESC(debug, "Debug message level bitmap");
59
60 static int cpi_alg = CPI_ALG_NONE;
61 module_param(cpi_alg, int, S_IRUGO);
62 MODULE_PARM_DESC(cpi_alg,
63 "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)");
64
65 static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx)
66 {
67 if (nic->sqs_mode)
68 return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS);
69 else
70 return qidx;
71 }
72
73 /* The Cavium ThunderX network controller can *only* be found in SoCs
74 * containing the ThunderX ARM64 CPU implementation. All accesses to the device
75 * registers on this platform are implicitly strongly ordered with respect
76 * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
77 * with no memory barriers in this driver. The readq()/writeq() functions add
78 * explicit ordering operation which in this case are redundant, and only
79 * add overhead.
80 */
81
82 /* Register read/write APIs */
83 void nicvf_reg_write(struct nicvf *nic, u64 offset, u64 val)
84 {
85 writeq_relaxed(val, nic->reg_base + offset);
86 }
87
88 u64 nicvf_reg_read(struct nicvf *nic, u64 offset)
89 {
90 return readq_relaxed(nic->reg_base + offset);
91 }
92
93 void nicvf_queue_reg_write(struct nicvf *nic, u64 offset,
94 u64 qidx, u64 val)
95 {
96 void __iomem *addr = nic->reg_base + offset;
97
98 writeq_relaxed(val, addr + (qidx << NIC_Q_NUM_SHIFT));
99 }
100
101 u64 nicvf_queue_reg_read(struct nicvf *nic, u64 offset, u64 qidx)
102 {
103 void __iomem *addr = nic->reg_base + offset;
104
105 return readq_relaxed(addr + (qidx << NIC_Q_NUM_SHIFT));
106 }
107
108 /* VF -> PF mailbox communication */
109 static void nicvf_write_to_mbx(struct nicvf *nic, union nic_mbx *mbx)
110 {
111 u64 *msg = (u64 *)mbx;
112
113 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 0, msg[0]);
114 nicvf_reg_write(nic, NIC_VF_PF_MAILBOX_0_1 + 8, msg[1]);
115 }
116
117 int nicvf_send_msg_to_pf(struct nicvf *nic, union nic_mbx *mbx)
118 {
119 int timeout = NIC_MBOX_MSG_TIMEOUT;
120 int sleep = 10;
121
122 nic->pf_acked = false;
123 nic->pf_nacked = false;
124
125 nicvf_write_to_mbx(nic, mbx);
126
127 /* Wait for previous message to be acked, timeout 2sec */
128 while (!nic->pf_acked) {
129 if (nic->pf_nacked) {
130 netdev_err(nic->netdev,
131 "PF NACK to mbox msg 0x%02x from VF%d\n",
132 (mbx->msg.msg & 0xFF), nic->vf_id);
133 return -EINVAL;
134 }
135 msleep(sleep);
136 if (nic->pf_acked)
137 break;
138 timeout -= sleep;
139 if (!timeout) {
140 netdev_err(nic->netdev,
141 "PF didn't ACK to mbox msg 0x%02x from VF%d\n",
142 (mbx->msg.msg & 0xFF), nic->vf_id);
143 return -EBUSY;
144 }
145 }
146 return 0;
147 }
148
149 /* Checks if VF is able to comminicate with PF
150 * and also gets the VNIC number this VF is associated to.
151 */
152 static int nicvf_check_pf_ready(struct nicvf *nic)
153 {
154 union nic_mbx mbx = {};
155
156 mbx.msg.msg = NIC_MBOX_MSG_READY;
157 if (nicvf_send_msg_to_pf(nic, &mbx)) {
158 netdev_err(nic->netdev,
159 "PF didn't respond to READY msg\n");
160 return 0;
161 }
162
163 return 1;
164 }
165
166 static void nicvf_read_bgx_stats(struct nicvf *nic, struct bgx_stats_msg *bgx)
167 {
168 if (bgx->rx)
169 nic->bgx_stats.rx_stats[bgx->idx] = bgx->stats;
170 else
171 nic->bgx_stats.tx_stats[bgx->idx] = bgx->stats;
172 }
173
174 static void nicvf_handle_mbx_intr(struct nicvf *nic)
175 {
176 union nic_mbx mbx = {};
177 u64 *mbx_data;
178 u64 mbx_addr;
179 int i;
180
181 mbx_addr = NIC_VF_PF_MAILBOX_0_1;
182 mbx_data = (u64 *)&mbx;
183
184 for (i = 0; i < NIC_PF_VF_MAILBOX_SIZE; i++) {
185 *mbx_data = nicvf_reg_read(nic, mbx_addr);
186 mbx_data++;
187 mbx_addr += sizeof(u64);
188 }
189
190 netdev_dbg(nic->netdev, "Mbox message: msg: 0x%x\n", mbx.msg.msg);
191 switch (mbx.msg.msg) {
192 case NIC_MBOX_MSG_READY:
193 nic->pf_acked = true;
194 nic->vf_id = mbx.nic_cfg.vf_id & 0x7F;
195 nic->tns_mode = mbx.nic_cfg.tns_mode & 0x7F;
196 nic->node = mbx.nic_cfg.node_id;
197 if (!nic->set_mac_pending)
198 ether_addr_copy(nic->netdev->dev_addr,
199 mbx.nic_cfg.mac_addr);
200 nic->sqs_mode = mbx.nic_cfg.sqs_mode;
201 nic->loopback_supported = mbx.nic_cfg.loopback_supported;
202 nic->link_up = false;
203 nic->duplex = 0;
204 nic->speed = 0;
205 break;
206 case NIC_MBOX_MSG_ACK:
207 nic->pf_acked = true;
208 break;
209 case NIC_MBOX_MSG_NACK:
210 nic->pf_nacked = true;
211 break;
212 case NIC_MBOX_MSG_RSS_SIZE:
213 nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
214 nic->pf_acked = true;
215 break;
216 case NIC_MBOX_MSG_BGX_STATS:
217 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
218 nic->pf_acked = true;
219 break;
220 case NIC_MBOX_MSG_BGX_LINK_CHANGE:
221 nic->pf_acked = true;
222 nic->link_up = mbx.link_status.link_up;
223 nic->duplex = mbx.link_status.duplex;
224 nic->speed = mbx.link_status.speed;
225 nic->mac_type = mbx.link_status.mac_type;
226 if (nic->link_up) {
227 netdev_info(nic->netdev, "%s: Link is Up %d Mbps %s\n",
228 nic->netdev->name, nic->speed,
229 nic->duplex == DUPLEX_FULL ?
230 "Full duplex" : "Half duplex");
231 netif_carrier_on(nic->netdev);
232 netif_tx_start_all_queues(nic->netdev);
233 } else {
234 netdev_info(nic->netdev, "%s: Link is Down\n",
235 nic->netdev->name);
236 netif_carrier_off(nic->netdev);
237 netif_tx_stop_all_queues(nic->netdev);
238 }
239 break;
240 case NIC_MBOX_MSG_ALLOC_SQS:
241 nic->sqs_count = mbx.sqs_alloc.qs_count;
242 nic->pf_acked = true;
243 break;
244 case NIC_MBOX_MSG_SNICVF_PTR:
245 /* Primary VF: make note of secondary VF's pointer
246 * to be used while packet transmission.
247 */
248 nic->snicvf[mbx.nicvf.sqs_id] =
249 (struct nicvf *)mbx.nicvf.nicvf;
250 nic->pf_acked = true;
251 break;
252 case NIC_MBOX_MSG_PNICVF_PTR:
253 /* Secondary VF/Qset: make note of primary VF's pointer
254 * to be used while packet reception, to handover packet
255 * to primary VF's netdev.
256 */
257 nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf;
258 nic->pf_acked = true;
259 break;
260 case NIC_MBOX_MSG_PFC:
261 nic->pfc.autoneg = mbx.pfc.autoneg;
262 nic->pfc.fc_rx = mbx.pfc.fc_rx;
263 nic->pfc.fc_tx = mbx.pfc.fc_tx;
264 nic->pf_acked = true;
265 break;
266 default:
267 netdev_err(nic->netdev,
268 "Invalid message from PF, msg 0x%x\n", mbx.msg.msg);
269 break;
270 }
271 nicvf_clear_intr(nic, NICVF_INTR_MBOX, 0);
272 }
273
274 static int nicvf_hw_set_mac_addr(struct nicvf *nic, struct net_device *netdev)
275 {
276 union nic_mbx mbx = {};
277
278 mbx.mac.msg = NIC_MBOX_MSG_SET_MAC;
279 mbx.mac.vf_id = nic->vf_id;
280 ether_addr_copy(mbx.mac.mac_addr, netdev->dev_addr);
281
282 return nicvf_send_msg_to_pf(nic, &mbx);
283 }
284
285 static void nicvf_config_cpi(struct nicvf *nic)
286 {
287 union nic_mbx mbx = {};
288
289 mbx.cpi_cfg.msg = NIC_MBOX_MSG_CPI_CFG;
290 mbx.cpi_cfg.vf_id = nic->vf_id;
291 mbx.cpi_cfg.cpi_alg = nic->cpi_alg;
292 mbx.cpi_cfg.rq_cnt = nic->qs->rq_cnt;
293
294 nicvf_send_msg_to_pf(nic, &mbx);
295 }
296
297 static void nicvf_get_rss_size(struct nicvf *nic)
298 {
299 union nic_mbx mbx = {};
300
301 mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
302 mbx.rss_size.vf_id = nic->vf_id;
303 nicvf_send_msg_to_pf(nic, &mbx);
304 }
305
306 void nicvf_config_rss(struct nicvf *nic)
307 {
308 union nic_mbx mbx = {};
309 struct nicvf_rss_info *rss = &nic->rss_info;
310 int ind_tbl_len = rss->rss_size;
311 int i, nextq = 0;
312
313 mbx.rss_cfg.vf_id = nic->vf_id;
314 mbx.rss_cfg.hash_bits = rss->hash_bits;
315 while (ind_tbl_len) {
316 mbx.rss_cfg.tbl_offset = nextq;
317 mbx.rss_cfg.tbl_len = min(ind_tbl_len,
318 RSS_IND_TBL_LEN_PER_MBX_MSG);
319 mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
320 NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
321
322 for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
323 mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
324
325 nicvf_send_msg_to_pf(nic, &mbx);
326
327 ind_tbl_len -= mbx.rss_cfg.tbl_len;
328 }
329 }
330
331 void nicvf_set_rss_key(struct nicvf *nic)
332 {
333 struct nicvf_rss_info *rss = &nic->rss_info;
334 u64 key_addr = NIC_VNIC_RSS_KEY_0_4;
335 int idx;
336
337 for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
338 nicvf_reg_write(nic, key_addr, rss->key[idx]);
339 key_addr += sizeof(u64);
340 }
341 }
342
343 static int nicvf_rss_init(struct nicvf *nic)
344 {
345 struct nicvf_rss_info *rss = &nic->rss_info;
346 int idx;
347
348 nicvf_get_rss_size(nic);
349
350 if (cpi_alg != CPI_ALG_NONE) {
351 rss->enable = false;
352 rss->hash_bits = 0;
353 return 0;
354 }
355
356 rss->enable = true;
357
358 netdev_rss_key_fill(rss->key, RSS_HASH_KEY_SIZE * sizeof(u64));
359 nicvf_set_rss_key(nic);
360
361 rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
362 nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
363
364 rss->hash_bits = ilog2(rounddown_pow_of_two(rss->rss_size));
365
366 for (idx = 0; idx < rss->rss_size; idx++)
367 rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx,
368 nic->rx_queues);
369 nicvf_config_rss(nic);
370 return 1;
371 }
372
373 /* Request PF to allocate additional Qsets */
374 static void nicvf_request_sqs(struct nicvf *nic)
375 {
376 union nic_mbx mbx = {};
377 int sqs;
378 int sqs_count = nic->sqs_count;
379 int rx_queues = 0, tx_queues = 0;
380
381 /* Only primary VF should request */
382 if (nic->sqs_mode || !nic->sqs_count)
383 return;
384
385 mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
386 mbx.sqs_alloc.vf_id = nic->vf_id;
387 mbx.sqs_alloc.qs_count = nic->sqs_count;
388 if (nicvf_send_msg_to_pf(nic, &mbx)) {
389 /* No response from PF */
390 nic->sqs_count = 0;
391 return;
392 }
393
394 /* Return if no Secondary Qsets available */
395 if (!nic->sqs_count)
396 return;
397
398 if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS)
399 rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS;
400 if (nic->tx_queues > MAX_SND_QUEUES_PER_QS)
401 tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS;
402
403 /* Set no of Rx/Tx queues in each of the SQsets */
404 for (sqs = 0; sqs < nic->sqs_count; sqs++) {
405 mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR;
406 mbx.nicvf.vf_id = nic->vf_id;
407 mbx.nicvf.sqs_id = sqs;
408 nicvf_send_msg_to_pf(nic, &mbx);
409
410 nic->snicvf[sqs]->sqs_id = sqs;
411 if (rx_queues > MAX_RCV_QUEUES_PER_QS) {
412 nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS;
413 rx_queues -= MAX_RCV_QUEUES_PER_QS;
414 } else {
415 nic->snicvf[sqs]->qs->rq_cnt = rx_queues;
416 rx_queues = 0;
417 }
418
419 if (tx_queues > MAX_SND_QUEUES_PER_QS) {
420 nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS;
421 tx_queues -= MAX_SND_QUEUES_PER_QS;
422 } else {
423 nic->snicvf[sqs]->qs->sq_cnt = tx_queues;
424 tx_queues = 0;
425 }
426
427 nic->snicvf[sqs]->qs->cq_cnt =
428 max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt);
429
430 /* Initialize secondary Qset's queues and its interrupts */
431 nicvf_open(nic->snicvf[sqs]->netdev);
432 }
433
434 /* Update stack with actual Rx/Tx queue count allocated */
435 if (sqs_count != nic->sqs_count)
436 nicvf_set_real_num_queues(nic->netdev,
437 nic->tx_queues, nic->rx_queues);
438 }
439
440 /* Send this Qset's nicvf pointer to PF.
441 * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs
442 * so that packets received by these Qsets can use primary VF's netdev
443 */
444 static void nicvf_send_vf_struct(struct nicvf *nic)
445 {
446 union nic_mbx mbx = {};
447
448 mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR;
449 mbx.nicvf.sqs_mode = nic->sqs_mode;
450 mbx.nicvf.nicvf = (u64)nic;
451 nicvf_send_msg_to_pf(nic, &mbx);
452 }
453
454 static void nicvf_get_primary_vf_struct(struct nicvf *nic)
455 {
456 union nic_mbx mbx = {};
457
458 mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR;
459 nicvf_send_msg_to_pf(nic, &mbx);
460 }
461
462 int nicvf_set_real_num_queues(struct net_device *netdev,
463 int tx_queues, int rx_queues)
464 {
465 int err = 0;
466
467 err = netif_set_real_num_tx_queues(netdev, tx_queues);
468 if (err) {
469 netdev_err(netdev,
470 "Failed to set no of Tx queues: %d\n", tx_queues);
471 return err;
472 }
473
474 err = netif_set_real_num_rx_queues(netdev, rx_queues);
475 if (err)
476 netdev_err(netdev,
477 "Failed to set no of Rx queues: %d\n", rx_queues);
478 return err;
479 }
480
481 static int nicvf_init_resources(struct nicvf *nic)
482 {
483 int err;
484
485 /* Enable Qset */
486 nicvf_qset_config(nic, true);
487
488 /* Initialize queues and HW for data transfer */
489 err = nicvf_config_data_transfer(nic, true);
490 if (err) {
491 netdev_err(nic->netdev,
492 "Failed to alloc/config VF's QSet resources\n");
493 return err;
494 }
495
496 return 0;
497 }
498
499 static void nicvf_snd_pkt_handler(struct net_device *netdev,
500 struct cqe_send_t *cqe_tx,
501 int cqe_type, int budget,
502 unsigned int *tx_pkts, unsigned int *tx_bytes)
503 {
504 struct sk_buff *skb = NULL;
505 struct nicvf *nic = netdev_priv(netdev);
506 struct snd_queue *sq;
507 struct sq_hdr_subdesc *hdr;
508 struct sq_hdr_subdesc *tso_sqe;
509
510 sq = &nic->qs->sq[cqe_tx->sq_idx];
511
512 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr);
513 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER)
514 return;
515
516 netdev_dbg(nic->netdev,
517 "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n",
518 __func__, cqe_tx->sq_qs, cqe_tx->sq_idx,
519 cqe_tx->sqe_ptr, hdr->subdesc_cnt);
520
521 nicvf_check_cqe_tx_errs(nic, cqe_tx);
522 skb = (struct sk_buff *)sq->skbuff[cqe_tx->sqe_ptr];
523 if (skb) {
524 /* Check for dummy descriptor used for HW TSO offload on 88xx */
525 if (hdr->dont_send) {
526 /* Get actual TSO descriptors and free them */
527 tso_sqe =
528 (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, hdr->rsvd2);
529 nicvf_unmap_sndq_buffers(nic, sq, hdr->rsvd2,
530 tso_sqe->subdesc_cnt);
531 nicvf_put_sq_desc(sq, tso_sqe->subdesc_cnt + 1);
532 } else {
533 nicvf_unmap_sndq_buffers(nic, sq, cqe_tx->sqe_ptr,
534 hdr->subdesc_cnt);
535 }
536 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
537 prefetch(skb);
538 (*tx_pkts)++;
539 *tx_bytes += skb->len;
540 napi_consume_skb(skb, budget);
541 sq->skbuff[cqe_tx->sqe_ptr] = (u64)NULL;
542 } else {
543 /* In case of SW TSO on 88xx, only last segment will have
544 * a SKB attached, so just free SQEs here.
545 */
546 if (!nic->hw_tso)
547 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1);
548 }
549 }
550
551 static inline void nicvf_set_rxhash(struct net_device *netdev,
552 struct cqe_rx_t *cqe_rx,
553 struct sk_buff *skb)
554 {
555 u8 hash_type;
556 u32 hash;
557
558 if (!(netdev->features & NETIF_F_RXHASH))
559 return;
560
561 switch (cqe_rx->rss_alg) {
562 case RSS_ALG_TCP_IP:
563 case RSS_ALG_UDP_IP:
564 hash_type = PKT_HASH_TYPE_L4;
565 hash = cqe_rx->rss_tag;
566 break;
567 case RSS_ALG_IP:
568 hash_type = PKT_HASH_TYPE_L3;
569 hash = cqe_rx->rss_tag;
570 break;
571 default:
572 hash_type = PKT_HASH_TYPE_NONE;
573 hash = 0;
574 }
575
576 skb_set_hash(skb, hash, hash_type);
577 }
578
579 static void nicvf_rcv_pkt_handler(struct net_device *netdev,
580 struct napi_struct *napi,
581 struct cqe_rx_t *cqe_rx)
582 {
583 struct sk_buff *skb;
584 struct nicvf *nic = netdev_priv(netdev);
585 struct nicvf *snic = nic;
586 int err = 0;
587 int rq_idx;
588
589 rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx);
590
591 if (nic->sqs_mode) {
592 /* Use primary VF's 'nicvf' struct */
593 nic = nic->pnicvf;
594 netdev = nic->netdev;
595 }
596
597 /* Check for errors */
598 err = nicvf_check_cqe_rx_errs(nic, cqe_rx);
599 if (err && !cqe_rx->rb_cnt)
600 return;
601
602 skb = nicvf_get_rcv_skb(snic, cqe_rx);
603 if (!skb) {
604 netdev_dbg(nic->netdev, "Packet not received\n");
605 return;
606 }
607
608 if (netif_msg_pktdata(nic)) {
609 netdev_info(nic->netdev, "%s: skb 0x%p, len=%d\n", netdev->name,
610 skb, skb->len);
611 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_OFFSET, 16, 1,
612 skb->data, skb->len, true);
613 }
614
615 /* If error packet, drop it here */
616 if (err) {
617 dev_kfree_skb_any(skb);
618 return;
619 }
620
621 nicvf_set_rxhash(netdev, cqe_rx, skb);
622
623 skb_record_rx_queue(skb, rq_idx);
624 if (netdev->hw_features & NETIF_F_RXCSUM) {
625 /* HW by default verifies TCP/UDP/SCTP checksums */
626 skb->ip_summed = CHECKSUM_UNNECESSARY;
627 } else {
628 skb_checksum_none_assert(skb);
629 }
630
631 skb->protocol = eth_type_trans(skb, netdev);
632
633 /* Check for stripped VLAN */
634 if (cqe_rx->vlan_found && cqe_rx->vlan_stripped)
635 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
636 ntohs((__force __be16)cqe_rx->vlan_tci));
637
638 if (napi && (netdev->features & NETIF_F_GRO))
639 napi_gro_receive(napi, skb);
640 else
641 netif_receive_skb(skb);
642 }
643
644 static int nicvf_cq_intr_handler(struct net_device *netdev, u8 cq_idx,
645 struct napi_struct *napi, int budget)
646 {
647 int processed_cqe, work_done = 0, tx_done = 0;
648 int cqe_count, cqe_head;
649 struct nicvf *nic = netdev_priv(netdev);
650 struct queue_set *qs = nic->qs;
651 struct cmp_queue *cq = &qs->cq[cq_idx];
652 struct cqe_rx_t *cq_desc;
653 struct netdev_queue *txq;
654 struct snd_queue *sq;
655 unsigned int tx_pkts = 0, tx_bytes = 0;
656
657 spin_lock_bh(&cq->lock);
658 loop:
659 processed_cqe = 0;
660 /* Get no of valid CQ entries to process */
661 cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx);
662 cqe_count &= CQ_CQE_COUNT;
663 if (!cqe_count)
664 goto done;
665
666 /* Get head of the valid CQ entries */
667 cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9;
668 cqe_head &= 0xFFFF;
669
670 netdev_dbg(nic->netdev, "%s CQ%d cqe_count %d cqe_head %d\n",
671 __func__, cq_idx, cqe_count, cqe_head);
672 while (processed_cqe < cqe_count) {
673 /* Get the CQ descriptor */
674 cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head);
675 cqe_head++;
676 cqe_head &= (cq->dmem.q_len - 1);
677 /* Initiate prefetch for next descriptor */
678 prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head));
679
680 if ((work_done >= budget) && napi &&
681 (cq_desc->cqe_type != CQE_TYPE_SEND)) {
682 break;
683 }
684
685 netdev_dbg(nic->netdev, "CQ%d cq_desc->cqe_type %d\n",
686 cq_idx, cq_desc->cqe_type);
687 switch (cq_desc->cqe_type) {
688 case CQE_TYPE_RX:
689 nicvf_rcv_pkt_handler(netdev, napi, cq_desc);
690 work_done++;
691 break;
692 case CQE_TYPE_SEND:
693 nicvf_snd_pkt_handler(netdev,
694 (void *)cq_desc, CQE_TYPE_SEND,
695 budget, &tx_pkts, &tx_bytes);
696 tx_done++;
697 break;
698 case CQE_TYPE_INVALID:
699 case CQE_TYPE_RX_SPLIT:
700 case CQE_TYPE_RX_TCP:
701 case CQE_TYPE_SEND_PTP:
702 /* Ignore for now */
703 break;
704 }
705 processed_cqe++;
706 }
707 netdev_dbg(nic->netdev,
708 "%s CQ%d processed_cqe %d work_done %d budget %d\n",
709 __func__, cq_idx, processed_cqe, work_done, budget);
710
711 /* Ring doorbell to inform H/W to reuse processed CQEs */
712 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR,
713 cq_idx, processed_cqe);
714
715 if ((work_done < budget) && napi)
716 goto loop;
717
718 done:
719 /* Wakeup TXQ if its stopped earlier due to SQ full */
720 sq = &nic->qs->sq[cq_idx];
721 if (tx_done ||
722 (atomic_read(&sq->free_cnt) >= MIN_SQ_DESC_PER_PKT_XMIT)) {
723 netdev = nic->pnicvf->netdev;
724 txq = netdev_get_tx_queue(netdev,
725 nicvf_netdev_qidx(nic, cq_idx));
726 if (tx_pkts)
727 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
728
729 /* To read updated queue and carrier status */
730 smp_mb();
731 if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) {
732 netif_tx_wake_queue(txq);
733 nic = nic->pnicvf;
734 this_cpu_inc(nic->drv_stats->txq_wake);
735 if (netif_msg_tx_err(nic))
736 netdev_warn(netdev,
737 "%s: Transmit queue wakeup SQ%d\n",
738 netdev->name, cq_idx);
739 }
740 }
741
742 spin_unlock_bh(&cq->lock);
743 return work_done;
744 }
745
746 static int nicvf_poll(struct napi_struct *napi, int budget)
747 {
748 u64 cq_head;
749 int work_done = 0;
750 struct net_device *netdev = napi->dev;
751 struct nicvf *nic = netdev_priv(netdev);
752 struct nicvf_cq_poll *cq;
753
754 cq = container_of(napi, struct nicvf_cq_poll, napi);
755 work_done = nicvf_cq_intr_handler(netdev, cq->cq_idx, napi, budget);
756
757 if (work_done < budget) {
758 /* Slow packet rate, exit polling */
759 napi_complete_done(napi, work_done);
760 /* Re-enable interrupts */
761 cq_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD,
762 cq->cq_idx);
763 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
764 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_HEAD,
765 cq->cq_idx, cq_head);
766 nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->cq_idx);
767 }
768 return work_done;
769 }
770
771 /* Qset error interrupt handler
772 *
773 * As of now only CQ errors are handled
774 */
775 static void nicvf_handle_qs_err(unsigned long data)
776 {
777 struct nicvf *nic = (struct nicvf *)data;
778 struct queue_set *qs = nic->qs;
779 int qidx;
780 u64 status;
781
782 netif_tx_disable(nic->netdev);
783
784 /* Check if it is CQ err */
785 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
786 status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS,
787 qidx);
788 if (!(status & CQ_ERR_MASK))
789 continue;
790 /* Process already queued CQEs and reconfig CQ */
791 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
792 nicvf_sq_disable(nic, qidx);
793 nicvf_cq_intr_handler(nic->netdev, qidx, NULL, 0);
794 nicvf_cmp_queue_config(nic, qs, qidx, true);
795 nicvf_sq_free_used_descs(nic->netdev, &qs->sq[qidx], qidx);
796 nicvf_sq_enable(nic, &qs->sq[qidx], qidx);
797
798 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
799 }
800
801 netif_tx_start_all_queues(nic->netdev);
802 /* Re-enable Qset error interrupt */
803 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
804 }
805
806 static void nicvf_dump_intr_status(struct nicvf *nic)
807 {
808 if (netif_msg_intr(nic))
809 netdev_info(nic->netdev, "%s: interrupt status 0x%llx\n",
810 nic->netdev->name, nicvf_reg_read(nic, NIC_VF_INT));
811 }
812
813 static irqreturn_t nicvf_misc_intr_handler(int irq, void *nicvf_irq)
814 {
815 struct nicvf *nic = (struct nicvf *)nicvf_irq;
816 u64 intr;
817
818 nicvf_dump_intr_status(nic);
819
820 intr = nicvf_reg_read(nic, NIC_VF_INT);
821 /* Check for spurious interrupt */
822 if (!(intr & NICVF_INTR_MBOX_MASK))
823 return IRQ_HANDLED;
824
825 nicvf_handle_mbx_intr(nic);
826
827 return IRQ_HANDLED;
828 }
829
830 static irqreturn_t nicvf_intr_handler(int irq, void *cq_irq)
831 {
832 struct nicvf_cq_poll *cq_poll = (struct nicvf_cq_poll *)cq_irq;
833 struct nicvf *nic = cq_poll->nicvf;
834 int qidx = cq_poll->cq_idx;
835
836 nicvf_dump_intr_status(nic);
837
838 /* Disable interrupts */
839 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
840
841 /* Schedule NAPI */
842 napi_schedule_irqoff(&cq_poll->napi);
843
844 /* Clear interrupt */
845 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
846
847 return IRQ_HANDLED;
848 }
849
850 static irqreturn_t nicvf_rbdr_intr_handler(int irq, void *nicvf_irq)
851 {
852 struct nicvf *nic = (struct nicvf *)nicvf_irq;
853 u8 qidx;
854
855
856 nicvf_dump_intr_status(nic);
857
858 /* Disable RBDR interrupt and schedule softirq */
859 for (qidx = 0; qidx < nic->qs->rbdr_cnt; qidx++) {
860 if (!nicvf_is_intr_enabled(nic, NICVF_INTR_RBDR, qidx))
861 continue;
862 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
863 tasklet_hi_schedule(&nic->rbdr_task);
864 /* Clear interrupt */
865 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
866 }
867
868 return IRQ_HANDLED;
869 }
870
871 static irqreturn_t nicvf_qs_err_intr_handler(int irq, void *nicvf_irq)
872 {
873 struct nicvf *nic = (struct nicvf *)nicvf_irq;
874
875 nicvf_dump_intr_status(nic);
876
877 /* Disable Qset err interrupt and schedule softirq */
878 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
879 tasklet_hi_schedule(&nic->qs_err_task);
880 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
881
882 return IRQ_HANDLED;
883 }
884
885 static int nicvf_enable_msix(struct nicvf *nic)
886 {
887 int ret, vec;
888
889 nic->num_vec = NIC_VF_MSIX_VECTORS;
890
891 for (vec = 0; vec < nic->num_vec; vec++)
892 nic->msix_entries[vec].entry = vec;
893
894 ret = pci_enable_msix(nic->pdev, nic->msix_entries, nic->num_vec);
895 if (ret) {
896 netdev_err(nic->netdev,
897 "Req for #%d msix vectors failed\n", nic->num_vec);
898 return 0;
899 }
900 nic->msix_enabled = 1;
901 return 1;
902 }
903
904 static void nicvf_disable_msix(struct nicvf *nic)
905 {
906 if (nic->msix_enabled) {
907 pci_disable_msix(nic->pdev);
908 nic->msix_enabled = 0;
909 nic->num_vec = 0;
910 }
911 }
912
913 static void nicvf_set_irq_affinity(struct nicvf *nic)
914 {
915 int vec, cpu;
916 int irqnum;
917
918 for (vec = 0; vec < nic->num_vec; vec++) {
919 if (!nic->irq_allocated[vec])
920 continue;
921
922 if (!zalloc_cpumask_var(&nic->affinity_mask[vec], GFP_KERNEL))
923 return;
924 /* CQ interrupts */
925 if (vec < NICVF_INTR_ID_SQ)
926 /* Leave CPU0 for RBDR and other interrupts */
927 cpu = nicvf_netdev_qidx(nic, vec) + 1;
928 else
929 cpu = 0;
930
931 cpumask_set_cpu(cpumask_local_spread(cpu, nic->node),
932 nic->affinity_mask[vec]);
933 irqnum = nic->msix_entries[vec].vector;
934 irq_set_affinity_hint(irqnum, nic->affinity_mask[vec]);
935 }
936 }
937
938 static int nicvf_register_interrupts(struct nicvf *nic)
939 {
940 int irq, ret = 0;
941 int vector;
942
943 for_each_cq_irq(irq)
944 sprintf(nic->irq_name[irq], "%s-rxtx-%d",
945 nic->pnicvf->netdev->name,
946 nicvf_netdev_qidx(nic, irq));
947
948 for_each_sq_irq(irq)
949 sprintf(nic->irq_name[irq], "%s-sq-%d",
950 nic->pnicvf->netdev->name,
951 nicvf_netdev_qidx(nic, irq - NICVF_INTR_ID_SQ));
952
953 for_each_rbdr_irq(irq)
954 sprintf(nic->irq_name[irq], "%s-rbdr-%d",
955 nic->pnicvf->netdev->name,
956 nic->sqs_mode ? (nic->sqs_id + 1) : 0);
957
958 /* Register CQ interrupts */
959 for (irq = 0; irq < nic->qs->cq_cnt; irq++) {
960 vector = nic->msix_entries[irq].vector;
961 ret = request_irq(vector, nicvf_intr_handler,
962 0, nic->irq_name[irq], nic->napi[irq]);
963 if (ret)
964 goto err;
965 nic->irq_allocated[irq] = true;
966 }
967
968 /* Register RBDR interrupt */
969 for (irq = NICVF_INTR_ID_RBDR;
970 irq < (NICVF_INTR_ID_RBDR + nic->qs->rbdr_cnt); irq++) {
971 vector = nic->msix_entries[irq].vector;
972 ret = request_irq(vector, nicvf_rbdr_intr_handler,
973 0, nic->irq_name[irq], nic);
974 if (ret)
975 goto err;
976 nic->irq_allocated[irq] = true;
977 }
978
979 /* Register QS error interrupt */
980 sprintf(nic->irq_name[NICVF_INTR_ID_QS_ERR], "%s-qset-err-%d",
981 nic->pnicvf->netdev->name,
982 nic->sqs_mode ? (nic->sqs_id + 1) : 0);
983 irq = NICVF_INTR_ID_QS_ERR;
984 ret = request_irq(nic->msix_entries[irq].vector,
985 nicvf_qs_err_intr_handler,
986 0, nic->irq_name[irq], nic);
987 if (ret)
988 goto err;
989
990 nic->irq_allocated[irq] = true;
991
992 /* Set IRQ affinities */
993 nicvf_set_irq_affinity(nic);
994
995 err:
996 if (ret)
997 netdev_err(nic->netdev, "request_irq failed, vector %d\n", irq);
998
999 return ret;
1000 }
1001
1002 static void nicvf_unregister_interrupts(struct nicvf *nic)
1003 {
1004 int irq;
1005
1006 /* Free registered interrupts */
1007 for (irq = 0; irq < nic->num_vec; irq++) {
1008 if (!nic->irq_allocated[irq])
1009 continue;
1010
1011 irq_set_affinity_hint(nic->msix_entries[irq].vector, NULL);
1012 free_cpumask_var(nic->affinity_mask[irq]);
1013
1014 if (irq < NICVF_INTR_ID_SQ)
1015 free_irq(nic->msix_entries[irq].vector, nic->napi[irq]);
1016 else
1017 free_irq(nic->msix_entries[irq].vector, nic);
1018
1019 nic->irq_allocated[irq] = false;
1020 }
1021
1022 /* Disable MSI-X */
1023 nicvf_disable_msix(nic);
1024 }
1025
1026 /* Initialize MSIX vectors and register MISC interrupt.
1027 * Send READY message to PF to check if its alive
1028 */
1029 static int nicvf_register_misc_interrupt(struct nicvf *nic)
1030 {
1031 int ret = 0;
1032 int irq = NICVF_INTR_ID_MISC;
1033
1034 /* Return if mailbox interrupt is already registered */
1035 if (nic->msix_enabled)
1036 return 0;
1037
1038 /* Enable MSI-X */
1039 if (!nicvf_enable_msix(nic))
1040 return 1;
1041
1042 sprintf(nic->irq_name[irq], "%s Mbox", "NICVF");
1043 /* Register Misc interrupt */
1044 ret = request_irq(nic->msix_entries[irq].vector,
1045 nicvf_misc_intr_handler, 0, nic->irq_name[irq], nic);
1046
1047 if (ret)
1048 return ret;
1049 nic->irq_allocated[irq] = true;
1050
1051 /* Enable mailbox interrupt */
1052 nicvf_enable_intr(nic, NICVF_INTR_MBOX, 0);
1053
1054 /* Check if VF is able to communicate with PF */
1055 if (!nicvf_check_pf_ready(nic)) {
1056 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1057 nicvf_unregister_interrupts(nic);
1058 return 1;
1059 }
1060
1061 return 0;
1062 }
1063
1064 static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev)
1065 {
1066 struct nicvf *nic = netdev_priv(netdev);
1067 int qid = skb_get_queue_mapping(skb);
1068 struct netdev_queue *txq = netdev_get_tx_queue(netdev, qid);
1069 struct nicvf *snic;
1070 struct snd_queue *sq;
1071 int tmp;
1072
1073 /* Check for minimum packet length */
1074 if (skb->len <= ETH_HLEN) {
1075 dev_kfree_skb(skb);
1076 return NETDEV_TX_OK;
1077 }
1078
1079 snic = nic;
1080 /* Get secondary Qset's SQ structure */
1081 if (qid >= MAX_SND_QUEUES_PER_QS) {
1082 tmp = qid / MAX_SND_QUEUES_PER_QS;
1083 snic = (struct nicvf *)nic->snicvf[tmp - 1];
1084 if (!snic) {
1085 netdev_warn(nic->netdev,
1086 "Secondary Qset#%d's ptr not initialized\n",
1087 tmp - 1);
1088 dev_kfree_skb(skb);
1089 return NETDEV_TX_OK;
1090 }
1091 qid = qid % MAX_SND_QUEUES_PER_QS;
1092 }
1093
1094 sq = &snic->qs->sq[qid];
1095 if (!netif_tx_queue_stopped(txq) &&
1096 !nicvf_sq_append_skb(snic, sq, skb, qid)) {
1097 netif_tx_stop_queue(txq);
1098
1099 /* Barrier, so that stop_queue visible to other cpus */
1100 smp_mb();
1101
1102 /* Check again, incase another cpu freed descriptors */
1103 if (atomic_read(&sq->free_cnt) > MIN_SQ_DESC_PER_PKT_XMIT) {
1104 netif_tx_wake_queue(txq);
1105 } else {
1106 this_cpu_inc(nic->drv_stats->txq_stop);
1107 if (netif_msg_tx_err(nic))
1108 netdev_warn(netdev,
1109 "%s: Transmit ring full, stopping SQ%d\n",
1110 netdev->name, qid);
1111 }
1112 return NETDEV_TX_BUSY;
1113 }
1114
1115 return NETDEV_TX_OK;
1116 }
1117
1118 static inline void nicvf_free_cq_poll(struct nicvf *nic)
1119 {
1120 struct nicvf_cq_poll *cq_poll;
1121 int qidx;
1122
1123 for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1124 cq_poll = nic->napi[qidx];
1125 if (!cq_poll)
1126 continue;
1127 nic->napi[qidx] = NULL;
1128 kfree(cq_poll);
1129 }
1130 }
1131
1132 int nicvf_stop(struct net_device *netdev)
1133 {
1134 int irq, qidx;
1135 struct nicvf *nic = netdev_priv(netdev);
1136 struct queue_set *qs = nic->qs;
1137 struct nicvf_cq_poll *cq_poll = NULL;
1138 union nic_mbx mbx = {};
1139
1140 mbx.msg.msg = NIC_MBOX_MSG_SHUTDOWN;
1141 nicvf_send_msg_to_pf(nic, &mbx);
1142
1143 netif_carrier_off(netdev);
1144 netif_tx_stop_all_queues(nic->netdev);
1145 nic->link_up = false;
1146
1147 /* Teardown secondary qsets first */
1148 if (!nic->sqs_mode) {
1149 for (qidx = 0; qidx < nic->sqs_count; qidx++) {
1150 if (!nic->snicvf[qidx])
1151 continue;
1152 nicvf_stop(nic->snicvf[qidx]->netdev);
1153 nic->snicvf[qidx] = NULL;
1154 }
1155 }
1156
1157 /* Disable RBDR & QS error interrupts */
1158 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) {
1159 nicvf_disable_intr(nic, NICVF_INTR_RBDR, qidx);
1160 nicvf_clear_intr(nic, NICVF_INTR_RBDR, qidx);
1161 }
1162 nicvf_disable_intr(nic, NICVF_INTR_QS_ERR, 0);
1163 nicvf_clear_intr(nic, NICVF_INTR_QS_ERR, 0);
1164
1165 /* Wait for pending IRQ handlers to finish */
1166 for (irq = 0; irq < nic->num_vec; irq++)
1167 synchronize_irq(nic->msix_entries[irq].vector);
1168
1169 tasklet_kill(&nic->rbdr_task);
1170 tasklet_kill(&nic->qs_err_task);
1171 if (nic->rb_work_scheduled)
1172 cancel_delayed_work_sync(&nic->rbdr_work);
1173
1174 for (qidx = 0; qidx < nic->qs->cq_cnt; qidx++) {
1175 cq_poll = nic->napi[qidx];
1176 if (!cq_poll)
1177 continue;
1178 napi_synchronize(&cq_poll->napi);
1179 /* CQ intr is enabled while napi_complete,
1180 * so disable it now
1181 */
1182 nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx);
1183 nicvf_clear_intr(nic, NICVF_INTR_CQ, qidx);
1184 napi_disable(&cq_poll->napi);
1185 netif_napi_del(&cq_poll->napi);
1186 }
1187
1188 netif_tx_disable(netdev);
1189
1190 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
1191 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
1192
1193 /* Free resources */
1194 nicvf_config_data_transfer(nic, false);
1195
1196 /* Disable HW Qset */
1197 nicvf_qset_config(nic, false);
1198
1199 /* disable mailbox interrupt */
1200 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1201
1202 nicvf_unregister_interrupts(nic);
1203
1204 nicvf_free_cq_poll(nic);
1205
1206 /* Clear multiqset info */
1207 nic->pnicvf = nic;
1208
1209 return 0;
1210 }
1211
1212 static int nicvf_update_hw_max_frs(struct nicvf *nic, int mtu)
1213 {
1214 union nic_mbx mbx = {};
1215
1216 mbx.frs.msg = NIC_MBOX_MSG_SET_MAX_FRS;
1217 mbx.frs.max_frs = mtu;
1218 mbx.frs.vf_id = nic->vf_id;
1219
1220 return nicvf_send_msg_to_pf(nic, &mbx);
1221 }
1222
1223 int nicvf_open(struct net_device *netdev)
1224 {
1225 int cpu, err, qidx;
1226 struct nicvf *nic = netdev_priv(netdev);
1227 struct queue_set *qs = nic->qs;
1228 struct nicvf_cq_poll *cq_poll = NULL;
1229 union nic_mbx mbx = {};
1230
1231 netif_carrier_off(netdev);
1232
1233 err = nicvf_register_misc_interrupt(nic);
1234 if (err)
1235 return err;
1236
1237 /* Register NAPI handler for processing CQEs */
1238 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1239 cq_poll = kzalloc(sizeof(*cq_poll), GFP_KERNEL);
1240 if (!cq_poll) {
1241 err = -ENOMEM;
1242 goto napi_del;
1243 }
1244 cq_poll->cq_idx = qidx;
1245 cq_poll->nicvf = nic;
1246 netif_napi_add(netdev, &cq_poll->napi, nicvf_poll,
1247 NAPI_POLL_WEIGHT);
1248 napi_enable(&cq_poll->napi);
1249 nic->napi[qidx] = cq_poll;
1250 }
1251
1252 /* Check if we got MAC address from PF or else generate a radom MAC */
1253 if (!nic->sqs_mode && is_zero_ether_addr(netdev->dev_addr)) {
1254 eth_hw_addr_random(netdev);
1255 nicvf_hw_set_mac_addr(nic, netdev);
1256 }
1257
1258 if (nic->set_mac_pending) {
1259 nic->set_mac_pending = false;
1260 nicvf_hw_set_mac_addr(nic, netdev);
1261 }
1262
1263 /* Init tasklet for handling Qset err interrupt */
1264 tasklet_init(&nic->qs_err_task, nicvf_handle_qs_err,
1265 (unsigned long)nic);
1266
1267 /* Init RBDR tasklet which will refill RBDR */
1268 tasklet_init(&nic->rbdr_task, nicvf_rbdr_task,
1269 (unsigned long)nic);
1270 INIT_DELAYED_WORK(&nic->rbdr_work, nicvf_rbdr_work);
1271
1272 /* Configure CPI alorithm */
1273 nic->cpi_alg = cpi_alg;
1274 if (!nic->sqs_mode)
1275 nicvf_config_cpi(nic);
1276
1277 nicvf_request_sqs(nic);
1278 if (nic->sqs_mode)
1279 nicvf_get_primary_vf_struct(nic);
1280
1281 /* Configure receive side scaling and MTU */
1282 if (!nic->sqs_mode) {
1283 nicvf_rss_init(nic);
1284 err = nicvf_update_hw_max_frs(nic, netdev->mtu);
1285 if (err)
1286 goto cleanup;
1287
1288 /* Clear percpu stats */
1289 for_each_possible_cpu(cpu)
1290 memset(per_cpu_ptr(nic->drv_stats, cpu), 0,
1291 sizeof(struct nicvf_drv_stats));
1292 }
1293
1294 err = nicvf_register_interrupts(nic);
1295 if (err)
1296 goto cleanup;
1297
1298 /* Initialize the queues */
1299 err = nicvf_init_resources(nic);
1300 if (err)
1301 goto cleanup;
1302
1303 /* Make sure queue initialization is written */
1304 wmb();
1305
1306 nicvf_reg_write(nic, NIC_VF_INT, -1);
1307 /* Enable Qset err interrupt */
1308 nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0);
1309
1310 /* Enable completion queue interrupt */
1311 for (qidx = 0; qidx < qs->cq_cnt; qidx++)
1312 nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx);
1313
1314 /* Enable RBDR threshold interrupt */
1315 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++)
1316 nicvf_enable_intr(nic, NICVF_INTR_RBDR, qidx);
1317
1318 /* Send VF config done msg to PF */
1319 mbx.msg.msg = NIC_MBOX_MSG_CFG_DONE;
1320 nicvf_write_to_mbx(nic, &mbx);
1321
1322 return 0;
1323 cleanup:
1324 nicvf_disable_intr(nic, NICVF_INTR_MBOX, 0);
1325 nicvf_unregister_interrupts(nic);
1326 tasklet_kill(&nic->qs_err_task);
1327 tasklet_kill(&nic->rbdr_task);
1328 napi_del:
1329 for (qidx = 0; qidx < qs->cq_cnt; qidx++) {
1330 cq_poll = nic->napi[qidx];
1331 if (!cq_poll)
1332 continue;
1333 napi_disable(&cq_poll->napi);
1334 netif_napi_del(&cq_poll->napi);
1335 }
1336 nicvf_free_cq_poll(nic);
1337 return err;
1338 }
1339
1340 static int nicvf_change_mtu(struct net_device *netdev, int new_mtu)
1341 {
1342 struct nicvf *nic = netdev_priv(netdev);
1343 int orig_mtu = netdev->mtu;
1344
1345 netdev->mtu = new_mtu;
1346
1347 if (!netif_running(netdev))
1348 return 0;
1349
1350 if (nicvf_update_hw_max_frs(nic, new_mtu)) {
1351 netdev->mtu = orig_mtu;
1352 return -EINVAL;
1353 }
1354
1355 return 0;
1356 }
1357
1358 static int nicvf_set_mac_address(struct net_device *netdev, void *p)
1359 {
1360 struct sockaddr *addr = p;
1361 struct nicvf *nic = netdev_priv(netdev);
1362
1363 if (!is_valid_ether_addr(addr->sa_data))
1364 return -EADDRNOTAVAIL;
1365
1366 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
1367
1368 if (nic->msix_enabled) {
1369 if (nicvf_hw_set_mac_addr(nic, netdev))
1370 return -EBUSY;
1371 } else {
1372 nic->set_mac_pending = true;
1373 }
1374
1375 return 0;
1376 }
1377
1378 void nicvf_update_lmac_stats(struct nicvf *nic)
1379 {
1380 int stat = 0;
1381 union nic_mbx mbx = {};
1382
1383 if (!netif_running(nic->netdev))
1384 return;
1385
1386 mbx.bgx_stats.msg = NIC_MBOX_MSG_BGX_STATS;
1387 mbx.bgx_stats.vf_id = nic->vf_id;
1388 /* Rx stats */
1389 mbx.bgx_stats.rx = 1;
1390 while (stat < BGX_RX_STATS_COUNT) {
1391 mbx.bgx_stats.idx = stat;
1392 if (nicvf_send_msg_to_pf(nic, &mbx))
1393 return;
1394 stat++;
1395 }
1396
1397 stat = 0;
1398
1399 /* Tx stats */
1400 mbx.bgx_stats.rx = 0;
1401 while (stat < BGX_TX_STATS_COUNT) {
1402 mbx.bgx_stats.idx = stat;
1403 if (nicvf_send_msg_to_pf(nic, &mbx))
1404 return;
1405 stat++;
1406 }
1407 }
1408
1409 void nicvf_update_stats(struct nicvf *nic)
1410 {
1411 int qidx, cpu;
1412 u64 tmp_stats = 0;
1413 struct nicvf_hw_stats *stats = &nic->hw_stats;
1414 struct nicvf_drv_stats *drv_stats;
1415 struct queue_set *qs = nic->qs;
1416
1417 #define GET_RX_STATS(reg) \
1418 nicvf_reg_read(nic, NIC_VNIC_RX_STAT_0_13 | (reg << 3))
1419 #define GET_TX_STATS(reg) \
1420 nicvf_reg_read(nic, NIC_VNIC_TX_STAT_0_4 | (reg << 3))
1421
1422 stats->rx_bytes = GET_RX_STATS(RX_OCTS);
1423 stats->rx_ucast_frames = GET_RX_STATS(RX_UCAST);
1424 stats->rx_bcast_frames = GET_RX_STATS(RX_BCAST);
1425 stats->rx_mcast_frames = GET_RX_STATS(RX_MCAST);
1426 stats->rx_fcs_errors = GET_RX_STATS(RX_FCS);
1427 stats->rx_l2_errors = GET_RX_STATS(RX_L2ERR);
1428 stats->rx_drop_red = GET_RX_STATS(RX_RED);
1429 stats->rx_drop_red_bytes = GET_RX_STATS(RX_RED_OCTS);
1430 stats->rx_drop_overrun = GET_RX_STATS(RX_ORUN);
1431 stats->rx_drop_overrun_bytes = GET_RX_STATS(RX_ORUN_OCTS);
1432 stats->rx_drop_bcast = GET_RX_STATS(RX_DRP_BCAST);
1433 stats->rx_drop_mcast = GET_RX_STATS(RX_DRP_MCAST);
1434 stats->rx_drop_l3_bcast = GET_RX_STATS(RX_DRP_L3BCAST);
1435 stats->rx_drop_l3_mcast = GET_RX_STATS(RX_DRP_L3MCAST);
1436
1437 stats->tx_bytes = GET_TX_STATS(TX_OCTS);
1438 stats->tx_ucast_frames = GET_TX_STATS(TX_UCAST);
1439 stats->tx_bcast_frames = GET_TX_STATS(TX_BCAST);
1440 stats->tx_mcast_frames = GET_TX_STATS(TX_MCAST);
1441 stats->tx_drops = GET_TX_STATS(TX_DROP);
1442
1443 /* On T88 pass 2.0, the dummy SQE added for TSO notification
1444 * via CQE has 'dont_send' set. Hence HW drops the pkt pointed
1445 * pointed by dummy SQE and results in tx_drops counter being
1446 * incremented. Subtracting it from tx_tso counter will give
1447 * exact tx_drops counter.
1448 */
1449 if (nic->t88 && nic->hw_tso) {
1450 for_each_possible_cpu(cpu) {
1451 drv_stats = per_cpu_ptr(nic->drv_stats, cpu);
1452 tmp_stats += drv_stats->tx_tso;
1453 }
1454 stats->tx_drops = tmp_stats - stats->tx_drops;
1455 }
1456 stats->tx_frames = stats->tx_ucast_frames +
1457 stats->tx_bcast_frames +
1458 stats->tx_mcast_frames;
1459 stats->rx_frames = stats->rx_ucast_frames +
1460 stats->rx_bcast_frames +
1461 stats->rx_mcast_frames;
1462 stats->rx_drops = stats->rx_drop_red +
1463 stats->rx_drop_overrun;
1464
1465 /* Update RQ and SQ stats */
1466 for (qidx = 0; qidx < qs->rq_cnt; qidx++)
1467 nicvf_update_rq_stats(nic, qidx);
1468 for (qidx = 0; qidx < qs->sq_cnt; qidx++)
1469 nicvf_update_sq_stats(nic, qidx);
1470 }
1471
1472 static void nicvf_get_stats64(struct net_device *netdev,
1473 struct rtnl_link_stats64 *stats)
1474 {
1475 struct nicvf *nic = netdev_priv(netdev);
1476 struct nicvf_hw_stats *hw_stats = &nic->hw_stats;
1477
1478 nicvf_update_stats(nic);
1479
1480 stats->rx_bytes = hw_stats->rx_bytes;
1481 stats->rx_packets = hw_stats->rx_frames;
1482 stats->rx_dropped = hw_stats->rx_drops;
1483 stats->multicast = hw_stats->rx_mcast_frames;
1484
1485 stats->tx_bytes = hw_stats->tx_bytes;
1486 stats->tx_packets = hw_stats->tx_frames;
1487 stats->tx_dropped = hw_stats->tx_drops;
1488
1489 }
1490
1491 static void nicvf_tx_timeout(struct net_device *dev)
1492 {
1493 struct nicvf *nic = netdev_priv(dev);
1494
1495 if (netif_msg_tx_err(nic))
1496 netdev_warn(dev, "%s: Transmit timed out, resetting\n",
1497 dev->name);
1498
1499 this_cpu_inc(nic->drv_stats->tx_timeout);
1500 schedule_work(&nic->reset_task);
1501 }
1502
1503 static void nicvf_reset_task(struct work_struct *work)
1504 {
1505 struct nicvf *nic;
1506
1507 nic = container_of(work, struct nicvf, reset_task);
1508
1509 if (!netif_running(nic->netdev))
1510 return;
1511
1512 nicvf_stop(nic->netdev);
1513 nicvf_open(nic->netdev);
1514 netif_trans_update(nic->netdev);
1515 }
1516
1517 static int nicvf_config_loopback(struct nicvf *nic,
1518 netdev_features_t features)
1519 {
1520 union nic_mbx mbx = {};
1521
1522 mbx.lbk.msg = NIC_MBOX_MSG_LOOPBACK;
1523 mbx.lbk.vf_id = nic->vf_id;
1524 mbx.lbk.enable = (features & NETIF_F_LOOPBACK) != 0;
1525
1526 return nicvf_send_msg_to_pf(nic, &mbx);
1527 }
1528
1529 static netdev_features_t nicvf_fix_features(struct net_device *netdev,
1530 netdev_features_t features)
1531 {
1532 struct nicvf *nic = netdev_priv(netdev);
1533
1534 if ((features & NETIF_F_LOOPBACK) &&
1535 netif_running(netdev) && !nic->loopback_supported)
1536 features &= ~NETIF_F_LOOPBACK;
1537
1538 return features;
1539 }
1540
1541 static int nicvf_set_features(struct net_device *netdev,
1542 netdev_features_t features)
1543 {
1544 struct nicvf *nic = netdev_priv(netdev);
1545 netdev_features_t changed = features ^ netdev->features;
1546
1547 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1548 nicvf_config_vlan_stripping(nic, features);
1549
1550 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
1551 return nicvf_config_loopback(nic, features);
1552
1553 return 0;
1554 }
1555
1556 static const struct net_device_ops nicvf_netdev_ops = {
1557 .ndo_open = nicvf_open,
1558 .ndo_stop = nicvf_stop,
1559 .ndo_start_xmit = nicvf_xmit,
1560 .ndo_change_mtu = nicvf_change_mtu,
1561 .ndo_set_mac_address = nicvf_set_mac_address,
1562 .ndo_get_stats64 = nicvf_get_stats64,
1563 .ndo_tx_timeout = nicvf_tx_timeout,
1564 .ndo_fix_features = nicvf_fix_features,
1565 .ndo_set_features = nicvf_set_features,
1566 };
1567
1568 static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1569 {
1570 struct device *dev = &pdev->dev;
1571 struct net_device *netdev;
1572 struct nicvf *nic;
1573 int err, qcount;
1574 u16 sdevid;
1575
1576 err = pci_enable_device(pdev);
1577 if (err) {
1578 dev_err(dev, "Failed to enable PCI device\n");
1579 return err;
1580 }
1581
1582 err = pci_request_regions(pdev, DRV_NAME);
1583 if (err) {
1584 dev_err(dev, "PCI request regions failed 0x%x\n", err);
1585 goto err_disable_device;
1586 }
1587
1588 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48));
1589 if (err) {
1590 dev_err(dev, "Unable to get usable DMA configuration\n");
1591 goto err_release_regions;
1592 }
1593
1594 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48));
1595 if (err) {
1596 dev_err(dev, "unable to get 48-bit DMA for consistent allocations\n");
1597 goto err_release_regions;
1598 }
1599
1600 qcount = netif_get_num_default_rss_queues();
1601
1602 /* Restrict multiqset support only for host bound VFs */
1603 if (pdev->is_virtfn) {
1604 /* Set max number of queues per VF */
1605 qcount = min_t(int, num_online_cpus(),
1606 (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS);
1607 }
1608
1609 netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount);
1610 if (!netdev) {
1611 err = -ENOMEM;
1612 goto err_release_regions;
1613 }
1614
1615 pci_set_drvdata(pdev, netdev);
1616
1617 SET_NETDEV_DEV(netdev, &pdev->dev);
1618
1619 nic = netdev_priv(netdev);
1620 nic->netdev = netdev;
1621 nic->pdev = pdev;
1622 nic->pnicvf = nic;
1623 nic->max_queues = qcount;
1624
1625 /* MAP VF's configuration registers */
1626 nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
1627 if (!nic->reg_base) {
1628 dev_err(dev, "Cannot map config register space, aborting\n");
1629 err = -ENOMEM;
1630 goto err_free_netdev;
1631 }
1632
1633 nic->drv_stats = netdev_alloc_pcpu_stats(struct nicvf_drv_stats);
1634 if (!nic->drv_stats) {
1635 err = -ENOMEM;
1636 goto err_free_netdev;
1637 }
1638
1639 err = nicvf_set_qset_resources(nic);
1640 if (err)
1641 goto err_free_netdev;
1642
1643 /* Check if PF is alive and get MAC address for this VF */
1644 err = nicvf_register_misc_interrupt(nic);
1645 if (err)
1646 goto err_free_netdev;
1647
1648 nicvf_send_vf_struct(nic);
1649
1650 if (!pass1_silicon(nic->pdev))
1651 nic->hw_tso = true;
1652
1653 /* Get iommu domain for iova to physical addr conversion */
1654 nic->iommu_domain = iommu_get_domain_for_dev(dev);
1655
1656 pci_read_config_word(nic->pdev, PCI_SUBSYSTEM_ID, &sdevid);
1657 if (sdevid == 0xA134)
1658 nic->t88 = true;
1659
1660 /* Check if this VF is in QS only mode */
1661 if (nic->sqs_mode)
1662 return 0;
1663
1664 err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues);
1665 if (err)
1666 goto err_unregister_interrupts;
1667
1668 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_SG |
1669 NETIF_F_TSO | NETIF_F_GRO |
1670 NETIF_F_HW_VLAN_CTAG_RX);
1671
1672 netdev->hw_features |= NETIF_F_RXHASH;
1673
1674 netdev->features |= netdev->hw_features;
1675 netdev->hw_features |= NETIF_F_LOOPBACK;
1676
1677 netdev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
1678
1679 netdev->netdev_ops = &nicvf_netdev_ops;
1680 netdev->watchdog_timeo = NICVF_TX_TIMEOUT;
1681
1682 /* MTU range: 64 - 9200 */
1683 netdev->min_mtu = NIC_HW_MIN_FRS;
1684 netdev->max_mtu = NIC_HW_MAX_FRS;
1685
1686 INIT_WORK(&nic->reset_task, nicvf_reset_task);
1687
1688 err = register_netdev(netdev);
1689 if (err) {
1690 dev_err(dev, "Failed to register netdevice\n");
1691 goto err_unregister_interrupts;
1692 }
1693
1694 nic->msg_enable = debug;
1695
1696 nicvf_set_ethtool_ops(netdev);
1697
1698 return 0;
1699
1700 err_unregister_interrupts:
1701 nicvf_unregister_interrupts(nic);
1702 err_free_netdev:
1703 pci_set_drvdata(pdev, NULL);
1704 if (nic->drv_stats)
1705 free_percpu(nic->drv_stats);
1706 free_netdev(netdev);
1707 err_release_regions:
1708 pci_release_regions(pdev);
1709 err_disable_device:
1710 pci_disable_device(pdev);
1711 return err;
1712 }
1713
1714 static void nicvf_remove(struct pci_dev *pdev)
1715 {
1716 struct net_device *netdev = pci_get_drvdata(pdev);
1717 struct nicvf *nic;
1718 struct net_device *pnetdev;
1719
1720 if (!netdev)
1721 return;
1722
1723 nic = netdev_priv(netdev);
1724 pnetdev = nic->pnicvf->netdev;
1725
1726 /* Check if this Qset is assigned to different VF.
1727 * If yes, clean primary and all secondary Qsets.
1728 */
1729 if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED))
1730 unregister_netdev(pnetdev);
1731 nicvf_unregister_interrupts(nic);
1732 pci_set_drvdata(pdev, NULL);
1733 if (nic->drv_stats)
1734 free_percpu(nic->drv_stats);
1735 free_netdev(netdev);
1736 pci_release_regions(pdev);
1737 pci_disable_device(pdev);
1738 }
1739
1740 static void nicvf_shutdown(struct pci_dev *pdev)
1741 {
1742 nicvf_remove(pdev);
1743 }
1744
1745 static struct pci_driver nicvf_driver = {
1746 .name = DRV_NAME,
1747 .id_table = nicvf_id_table,
1748 .probe = nicvf_probe,
1749 .remove = nicvf_remove,
1750 .shutdown = nicvf_shutdown,
1751 };
1752
1753 static int __init nicvf_init_module(void)
1754 {
1755 pr_info("%s, ver %s\n", DRV_NAME, DRV_VERSION);
1756
1757 return pci_register_driver(&nicvf_driver);
1758 }
1759
1760 static void __exit nicvf_cleanup_module(void)
1761 {
1762 pci_unregister_driver(&nicvf_driver);
1763 }
1764
1765 module_init(nicvf_init_module);
1766 module_exit(nicvf_cleanup_module);
Linux with added FPC-III support