?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / net / ethernet / sfc / efx.c
blobf448e7d669cf01078d0d24422e88f59d054a9169
1 /****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2013 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11 #include <linux/module.h>
12 #include <linux/pci.h>
13 #include <linux/netdevice.h>
14 #include <linux/etherdevice.h>
15 #include <linux/delay.h>
16 #include <linux/notifier.h>
17 #include <linux/ip.h>
18 #include <linux/tcp.h>
19 #include <linux/in.h>
20 #include <linux/ethtool.h>
21 #include <linux/topology.h>
22 #include <linux/gfp.h>
23 #include <linux/aer.h>
24 #include <linux/interrupt.h>
25 #include "net_driver.h"
26 #include <net/gre.h>
27 #include <net/udp_tunnel.h>
28 #include "efx.h"
29 #include "nic.h"
30 #include "io.h"
31 #include "selftest.h"
32 #include "sriov.h"
33
34 #include "mcdi.h"
35 #include "mcdi_pcol.h"
36 #include "workarounds.h"
37
38 /**************************************************************************
39 *
40 * Type name strings
41 *
42 **************************************************************************
43 */
44
45 /* Loopback mode names (see LOOPBACK_MODE()) */
46 const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
47 const char *const efx_loopback_mode_names[] = {
48 [LOOPBACK_NONE] = "NONE",
49 [LOOPBACK_DATA] = "DATAPATH",
50 [LOOPBACK_GMAC] = "GMAC",
51 [LOOPBACK_XGMII] = "XGMII",
52 [LOOPBACK_XGXS] = "XGXS",
53 [LOOPBACK_XAUI] = "XAUI",
54 [LOOPBACK_GMII] = "GMII",
55 [LOOPBACK_SGMII] = "SGMII",
56 [LOOPBACK_XGBR] = "XGBR",
57 [LOOPBACK_XFI] = "XFI",
58 [LOOPBACK_XAUI_FAR] = "XAUI_FAR",
59 [LOOPBACK_GMII_FAR] = "GMII_FAR",
60 [LOOPBACK_SGMII_FAR] = "SGMII_FAR",
61 [LOOPBACK_XFI_FAR] = "XFI_FAR",
62 [LOOPBACK_GPHY] = "GPHY",
63 [LOOPBACK_PHYXS] = "PHYXS",
64 [LOOPBACK_PCS] = "PCS",
65 [LOOPBACK_PMAPMD] = "PMA/PMD",
66 [LOOPBACK_XPORT] = "XPORT",
67 [LOOPBACK_XGMII_WS] = "XGMII_WS",
68 [LOOPBACK_XAUI_WS] = "XAUI_WS",
69 [LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR",
70 [LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
71 [LOOPBACK_GMII_WS] = "GMII_WS",
72 [LOOPBACK_XFI_WS] = "XFI_WS",
73 [LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR",
74 [LOOPBACK_PHYXS_WS] = "PHYXS_WS",
75 };
76
77 const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
78 const char *const efx_reset_type_names[] = {
79 [RESET_TYPE_INVISIBLE] = "INVISIBLE",
80 [RESET_TYPE_ALL] = "ALL",
81 [RESET_TYPE_RECOVER_OR_ALL] = "RECOVER_OR_ALL",
82 [RESET_TYPE_WORLD] = "WORLD",
83 [RESET_TYPE_RECOVER_OR_DISABLE] = "RECOVER_OR_DISABLE",
84 [RESET_TYPE_DATAPATH] = "DATAPATH",
85 [RESET_TYPE_MC_BIST] = "MC_BIST",
86 [RESET_TYPE_DISABLE] = "DISABLE",
87 [RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG",
88 [RESET_TYPE_INT_ERROR] = "INT_ERROR",
89 [RESET_TYPE_DMA_ERROR] = "DMA_ERROR",
90 [RESET_TYPE_TX_SKIP] = "TX_SKIP",
91 [RESET_TYPE_MC_FAILURE] = "MC_FAILURE",
92 [RESET_TYPE_MCDI_TIMEOUT] = "MCDI_TIMEOUT (FLR)",
93 };
94
95 /* UDP tunnel type names */
96 static const char *const efx_udp_tunnel_type_names[] = {
97 [TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN] = "vxlan",
98 [TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE] = "geneve",
99 };
100
101 void efx_get_udp_tunnel_type_name(u16 type, char *buf, size_t buflen)
102 {
103 if (type < ARRAY_SIZE(efx_udp_tunnel_type_names) &&
104 efx_udp_tunnel_type_names[type] != NULL)
105 snprintf(buf, buflen, "%s", efx_udp_tunnel_type_names[type]);
106 else
107 snprintf(buf, buflen, "type %d", type);
108 }
109
110 /* Reset workqueue. If any NIC has a hardware failure then a reset will be
111 * queued onto this work queue. This is not a per-nic work queue, because
112 * efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
113 */
114 static struct workqueue_struct *reset_workqueue;
115
116 /* How often and how many times to poll for a reset while waiting for a
117 * BIST that another function started to complete.
118 */
119 #define BIST_WAIT_DELAY_MS 100
120 #define BIST_WAIT_DELAY_COUNT 100
121
122 /**************************************************************************
123 *
124 * Configurable values
125 *
126 *************************************************************************/
127
128 /*
129 * Use separate channels for TX and RX events
130 *
131 * Set this to 1 to use separate channels for TX and RX. It allows us
132 * to control interrupt affinity separately for TX and RX.
133 *
134 * This is only used in MSI-X interrupt mode
135 */
136 bool efx_separate_tx_channels;
137 module_param(efx_separate_tx_channels, bool, 0444);
138 MODULE_PARM_DESC(efx_separate_tx_channels,
139 "Use separate channels for TX and RX");
140
141 /* This is the weight assigned to each of the (per-channel) virtual
142 * NAPI devices.
143 */
144 static int napi_weight = 64;
145
146 /* This is the time (in jiffies) between invocations of the hardware
147 * monitor.
148 * On Falcon-based NICs, this will:
149 * - Check the on-board hardware monitor;
150 * - Poll the link state and reconfigure the hardware as necessary.
151 * On Siena-based NICs for power systems with EEH support, this will give EEH a
152 * chance to start.
153 */
154 static unsigned int efx_monitor_interval = 1 * HZ;
155
156 /* Initial interrupt moderation settings. They can be modified after
157 * module load with ethtool.
158 *
159 * The default for RX should strike a balance between increasing the
160 * round-trip latency and reducing overhead.
161 */
162 static unsigned int rx_irq_mod_usec = 60;
163
164 /* Initial interrupt moderation settings. They can be modified after
165 * module load with ethtool.
166 *
167 * This default is chosen to ensure that a 10G link does not go idle
168 * while a TX queue is stopped after it has become full. A queue is
169 * restarted when it drops below half full. The time this takes (assuming
170 * worst case 3 descriptors per packet and 1024 descriptors) is
171 * 512 / 3 * 1.2 = 205 usec.
172 */
173 static unsigned int tx_irq_mod_usec = 150;
174
175 /* This is the first interrupt mode to try out of:
176 * 0 => MSI-X
177 * 1 => MSI
178 * 2 => legacy
179 */
180 static unsigned int interrupt_mode;
181
182 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
183 * i.e. the number of CPUs among which we may distribute simultaneous
184 * interrupt handling.
185 *
186 * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
187 * The default (0) means to assign an interrupt to each core.
188 */
189 static unsigned int rss_cpus;
190 module_param(rss_cpus, uint, 0444);
191 MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
192
193 static bool phy_flash_cfg;
194 module_param(phy_flash_cfg, bool, 0644);
195 MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
196
197 static unsigned irq_adapt_low_thresh = 8000;
198 module_param(irq_adapt_low_thresh, uint, 0644);
199 MODULE_PARM_DESC(irq_adapt_low_thresh,
200 "Threshold score for reducing IRQ moderation");
201
202 static unsigned irq_adapt_high_thresh = 16000;
203 module_param(irq_adapt_high_thresh, uint, 0644);
204 MODULE_PARM_DESC(irq_adapt_high_thresh,
205 "Threshold score for increasing IRQ moderation");
206
207 static unsigned debug = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
208 NETIF_MSG_LINK | NETIF_MSG_IFDOWN |
209 NETIF_MSG_IFUP | NETIF_MSG_RX_ERR |
210 NETIF_MSG_TX_ERR | NETIF_MSG_HW);
211 module_param(debug, uint, 0);
212 MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
213
214 /**************************************************************************
215 *
216 * Utility functions and prototypes
217 *
218 *************************************************************************/
219
220 static int efx_soft_enable_interrupts(struct efx_nic *efx);
221 static void efx_soft_disable_interrupts(struct efx_nic *efx);
222 static void efx_remove_channel(struct efx_channel *channel);
223 static void efx_remove_channels(struct efx_nic *efx);
224 static const struct efx_channel_type efx_default_channel_type;
225 static void efx_remove_port(struct efx_nic *efx);
226 static void efx_init_napi_channel(struct efx_channel *channel);
227 static void efx_fini_napi(struct efx_nic *efx);
228 static void efx_fini_napi_channel(struct efx_channel *channel);
229 static void efx_fini_struct(struct efx_nic *efx);
230 static void efx_start_all(struct efx_nic *efx);
231 static void efx_stop_all(struct efx_nic *efx);
232
233 #define EFX_ASSERT_RESET_SERIALISED(efx) \
234 do { \
235 if ((efx->state == STATE_READY) || \
236 (efx->state == STATE_RECOVERY) || \
237 (efx->state == STATE_DISABLED)) \
238 ASSERT_RTNL(); \
239 } while (0)
240
241 static int efx_check_disabled(struct efx_nic *efx)
242 {
243 if (efx->state == STATE_DISABLED || efx->state == STATE_RECOVERY) {
244 netif_err(efx, drv, efx->net_dev,
245 "device is disabled due to earlier errors\n");
246 return -EIO;
247 }
248 return 0;
249 }
250
251 /**************************************************************************
252 *
253 * Event queue processing
254 *
255 *************************************************************************/
256
257 /* Process channel's event queue
258 *
259 * This function is responsible for processing the event queue of a
260 * single channel. The caller must guarantee that this function will
261 * never be concurrently called more than once on the same channel,
262 * though different channels may be being processed concurrently.
263 */
264 static int efx_process_channel(struct efx_channel *channel, int budget)
265 {
266 struct efx_tx_queue *tx_queue;
267 struct list_head rx_list;
268 int spent;
269
270 if (unlikely(!channel->enabled))
271 return 0;
272
273 /* Prepare the batch receive list */
274 EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
275 INIT_LIST_HEAD(&rx_list);
276 channel->rx_list = &rx_list;
277
278 efx_for_each_channel_tx_queue(tx_queue, channel) {
279 tx_queue->pkts_compl = 0;
280 tx_queue->bytes_compl = 0;
281 }
282
283 spent = efx_nic_process_eventq(channel, budget);
284 if (spent && efx_channel_has_rx_queue(channel)) {
285 struct efx_rx_queue *rx_queue =
286 efx_channel_get_rx_queue(channel);
287
288 efx_rx_flush_packet(channel);
289 efx_fast_push_rx_descriptors(rx_queue, true);
290 }
291
292 /* Update BQL */
293 efx_for_each_channel_tx_queue(tx_queue, channel) {
294 if (tx_queue->bytes_compl) {
295 netdev_tx_completed_queue(tx_queue->core_txq,
296 tx_queue->pkts_compl, tx_queue->bytes_compl);
297 }
298 }
299
300 /* Receive any packets we queued up */
301 netif_receive_skb_list(channel->rx_list);
302 channel->rx_list = NULL;
303
304 return spent;
305 }
306
307 /* NAPI poll handler
308 *
309 * NAPI guarantees serialisation of polls of the same device, which
310 * provides the guarantee required by efx_process_channel().
311 */
312 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
313 {
314 int step = efx->irq_mod_step_us;
315
316 if (channel->irq_mod_score < irq_adapt_low_thresh) {
317 if (channel->irq_moderation_us > step) {
318 channel->irq_moderation_us -= step;
319 efx->type->push_irq_moderation(channel);
320 }
321 } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
322 if (channel->irq_moderation_us <
323 efx->irq_rx_moderation_us) {
324 channel->irq_moderation_us += step;
325 efx->type->push_irq_moderation(channel);
326 }
327 }
328
329 channel->irq_count = 0;
330 channel->irq_mod_score = 0;
331 }
332
333 static int efx_poll(struct napi_struct *napi, int budget)
334 {
335 struct efx_channel *channel =
336 container_of(napi, struct efx_channel, napi_str);
337 struct efx_nic *efx = channel->efx;
338 int spent;
339
340 netif_vdbg(efx, intr, efx->net_dev,
341 "channel %d NAPI poll executing on CPU %d\n",
342 channel->channel, raw_smp_processor_id());
343
344 spent = efx_process_channel(channel, budget);
345
346 if (spent < budget) {
347 if (efx_channel_has_rx_queue(channel) &&
348 efx->irq_rx_adaptive &&
349 unlikely(++channel->irq_count == 1000)) {
350 efx_update_irq_mod(efx, channel);
351 }
352
353 #ifdef CONFIG_RFS_ACCEL
354 /* Perhaps expire some ARFS filters */
355 schedule_work(&channel->filter_work);
356 #endif
357
358 /* There is no race here; although napi_disable() will
359 * only wait for napi_complete(), this isn't a problem
360 * since efx_nic_eventq_read_ack() will have no effect if
361 * interrupts have already been disabled.
362 */
363 if (napi_complete_done(napi, spent))
364 efx_nic_eventq_read_ack(channel);
365 }
366
367 return spent;
368 }
369
370 /* Create event queue
371 * Event queue memory allocations are done only once. If the channel
372 * is reset, the memory buffer will be reused; this guards against
373 * errors during channel reset and also simplifies interrupt handling.
374 */
375 static int efx_probe_eventq(struct efx_channel *channel)
376 {
377 struct efx_nic *efx = channel->efx;
378 unsigned long entries;
379
380 netif_dbg(efx, probe, efx->net_dev,
381 "chan %d create event queue\n", channel->channel);
382
383 /* Build an event queue with room for one event per tx and rx buffer,
384 * plus some extra for link state events and MCDI completions. */
385 entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
386 EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
387 channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
388
389 return efx_nic_probe_eventq(channel);
390 }
391
392 /* Prepare channel's event queue */
393 static int efx_init_eventq(struct efx_channel *channel)
394 {
395 struct efx_nic *efx = channel->efx;
396 int rc;
397
398 EFX_WARN_ON_PARANOID(channel->eventq_init);
399
400 netif_dbg(efx, drv, efx->net_dev,
401 "chan %d init event queue\n", channel->channel);
402
403 rc = efx_nic_init_eventq(channel);
404 if (rc == 0) {
405 efx->type->push_irq_moderation(channel);
406 channel->eventq_read_ptr = 0;
407 channel->eventq_init = true;
408 }
409 return rc;
410 }
411
412 /* Enable event queue processing and NAPI */
413 void efx_start_eventq(struct efx_channel *channel)
414 {
415 netif_dbg(channel->efx, ifup, channel->efx->net_dev,
416 "chan %d start event queue\n", channel->channel);
417
418 /* Make sure the NAPI handler sees the enabled flag set */
419 channel->enabled = true;
420 smp_wmb();
421
422 napi_enable(&channel->napi_str);
423 efx_nic_eventq_read_ack(channel);
424 }
425
426 /* Disable event queue processing and NAPI */
427 void efx_stop_eventq(struct efx_channel *channel)
428 {
429 if (!channel->enabled)
430 return;
431
432 napi_disable(&channel->napi_str);
433 channel->enabled = false;
434 }
435
436 static void efx_fini_eventq(struct efx_channel *channel)
437 {
438 if (!channel->eventq_init)
439 return;
440
441 netif_dbg(channel->efx, drv, channel->efx->net_dev,
442 "chan %d fini event queue\n", channel->channel);
443
444 efx_nic_fini_eventq(channel);
445 channel->eventq_init = false;
446 }
447
448 static void efx_remove_eventq(struct efx_channel *channel)
449 {
450 netif_dbg(channel->efx, drv, channel->efx->net_dev,
451 "chan %d remove event queue\n", channel->channel);
452
453 efx_nic_remove_eventq(channel);
454 }
455
456 /**************************************************************************
457 *
458 * Channel handling
459 *
460 *************************************************************************/
461
462 /* Allocate and initialise a channel structure. */
463 static struct efx_channel *
464 efx_alloc_channel(struct efx_nic *efx, int i, struct efx_channel *old_channel)
465 {
466 struct efx_channel *channel;
467 struct efx_rx_queue *rx_queue;
468 struct efx_tx_queue *tx_queue;
469 int j;
470
471 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
472 if (!channel)
473 return NULL;
474
475 channel->efx = efx;
476 channel->channel = i;
477 channel->type = &efx_default_channel_type;
478
479 for (j = 0; j < EFX_TXQ_TYPES; j++) {
480 tx_queue = &channel->tx_queue[j];
481 tx_queue->efx = efx;
482 tx_queue->queue = i * EFX_TXQ_TYPES + j;
483 tx_queue->channel = channel;
484 }
485
486 #ifdef CONFIG_RFS_ACCEL
487 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire);
488 #endif
489
490 rx_queue = &channel->rx_queue;
491 rx_queue->efx = efx;
492 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
493
494 return channel;
495 }
496
497 /* Allocate and initialise a channel structure, copying parameters
498 * (but not resources) from an old channel structure.
499 */
500 static struct efx_channel *
501 efx_copy_channel(const struct efx_channel *old_channel)
502 {
503 struct efx_channel *channel;
504 struct efx_rx_queue *rx_queue;
505 struct efx_tx_queue *tx_queue;
506 int j;
507
508 channel = kmalloc(sizeof(*channel), GFP_KERNEL);
509 if (!channel)
510 return NULL;
511
512 *channel = *old_channel;
513
514 channel->napi_dev = NULL;
515 INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
516 channel->napi_str.napi_id = 0;
517 channel->napi_str.state = 0;
518 memset(&channel->eventq, 0, sizeof(channel->eventq));
519
520 for (j = 0; j < EFX_TXQ_TYPES; j++) {
521 tx_queue = &channel->tx_queue[j];
522 if (tx_queue->channel)
523 tx_queue->channel = channel;
524 tx_queue->buffer = NULL;
525 tx_queue->cb_page = NULL;
526 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
527 }
528
529 rx_queue = &channel->rx_queue;
530 rx_queue->buffer = NULL;
531 memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
532 timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
533 #ifdef CONFIG_RFS_ACCEL
534 INIT_WORK(&channel->filter_work, efx_filter_rfs_expire);
535 #endif
536
537 return channel;
538 }
539
540 static int efx_probe_channel(struct efx_channel *channel)
541 {
542 struct efx_tx_queue *tx_queue;
543 struct efx_rx_queue *rx_queue;
544 int rc;
545
546 netif_dbg(channel->efx, probe, channel->efx->net_dev,
547 "creating channel %d\n", channel->channel);
548
549 rc = channel->type->pre_probe(channel);
550 if (rc)
551 goto fail;
552
553 rc = efx_probe_eventq(channel);
554 if (rc)
555 goto fail;
556
557 efx_for_each_channel_tx_queue(tx_queue, channel) {
558 rc = efx_probe_tx_queue(tx_queue);
559 if (rc)
560 goto fail;
561 }
562
563 efx_for_each_channel_rx_queue(rx_queue, channel) {
564 rc = efx_probe_rx_queue(rx_queue);
565 if (rc)
566 goto fail;
567 }
568
569 channel->rx_list = NULL;
570
571 return 0;
572
573 fail:
574 efx_remove_channel(channel);
575 return rc;
576 }
577
578 static void
579 efx_get_channel_name(struct efx_channel *channel, char *buf, size_t len)
580 {
581 struct efx_nic *efx = channel->efx;
582 const char *type;
583 int number;
584
585 number = channel->channel;
586 if (efx->tx_channel_offset == 0) {
587 type = "";
588 } else if (channel->channel < efx->tx_channel_offset) {
589 type = "-rx";
590 } else {
591 type = "-tx";
592 number -= efx->tx_channel_offset;
593 }
594 snprintf(buf, len, "%s%s-%d", efx->name, type, number);
595 }
596
597 static void efx_set_channel_names(struct efx_nic *efx)
598 {
599 struct efx_channel *channel;
600
601 efx_for_each_channel(channel, efx)
602 channel->type->get_name(channel,
603 efx->msi_context[channel->channel].name,
604 sizeof(efx->msi_context[0].name));
605 }
606
607 static int efx_probe_channels(struct efx_nic *efx)
608 {
609 struct efx_channel *channel;
610 int rc;
611
612 /* Restart special buffer allocation */
613 efx->next_buffer_table = 0;
614
615 /* Probe channels in reverse, so that any 'extra' channels
616 * use the start of the buffer table. This allows the traffic
617 * channels to be resized without moving them or wasting the
618 * entries before them.
619 */
620 efx_for_each_channel_rev(channel, efx) {
621 rc = efx_probe_channel(channel);
622 if (rc) {
623 netif_err(efx, probe, efx->net_dev,
624 "failed to create channel %d\n",
625 channel->channel);
626 goto fail;
627 }
628 }
629 efx_set_channel_names(efx);
630
631 return 0;
632
633 fail:
634 efx_remove_channels(efx);
635 return rc;
636 }
637
638 /* Channels are shutdown and reinitialised whilst the NIC is running
639 * to propagate configuration changes (mtu, checksum offload), or
640 * to clear hardware error conditions
641 */
642 static void efx_start_datapath(struct efx_nic *efx)
643 {
644 netdev_features_t old_features = efx->net_dev->features;
645 bool old_rx_scatter = efx->rx_scatter;
646 struct efx_tx_queue *tx_queue;
647 struct efx_rx_queue *rx_queue;
648 struct efx_channel *channel;
649 size_t rx_buf_len;
650
651 /* Calculate the rx buffer allocation parameters required to
652 * support the current MTU, including padding for header
653 * alignment and overruns.
654 */
655 efx->rx_dma_len = (efx->rx_prefix_size +
656 EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
657 efx->type->rx_buffer_padding);
658 rx_buf_len = (sizeof(struct efx_rx_page_state) +
659 efx->rx_ip_align + efx->rx_dma_len);
660 if (rx_buf_len <= PAGE_SIZE) {
661 efx->rx_scatter = efx->type->always_rx_scatter;
662 efx->rx_buffer_order = 0;
663 } else if (efx->type->can_rx_scatter) {
664 BUILD_BUG_ON(EFX_RX_USR_BUF_SIZE % L1_CACHE_BYTES);
665 BUILD_BUG_ON(sizeof(struct efx_rx_page_state) +
666 2 * ALIGN(NET_IP_ALIGN + EFX_RX_USR_BUF_SIZE,
667 EFX_RX_BUF_ALIGNMENT) >
668 PAGE_SIZE);
669 efx->rx_scatter = true;
670 efx->rx_dma_len = EFX_RX_USR_BUF_SIZE;
671 efx->rx_buffer_order = 0;
672 } else {
673 efx->rx_scatter = false;
674 efx->rx_buffer_order = get_order(rx_buf_len);
675 }
676
677 efx_rx_config_page_split(efx);
678 if (efx->rx_buffer_order)
679 netif_dbg(efx, drv, efx->net_dev,
680 "RX buf len=%u; page order=%u batch=%u\n",
681 efx->rx_dma_len, efx->rx_buffer_order,
682 efx->rx_pages_per_batch);
683 else
684 netif_dbg(efx, drv, efx->net_dev,
685 "RX buf len=%u step=%u bpp=%u; page batch=%u\n",
686 efx->rx_dma_len, efx->rx_page_buf_step,
687 efx->rx_bufs_per_page, efx->rx_pages_per_batch);
688
689 /* Restore previously fixed features in hw_features and remove
690 * features which are fixed now
691 */
692 efx->net_dev->hw_features |= efx->net_dev->features;
693 efx->net_dev->hw_features &= ~efx->fixed_features;
694 efx->net_dev->features |= efx->fixed_features;
695 if (efx->net_dev->features != old_features)
696 netdev_features_change(efx->net_dev);
697
698 /* RX filters may also have scatter-enabled flags */
699 if (efx->rx_scatter != old_rx_scatter)
700 efx->type->filter_update_rx_scatter(efx);
701
702 /* We must keep at least one descriptor in a TX ring empty.
703 * We could avoid this when the queue size does not exactly
704 * match the hardware ring size, but it's not that important.
705 * Therefore we stop the queue when one more skb might fill
706 * the ring completely. We wake it when half way back to
707 * empty.
708 */
709 efx->txq_stop_thresh = efx->txq_entries - efx_tx_max_skb_descs(efx);
710 efx->txq_wake_thresh = efx->txq_stop_thresh / 2;
711
712 /* Initialise the channels */
713 efx_for_each_channel(channel, efx) {
714 efx_for_each_channel_tx_queue(tx_queue, channel) {
715 efx_init_tx_queue(tx_queue);
716 atomic_inc(&efx->active_queues);
717 }
718
719 efx_for_each_channel_rx_queue(rx_queue, channel) {
720 efx_init_rx_queue(rx_queue);
721 atomic_inc(&efx->active_queues);
722 efx_stop_eventq(channel);
723 efx_fast_push_rx_descriptors(rx_queue, false);
724 efx_start_eventq(channel);
725 }
726
727 WARN_ON(channel->rx_pkt_n_frags);
728 }
729
730 efx_ptp_start_datapath(efx);
731
732 if (netif_device_present(efx->net_dev))
733 netif_tx_wake_all_queues(efx->net_dev);
734 }
735
736 static void efx_stop_datapath(struct efx_nic *efx)
737 {
738 struct efx_channel *channel;
739 struct efx_tx_queue *tx_queue;
740 struct efx_rx_queue *rx_queue;
741 int rc;
742
743 EFX_ASSERT_RESET_SERIALISED(efx);
744 BUG_ON(efx->port_enabled);
745
746 efx_ptp_stop_datapath(efx);
747
748 /* Stop RX refill */
749 efx_for_each_channel(channel, efx) {
750 efx_for_each_channel_rx_queue(rx_queue, channel)
751 rx_queue->refill_enabled = false;
752 }
753
754 efx_for_each_channel(channel, efx) {
755 /* RX packet processing is pipelined, so wait for the
756 * NAPI handler to complete. At least event queue 0
757 * might be kept active by non-data events, so don't
758 * use napi_synchronize() but actually disable NAPI
759 * temporarily.
760 */
761 if (efx_channel_has_rx_queue(channel)) {
762 efx_stop_eventq(channel);
763 efx_start_eventq(channel);
764 }
765 }
766
767 rc = efx->type->fini_dmaq(efx);
768 if (rc) {
769 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
770 } else {
771 netif_dbg(efx, drv, efx->net_dev,
772 "successfully flushed all queues\n");
773 }
774
775 efx_for_each_channel(channel, efx) {
776 efx_for_each_channel_rx_queue(rx_queue, channel)
777 efx_fini_rx_queue(rx_queue);
778 efx_for_each_possible_channel_tx_queue(tx_queue, channel)
779 efx_fini_tx_queue(tx_queue);
780 }
781 }
782
783 static void efx_remove_channel(struct efx_channel *channel)
784 {
785 struct efx_tx_queue *tx_queue;
786 struct efx_rx_queue *rx_queue;
787
788 netif_dbg(channel->efx, drv, channel->efx->net_dev,
789 "destroy chan %d\n", channel->channel);
790
791 efx_for_each_channel_rx_queue(rx_queue, channel)
792 efx_remove_rx_queue(rx_queue);
793 efx_for_each_possible_channel_tx_queue(tx_queue, channel)
794 efx_remove_tx_queue(tx_queue);
795 efx_remove_eventq(channel);
796 channel->type->post_remove(channel);
797 }
798
799 static void efx_remove_channels(struct efx_nic *efx)
800 {
801 struct efx_channel *channel;
802
803 efx_for_each_channel(channel, efx)
804 efx_remove_channel(channel);
805 }
806
807 int
808 efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
809 {
810 struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel;
811 u32 old_rxq_entries, old_txq_entries;
812 unsigned i, next_buffer_table = 0;
813 int rc, rc2;
814
815 rc = efx_check_disabled(efx);
816 if (rc)
817 return rc;
818
819 /* Not all channels should be reallocated. We must avoid
820 * reallocating their buffer table entries.
821 */
822 efx_for_each_channel(channel, efx) {
823 struct efx_rx_queue *rx_queue;
824 struct efx_tx_queue *tx_queue;
825
826 if (channel->type->copy)
827 continue;
828 next_buffer_table = max(next_buffer_table,
829 channel->eventq.index +
830 channel->eventq.entries);
831 efx_for_each_channel_rx_queue(rx_queue, channel)
832 next_buffer_table = max(next_buffer_table,
833 rx_queue->rxd.index +
834 rx_queue->rxd.entries);
835 efx_for_each_channel_tx_queue(tx_queue, channel)
836 next_buffer_table = max(next_buffer_table,
837 tx_queue->txd.index +
838 tx_queue->txd.entries);
839 }
840
841 efx_device_detach_sync(efx);
842 efx_stop_all(efx);
843 efx_soft_disable_interrupts(efx);
844
845 /* Clone channels (where possible) */
846 memset(other_channel, 0, sizeof(other_channel));
847 for (i = 0; i < efx->n_channels; i++) {
848 channel = efx->channel[i];
849 if (channel->type->copy)
850 channel = channel->type->copy(channel);
851 if (!channel) {
852 rc = -ENOMEM;
853 goto out;
854 }
855 other_channel[i] = channel;
856 }
857
858 /* Swap entry counts and channel pointers */
859 old_rxq_entries = efx->rxq_entries;
860 old_txq_entries = efx->txq_entries;
861 efx->rxq_entries = rxq_entries;
862 efx->txq_entries = txq_entries;
863 for (i = 0; i < efx->n_channels; i++) {
864 channel = efx->channel[i];
865 efx->channel[i] = other_channel[i];
866 other_channel[i] = channel;
867 }
868
869 /* Restart buffer table allocation */
870 efx->next_buffer_table = next_buffer_table;
871
872 for (i = 0; i < efx->n_channels; i++) {
873 channel = efx->channel[i];
874 if (!channel->type->copy)
875 continue;
876 rc = efx_probe_channel(channel);
877 if (rc)
878 goto rollback;
879 efx_init_napi_channel(efx->channel[i]);
880 }
881
882 out:
883 /* Destroy unused channel structures */
884 for (i = 0; i < efx->n_channels; i++) {
885 channel = other_channel[i];
886 if (channel && channel->type->copy) {
887 efx_fini_napi_channel(channel);
888 efx_remove_channel(channel);
889 kfree(channel);
890 }
891 }
892
893 rc2 = efx_soft_enable_interrupts(efx);
894 if (rc2) {
895 rc = rc ? rc : rc2;
896 netif_err(efx, drv, efx->net_dev,
897 "unable to restart interrupts on channel reallocation\n");
898 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
899 } else {
900 efx_start_all(efx);
901 efx_device_attach_if_not_resetting(efx);
902 }
903 return rc;
904
905 rollback:
906 /* Swap back */
907 efx->rxq_entries = old_rxq_entries;
908 efx->txq_entries = old_txq_entries;
909 for (i = 0; i < efx->n_channels; i++) {
910 channel = efx->channel[i];
911 efx->channel[i] = other_channel[i];
912 other_channel[i] = channel;
913 }
914 goto out;
915 }
916
917 void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
918 {
919 mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
920 }
921
922 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
923 {
924 return channel->channel - channel->efx->tx_channel_offset <
925 channel->efx->n_tx_channels;
926 }
927
928 static const struct efx_channel_type efx_default_channel_type = {
929 .pre_probe = efx_channel_dummy_op_int,
930 .post_remove = efx_channel_dummy_op_void,
931 .get_name = efx_get_channel_name,
932 .copy = efx_copy_channel,
933 .want_txqs = efx_default_channel_want_txqs,
934 .keep_eventq = false,
935 .want_pio = true,
936 };
937
938 int efx_channel_dummy_op_int(struct efx_channel *channel)
939 {
940 return 0;
941 }
942
943 void efx_channel_dummy_op_void(struct efx_channel *channel)
944 {
945 }
946
947 /**************************************************************************
948 *
949 * Port handling
950 *
951 **************************************************************************/
952
953 /* This ensures that the kernel is kept informed (via
954 * netif_carrier_on/off) of the link status, and also maintains the
955 * link status's stop on the port's TX queue.
956 */
957 void efx_link_status_changed(struct efx_nic *efx)
958 {
959 struct efx_link_state *link_state = &efx->link_state;
960
961 /* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
962 * that no events are triggered between unregister_netdev() and the
963 * driver unloading. A more general condition is that NETDEV_CHANGE
964 * can only be generated between NETDEV_UP and NETDEV_DOWN */
965 if (!netif_running(efx->net_dev))
966 return;
967
968 if (link_state->up != netif_carrier_ok(efx->net_dev)) {
969 efx->n_link_state_changes++;
970
971 if (link_state->up)
972 netif_carrier_on(efx->net_dev);
973 else
974 netif_carrier_off(efx->net_dev);
975 }
976
977 /* Status message for kernel log */
978 if (link_state->up)
979 netif_info(efx, link, efx->net_dev,
980 "link up at %uMbps %s-duplex (MTU %d)\n",
981 link_state->speed, link_state->fd ? "full" : "half",
982 efx->net_dev->mtu);
983 else
984 netif_info(efx, link, efx->net_dev, "link down\n");
985 }
986
987 void efx_link_set_advertising(struct efx_nic *efx,
988 const unsigned long *advertising)
989 {
990 memcpy(efx->link_advertising, advertising,
991 sizeof(__ETHTOOL_DECLARE_LINK_MODE_MASK()));
992
993 efx->link_advertising[0] |= ADVERTISED_Autoneg;
994 if (advertising[0] & ADVERTISED_Pause)
995 efx->wanted_fc |= (EFX_FC_TX | EFX_FC_RX);
996 else
997 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
998 if (advertising[0] & ADVERTISED_Asym_Pause)
999 efx->wanted_fc ^= EFX_FC_TX;
1000 }
1001
1002 /* Equivalent to efx_link_set_advertising with all-zeroes, except does not
1003 * force the Autoneg bit on.
1004 */
1005 void efx_link_clear_advertising(struct efx_nic *efx)
1006 {
1007 bitmap_zero(efx->link_advertising, __ETHTOOL_LINK_MODE_MASK_NBITS);
1008 efx->wanted_fc &= ~(EFX_FC_TX | EFX_FC_RX);
1009 }
1010
1011 void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
1012 {
1013 efx->wanted_fc = wanted_fc;
1014 if (efx->link_advertising[0]) {
1015 if (wanted_fc & EFX_FC_RX)
1016 efx->link_advertising[0] |= (ADVERTISED_Pause |
1017 ADVERTISED_Asym_Pause);
1018 else
1019 efx->link_advertising[0] &= ~(ADVERTISED_Pause |
1020 ADVERTISED_Asym_Pause);
1021 if (wanted_fc & EFX_FC_TX)
1022 efx->link_advertising[0] ^= ADVERTISED_Asym_Pause;
1023 }
1024 }
1025
1026 static void efx_fini_port(struct efx_nic *efx);
1027
1028 /* We assume that efx->type->reconfigure_mac will always try to sync RX
1029 * filters and therefore needs to read-lock the filter table against freeing
1030 */
1031 void efx_mac_reconfigure(struct efx_nic *efx)
1032 {
1033 down_read(&efx->filter_sem);
1034 efx->type->reconfigure_mac(efx);
1035 up_read(&efx->filter_sem);
1036 }
1037
1038 /* Push loopback/power/transmit disable settings to the PHY, and reconfigure
1039 * the MAC appropriately. All other PHY configuration changes are pushed
1040 * through phy_op->set_settings(), and pushed asynchronously to the MAC
1041 * through efx_monitor().
1042 *
1043 * Callers must hold the mac_lock
1044 */
1045 int __efx_reconfigure_port(struct efx_nic *efx)
1046 {
1047 enum efx_phy_mode phy_mode;
1048 int rc;
1049
1050 WARN_ON(!mutex_is_locked(&efx->mac_lock));
1051
1052 /* Disable PHY transmit in mac level loopbacks */
1053 phy_mode = efx->phy_mode;
1054 if (LOOPBACK_INTERNAL(efx))
1055 efx->phy_mode |= PHY_MODE_TX_DISABLED;
1056 else
1057 efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
1058
1059 rc = efx->type->reconfigure_port(efx);
1060
1061 if (rc)
1062 efx->phy_mode = phy_mode;
1063
1064 return rc;
1065 }
1066
1067 /* Reinitialise the MAC to pick up new PHY settings, even if the port is
1068 * disabled. */
1069 int efx_reconfigure_port(struct efx_nic *efx)
1070 {
1071 int rc;
1072
1073 EFX_ASSERT_RESET_SERIALISED(efx);
1074
1075 mutex_lock(&efx->mac_lock);
1076 rc = __efx_reconfigure_port(efx);
1077 mutex_unlock(&efx->mac_lock);
1078
1079 return rc;
1080 }
1081
1082 /* Asynchronous work item for changing MAC promiscuity and multicast
1083 * hash. Avoid a drain/rx_ingress enable by reconfiguring the current
1084 * MAC directly. */
1085 static void efx_mac_work(struct work_struct *data)
1086 {
1087 struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
1088
1089 mutex_lock(&efx->mac_lock);
1090 if (efx->port_enabled)
1091 efx_mac_reconfigure(efx);
1092 mutex_unlock(&efx->mac_lock);
1093 }
1094
1095 static int efx_probe_port(struct efx_nic *efx)
1096 {
1097 int rc;
1098
1099 netif_dbg(efx, probe, efx->net_dev, "create port\n");
1100
1101 if (phy_flash_cfg)
1102 efx->phy_mode = PHY_MODE_SPECIAL;
1103
1104 /* Connect up MAC/PHY operations table */
1105 rc = efx->type->probe_port(efx);
1106 if (rc)
1107 return rc;
1108
1109 /* Initialise MAC address to permanent address */
1110 ether_addr_copy(efx->net_dev->dev_addr, efx->net_dev->perm_addr);
1111
1112 return 0;
1113 }
1114
1115 static int efx_init_port(struct efx_nic *efx)
1116 {
1117 int rc;
1118
1119 netif_dbg(efx, drv, efx->net_dev, "init port\n");
1120
1121 mutex_lock(&efx->mac_lock);
1122
1123 rc = efx->phy_op->init(efx);
1124 if (rc)
1125 goto fail1;
1126
1127 efx->port_initialized = true;
1128
1129 /* Reconfigure the MAC before creating dma queues (required for
1130 * Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
1131 efx_mac_reconfigure(efx);
1132
1133 /* Ensure the PHY advertises the correct flow control settings */
1134 rc = efx->phy_op->reconfigure(efx);
1135 if (rc && rc != -EPERM)
1136 goto fail2;
1137
1138 mutex_unlock(&efx->mac_lock);
1139 return 0;
1140
1141 fail2:
1142 efx->phy_op->fini(efx);
1143 fail1:
1144 mutex_unlock(&efx->mac_lock);
1145 return rc;
1146 }
1147
1148 static void efx_start_port(struct efx_nic *efx)
1149 {
1150 netif_dbg(efx, ifup, efx->net_dev, "start port\n");
1151 BUG_ON(efx->port_enabled);
1152
1153 mutex_lock(&efx->mac_lock);
1154 efx->port_enabled = true;
1155
1156 /* Ensure MAC ingress/egress is enabled */
1157 efx_mac_reconfigure(efx);
1158
1159 mutex_unlock(&efx->mac_lock);
1160 }
1161
1162 /* Cancel work for MAC reconfiguration, periodic hardware monitoring
1163 * and the async self-test, wait for them to finish and prevent them
1164 * being scheduled again. This doesn't cover online resets, which
1165 * should only be cancelled when removing the device.
1166 */
1167 static void efx_stop_port(struct efx_nic *efx)
1168 {
1169 netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
1170
1171 EFX_ASSERT_RESET_SERIALISED(efx);
1172
1173 mutex_lock(&efx->mac_lock);
1174 efx->port_enabled = false;
1175 mutex_unlock(&efx->mac_lock);
1176
1177 /* Serialise against efx_set_multicast_list() */
1178 netif_addr_lock_bh(efx->net_dev);
1179 netif_addr_unlock_bh(efx->net_dev);
1180
1181 cancel_delayed_work_sync(&efx->monitor_work);
1182 efx_selftest_async_cancel(efx);
1183 cancel_work_sync(&efx->mac_work);
1184 }
1185
1186 static void efx_fini_port(struct efx_nic *efx)
1187 {
1188 netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
1189
1190 if (!efx->port_initialized)
1191 return;
1192
1193 efx->phy_op->fini(efx);
1194 efx->port_initialized = false;
1195
1196 efx->link_state.up = false;
1197 efx_link_status_changed(efx);
1198 }
1199
1200 static void efx_remove_port(struct efx_nic *efx)
1201 {
1202 netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
1203
1204 efx->type->remove_port(efx);
1205 }
1206
1207 /**************************************************************************
1208 *
1209 * NIC handling
1210 *
1211 **************************************************************************/
1212
1213 static LIST_HEAD(efx_primary_list);
1214 static LIST_HEAD(efx_unassociated_list);
1215
1216 static bool efx_same_controller(struct efx_nic *left, struct efx_nic *right)
1217 {
1218 return left->type == right->type &&
1219 left->vpd_sn && right->vpd_sn &&
1220 !strcmp(left->vpd_sn, right->vpd_sn);
1221 }
1222
1223 static void efx_associate(struct efx_nic *efx)
1224 {
1225 struct efx_nic *other, *next;
1226
1227 if (efx->primary == efx) {
1228 /* Adding primary function; look for secondaries */
1229
1230 netif_dbg(efx, probe, efx->net_dev, "adding to primary list\n");
1231 list_add_tail(&efx->node, &efx_primary_list);
1232
1233 list_for_each_entry_safe(other, next, &efx_unassociated_list,
1234 node) {
1235 if (efx_same_controller(efx, other)) {
1236 list_del(&other->node);
1237 netif_dbg(other, probe, other->net_dev,
1238 "moving to secondary list of %s %s\n",
1239 pci_name(efx->pci_dev),
1240 efx->net_dev->name);
1241 list_add_tail(&other->node,
1242 &efx->secondary_list);
1243 other->primary = efx;
1244 }
1245 }
1246 } else {
1247 /* Adding secondary function; look for primary */
1248
1249 list_for_each_entry(other, &efx_primary_list, node) {
1250 if (efx_same_controller(efx, other)) {
1251 netif_dbg(efx, probe, efx->net_dev,
1252 "adding to secondary list of %s %s\n",
1253 pci_name(other->pci_dev),
1254 other->net_dev->name);
1255 list_add_tail(&efx->node,
1256 &other->secondary_list);
1257 efx->primary = other;
1258 return;
1259 }
1260 }
1261
1262 netif_dbg(efx, probe, efx->net_dev,
1263 "adding to unassociated list\n");
1264 list_add_tail(&efx->node, &efx_unassociated_list);
1265 }
1266 }
1267
1268 static void efx_dissociate(struct efx_nic *efx)
1269 {
1270 struct efx_nic *other, *next;
1271
1272 list_del(&efx->node);
1273 efx->primary = NULL;
1274
1275 list_for_each_entry_safe(other, next, &efx->secondary_list, node) {
1276 list_del(&other->node);
1277 netif_dbg(other, probe, other->net_dev,
1278 "moving to unassociated list\n");
1279 list_add_tail(&other->node, &efx_unassociated_list);
1280 other->primary = NULL;
1281 }
1282 }
1283
1284 /* This configures the PCI device to enable I/O and DMA. */
1285 static int efx_init_io(struct efx_nic *efx)
1286 {
1287 struct pci_dev *pci_dev = efx->pci_dev;
1288 dma_addr_t dma_mask = efx->type->max_dma_mask;
1289 unsigned int mem_map_size = efx->type->mem_map_size(efx);
1290 int rc, bar;
1291
1292 netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
1293
1294 bar = efx->type->mem_bar(efx);
1295
1296 rc = pci_enable_device(pci_dev);
1297 if (rc) {
1298 netif_err(efx, probe, efx->net_dev,
1299 "failed to enable PCI device\n");
1300 goto fail1;
1301 }
1302
1303 pci_set_master(pci_dev);
1304
1305 /* Set the PCI DMA mask. Try all possibilities from our genuine mask
1306 * down to 32 bits, because some architectures will allow 40 bit
1307 * masks event though they reject 46 bit masks.
1308 */
1309 while (dma_mask > 0x7fffffffUL) {
1310 rc = dma_set_mask_and_coherent(&pci_dev->dev, dma_mask);
1311 if (rc == 0)
1312 break;
1313 dma_mask >>= 1;
1314 }
1315 if (rc) {
1316 netif_err(efx, probe, efx->net_dev,
1317 "could not find a suitable DMA mask\n");
1318 goto fail2;
1319 }
1320 netif_dbg(efx, probe, efx->net_dev,
1321 "using DMA mask %llx\n", (unsigned long long) dma_mask);
1322
1323 efx->membase_phys = pci_resource_start(efx->pci_dev, bar);
1324 rc = pci_request_region(pci_dev, bar, "sfc");
1325 if (rc) {
1326 netif_err(efx, probe, efx->net_dev,
1327 "request for memory BAR failed\n");
1328 rc = -EIO;
1329 goto fail3;
1330 }
1331 efx->membase = ioremap_nocache(efx->membase_phys, mem_map_size);
1332 if (!efx->membase) {
1333 netif_err(efx, probe, efx->net_dev,
1334 "could not map memory BAR at %llx+%x\n",
1335 (unsigned long long)efx->membase_phys, mem_map_size);
1336 rc = -ENOMEM;
1337 goto fail4;
1338 }
1339 netif_dbg(efx, probe, efx->net_dev,
1340 "memory BAR at %llx+%x (virtual %p)\n",
1341 (unsigned long long)efx->membase_phys, mem_map_size,
1342 efx->membase);
1343
1344 return 0;
1345
1346 fail4:
1347 pci_release_region(efx->pci_dev, bar);
1348 fail3:
1349 efx->membase_phys = 0;
1350 fail2:
1351 pci_disable_device(efx->pci_dev);
1352 fail1:
1353 return rc;
1354 }
1355
1356 static void efx_fini_io(struct efx_nic *efx)
1357 {
1358 int bar;
1359
1360 netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
1361
1362 if (efx->membase) {
1363 iounmap(efx->membase);
1364 efx->membase = NULL;
1365 }
1366
1367 if (efx->membase_phys) {
1368 bar = efx->type->mem_bar(efx);
1369 pci_release_region(efx->pci_dev, bar);
1370 efx->membase_phys = 0;
1371 }
1372
1373 /* Don't disable bus-mastering if VFs are assigned */
1374 if (!pci_vfs_assigned(efx->pci_dev))
1375 pci_disable_device(efx->pci_dev);
1376 }
1377
1378 void efx_set_default_rx_indir_table(struct efx_nic *efx,
1379 struct efx_rss_context *ctx)
1380 {
1381 size_t i;
1382
1383 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
1384 ctx->rx_indir_table[i] =
1385 ethtool_rxfh_indir_default(i, efx->rss_spread);
1386 }
1387
1388 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
1389 {
1390 cpumask_var_t thread_mask;
1391 unsigned int count;
1392 int cpu;
1393
1394 if (rss_cpus) {
1395 count = rss_cpus;
1396 } else {
1397 if (unlikely(!zalloc_cpumask_var(&thread_mask, GFP_KERNEL))) {
1398 netif_warn(efx, probe, efx->net_dev,
1399 "RSS disabled due to allocation failure\n");
1400 return 1;
1401 }
1402
1403 count = 0;
1404 for_each_online_cpu(cpu) {
1405 if (!cpumask_test_cpu(cpu, thread_mask)) {
1406 ++count;
1407 cpumask_or(thread_mask, thread_mask,
1408 topology_sibling_cpumask(cpu));
1409 }
1410 }
1411
1412 free_cpumask_var(thread_mask);
1413 }
1414
1415 if (count > EFX_MAX_RX_QUEUES) {
1416 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
1417 "Reducing number of rx queues from %u to %u.\n",
1418 count, EFX_MAX_RX_QUEUES);
1419 count = EFX_MAX_RX_QUEUES;
1420 }
1421
1422 /* If RSS is requested for the PF *and* VFs then we can't write RSS
1423 * table entries that are inaccessible to VFs
1424 */
1425 #ifdef CONFIG_SFC_SRIOV
1426 if (efx->type->sriov_wanted) {
1427 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
1428 count > efx_vf_size(efx)) {
1429 netif_warn(efx, probe, efx->net_dev,
1430 "Reducing number of RSS channels from %u to %u for "
1431 "VF support. Increase vf-msix-limit to use more "
1432 "channels on the PF.\n",
1433 count, efx_vf_size(efx));
1434 count = efx_vf_size(efx);
1435 }
1436 }
1437 #endif
1438
1439 return count;
1440 }
1441
1442 /* Probe the number and type of interrupts we are able to obtain, and
1443 * the resulting numbers of channels and RX queues.
1444 */
1445 static int efx_probe_interrupts(struct efx_nic *efx)
1446 {
1447 unsigned int extra_channels = 0;
1448 unsigned int i, j;
1449 int rc;
1450
1451 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
1452 if (efx->extra_channel_type[i])
1453 ++extra_channels;
1454
1455 if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
1456 struct msix_entry xentries[EFX_MAX_CHANNELS];
1457 unsigned int n_channels;
1458
1459 n_channels = efx_wanted_parallelism(efx);
1460 if (efx_separate_tx_channels)
1461 n_channels *= 2;
1462 n_channels += extra_channels;
1463 n_channels = min(n_channels, efx->max_channels);
1464
1465 for (i = 0; i < n_channels; i++)
1466 xentries[i].entry = i;
1467 rc = pci_enable_msix_range(efx->pci_dev,
1468 xentries, 1, n_channels);
1469 if (rc < 0) {
1470 /* Fall back to single channel MSI */
1471 netif_err(efx, drv, efx->net_dev,
1472 "could not enable MSI-X\n");
1473 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
1474 efx->interrupt_mode = EFX_INT_MODE_MSI;
1475 else
1476 return rc;
1477 } else if (rc < n_channels) {
1478 netif_err(efx, drv, efx->net_dev,
1479 "WARNING: Insufficient MSI-X vectors"
1480 " available (%d < %u).\n", rc, n_channels);
1481 netif_err(efx, drv, efx->net_dev,
1482 "WARNING: Performance may be reduced.\n");
1483 n_channels = rc;
1484 }
1485
1486 if (rc > 0) {
1487 efx->n_channels = n_channels;
1488 if (n_channels > extra_channels)
1489 n_channels -= extra_channels;
1490 if (efx_separate_tx_channels) {
1491 efx->n_tx_channels = min(max(n_channels / 2,
1492 1U),
1493 efx->max_tx_channels);
1494 efx->n_rx_channels = max(n_channels -
1495 efx->n_tx_channels,
1496 1U);
1497 } else {
1498 efx->n_tx_channels = min(n_channels,
1499 efx->max_tx_channels);
1500 efx->n_rx_channels = n_channels;
1501 }
1502 for (i = 0; i < efx->n_channels; i++)
1503 efx_get_channel(efx, i)->irq =
1504 xentries[i].vector;
1505 }
1506 }
1507
1508 /* Try single interrupt MSI */
1509 if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
1510 efx->n_channels = 1;
1511 efx->n_rx_channels = 1;
1512 efx->n_tx_channels = 1;
1513 rc = pci_enable_msi(efx->pci_dev);
1514 if (rc == 0) {
1515 efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1516 } else {
1517 netif_err(efx, drv, efx->net_dev,
1518 "could not enable MSI\n");
1519 if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
1520 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
1521 else
1522 return rc;
1523 }
1524 }
1525
1526 /* Assume legacy interrupts */
1527 if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1528 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
1529 efx->n_rx_channels = 1;
1530 efx->n_tx_channels = 1;
1531 efx->legacy_irq = efx->pci_dev->irq;
1532 }
1533
1534 /* Assign extra channels if possible */
1535 efx->n_extra_tx_channels = 0;
1536 j = efx->n_channels;
1537 for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
1538 if (!efx->extra_channel_type[i])
1539 continue;
1540 if (efx->interrupt_mode != EFX_INT_MODE_MSIX ||
1541 efx->n_channels <= extra_channels) {
1542 efx->extra_channel_type[i]->handle_no_channel(efx);
1543 } else {
1544 --j;
1545 efx_get_channel(efx, j)->type =
1546 efx->extra_channel_type[i];
1547 if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
1548 efx->n_extra_tx_channels++;
1549 }
1550 }
1551
1552 /* RSS might be usable on VFs even if it is disabled on the PF */
1553 #ifdef CONFIG_SFC_SRIOV
1554 if (efx->type->sriov_wanted) {
1555 efx->rss_spread = ((efx->n_rx_channels > 1 ||
1556 !efx->type->sriov_wanted(efx)) ?
1557 efx->n_rx_channels : efx_vf_size(efx));
1558 return 0;
1559 }
1560 #endif
1561 efx->rss_spread = efx->n_rx_channels;
1562
1563 return 0;
1564 }
1565
1566 #if defined(CONFIG_SMP)
1567 static void efx_set_interrupt_affinity(struct efx_nic *efx)
1568 {
1569 struct efx_channel *channel;
1570 unsigned int cpu;
1571
1572 efx_for_each_channel(channel, efx) {
1573 cpu = cpumask_local_spread(channel->channel,
1574 pcibus_to_node(efx->pci_dev->bus));
1575 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
1576 }
1577 }
1578
1579 static void efx_clear_interrupt_affinity(struct efx_nic *efx)
1580 {
1581 struct efx_channel *channel;
1582
1583 efx_for_each_channel(channel, efx)
1584 irq_set_affinity_hint(channel->irq, NULL);
1585 }
1586 #else
1587 static void
1588 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
1589 {
1590 }
1591
1592 static void
1593 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
1594 {
1595 }
1596 #endif /* CONFIG_SMP */
1597
1598 static int efx_soft_enable_interrupts(struct efx_nic *efx)
1599 {
1600 struct efx_channel *channel, *end_channel;
1601 int rc;
1602
1603 BUG_ON(efx->state == STATE_DISABLED);
1604
1605 efx->irq_soft_enabled = true;
1606 smp_wmb();
1607
1608 efx_for_each_channel(channel, efx) {
1609 if (!channel->type->keep_eventq) {
1610 rc = efx_init_eventq(channel);
1611 if (rc)
1612 goto fail;
1613 }
1614 efx_start_eventq(channel);
1615 }
1616
1617 efx_mcdi_mode_event(efx);
1618
1619 return 0;
1620 fail:
1621 end_channel = channel;
1622 efx_for_each_channel(channel, efx) {
1623 if (channel == end_channel)
1624 break;
1625 efx_stop_eventq(channel);
1626 if (!channel->type->keep_eventq)
1627 efx_fini_eventq(channel);
1628 }
1629
1630 return rc;
1631 }
1632
1633 static void efx_soft_disable_interrupts(struct efx_nic *efx)
1634 {
1635 struct efx_channel *channel;
1636
1637 if (efx->state == STATE_DISABLED)
1638 return;
1639
1640 efx_mcdi_mode_poll(efx);
1641
1642 efx->irq_soft_enabled = false;
1643 smp_wmb();
1644
1645 if (efx->legacy_irq)
1646 synchronize_irq(efx->legacy_irq);
1647
1648 efx_for_each_channel(channel, efx) {
1649 if (channel->irq)
1650 synchronize_irq(channel->irq);
1651
1652 efx_stop_eventq(channel);
1653 if (!channel->type->keep_eventq)
1654 efx_fini_eventq(channel);
1655 }
1656
1657 /* Flush the asynchronous MCDI request queue */
1658 efx_mcdi_flush_async(efx);
1659 }
1660
1661 static int efx_enable_interrupts(struct efx_nic *efx)
1662 {
1663 struct efx_channel *channel, *end_channel;
1664 int rc;
1665
1666 BUG_ON(efx->state == STATE_DISABLED);
1667
1668 if (efx->eeh_disabled_legacy_irq) {
1669 enable_irq(efx->legacy_irq);
1670 efx->eeh_disabled_legacy_irq = false;
1671 }
1672
1673 efx->type->irq_enable_master(efx);
1674
1675 efx_for_each_channel(channel, efx) {
1676 if (channel->type->keep_eventq) {
1677 rc = efx_init_eventq(channel);
1678 if (rc)
1679 goto fail;
1680 }
1681 }
1682
1683 rc = efx_soft_enable_interrupts(efx);
1684 if (rc)
1685 goto fail;
1686
1687 return 0;
1688
1689 fail:
1690 end_channel = channel;
1691 efx_for_each_channel(channel, efx) {
1692 if (channel == end_channel)
1693 break;
1694 if (channel->type->keep_eventq)
1695 efx_fini_eventq(channel);
1696 }
1697
1698 efx->type->irq_disable_non_ev(efx);
1699
1700 return rc;
1701 }
1702
1703 static void efx_disable_interrupts(struct efx_nic *efx)
1704 {
1705 struct efx_channel *channel;
1706
1707 efx_soft_disable_interrupts(efx);
1708
1709 efx_for_each_channel(channel, efx) {
1710 if (channel->type->keep_eventq)
1711 efx_fini_eventq(channel);
1712 }
1713
1714 efx->type->irq_disable_non_ev(efx);
1715 }
1716
1717 static void efx_remove_interrupts(struct efx_nic *efx)
1718 {
1719 struct efx_channel *channel;
1720
1721 /* Remove MSI/MSI-X interrupts */
1722 efx_for_each_channel(channel, efx)
1723 channel->irq = 0;
1724 pci_disable_msi(efx->pci_dev);
1725 pci_disable_msix(efx->pci_dev);
1726
1727 /* Remove legacy interrupt */
1728 efx->legacy_irq = 0;
1729 }
1730
1731 static void efx_set_channels(struct efx_nic *efx)
1732 {
1733 struct efx_channel *channel;
1734 struct efx_tx_queue *tx_queue;
1735
1736 efx->tx_channel_offset =
1737 efx_separate_tx_channels ?
1738 efx->n_channels - efx->n_tx_channels : 0;
1739
1740 /* We need to mark which channels really have RX and TX
1741 * queues, and adjust the TX queue numbers if we have separate
1742 * RX-only and TX-only channels.
1743 */
1744 efx_for_each_channel(channel, efx) {
1745 if (channel->channel < efx->n_rx_channels)
1746 channel->rx_queue.core_index = channel->channel;
1747 else
1748 channel->rx_queue.core_index = -1;
1749
1750 efx_for_each_channel_tx_queue(tx_queue, channel)
1751 tx_queue->queue -= (efx->tx_channel_offset *
1752 EFX_TXQ_TYPES);
1753 }
1754 }
1755
1756 static int efx_probe_nic(struct efx_nic *efx)
1757 {
1758 int rc;
1759
1760 netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1761
1762 /* Carry out hardware-type specific initialisation */
1763 rc = efx->type->probe(efx);
1764 if (rc)
1765 return rc;
1766
1767 do {
1768 if (!efx->max_channels || !efx->max_tx_channels) {
1769 netif_err(efx, drv, efx->net_dev,
1770 "Insufficient resources to allocate"
1771 " any channels\n");
1772 rc = -ENOSPC;
1773 goto fail1;
1774 }
1775
1776 /* Determine the number of channels and queues by trying
1777 * to hook in MSI-X interrupts.
1778 */
1779 rc = efx_probe_interrupts(efx);
1780 if (rc)
1781 goto fail1;
1782
1783 efx_set_channels(efx);
1784
1785 /* dimension_resources can fail with EAGAIN */
1786 rc = efx->type->dimension_resources(efx);
1787 if (rc != 0 && rc != -EAGAIN)
1788 goto fail2;
1789
1790 if (rc == -EAGAIN)
1791 /* try again with new max_channels */
1792 efx_remove_interrupts(efx);
1793
1794 } while (rc == -EAGAIN);
1795
1796 if (efx->n_channels > 1)
1797 netdev_rss_key_fill(efx->rss_context.rx_hash_key,
1798 sizeof(efx->rss_context.rx_hash_key));
1799 efx_set_default_rx_indir_table(efx, &efx->rss_context);
1800
1801 netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1802 netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1803
1804 /* Initialise the interrupt moderation settings */
1805 efx->irq_mod_step_us = DIV_ROUND_UP(efx->timer_quantum_ns, 1000);
1806 efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true,
1807 true);
1808
1809 return 0;
1810
1811 fail2:
1812 efx_remove_interrupts(efx);
1813 fail1:
1814 efx->type->remove(efx);
1815 return rc;
1816 }
1817
1818 static void efx_remove_nic(struct efx_nic *efx)
1819 {
1820 netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1821
1822 efx_remove_interrupts(efx);
1823 efx->type->remove(efx);
1824 }
1825
1826 static int efx_probe_filters(struct efx_nic *efx)
1827 {
1828 int rc;
1829
1830 init_rwsem(&efx->filter_sem);
1831 mutex_lock(&efx->mac_lock);
1832 down_write(&efx->filter_sem);
1833 rc = efx->type->filter_table_probe(efx);
1834 if (rc)
1835 goto out_unlock;
1836
1837 #ifdef CONFIG_RFS_ACCEL
1838 if (efx->type->offload_features & NETIF_F_NTUPLE) {
1839 struct efx_channel *channel;
1840 int i, success = 1;
1841
1842 efx_for_each_channel(channel, efx) {
1843 channel->rps_flow_id =
1844 kcalloc(efx->type->max_rx_ip_filters,
1845 sizeof(*channel->rps_flow_id),
1846 GFP_KERNEL);
1847 if (!channel->rps_flow_id)
1848 success = 0;
1849 else
1850 for (i = 0;
1851 i < efx->type->max_rx_ip_filters;
1852 ++i)
1853 channel->rps_flow_id[i] =
1854 RPS_FLOW_ID_INVALID;
1855 }
1856
1857 if (!success) {
1858 efx_for_each_channel(channel, efx)
1859 kfree(channel->rps_flow_id);
1860 efx->type->filter_table_remove(efx);
1861 rc = -ENOMEM;
1862 goto out_unlock;
1863 }
1864
1865 efx->rps_expire_index = efx->rps_expire_channel = 0;
1866 }
1867 #endif
1868 out_unlock:
1869 up_write(&efx->filter_sem);
1870 mutex_unlock(&efx->mac_lock);
1871 return rc;
1872 }
1873
1874 static void efx_remove_filters(struct efx_nic *efx)
1875 {
1876 #ifdef CONFIG_RFS_ACCEL
1877 struct efx_channel *channel;
1878
1879 efx_for_each_channel(channel, efx)
1880 kfree(channel->rps_flow_id);
1881 #endif
1882 down_write(&efx->filter_sem);
1883 efx->type->filter_table_remove(efx);
1884 up_write(&efx->filter_sem);
1885 }
1886
1887
1888 /**************************************************************************
1889 *
1890 * NIC startup/shutdown
1891 *
1892 *************************************************************************/
1893
1894 static int efx_probe_all(struct efx_nic *efx)
1895 {
1896 int rc;
1897
1898 rc = efx_probe_nic(efx);
1899 if (rc) {
1900 netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
1901 goto fail1;
1902 }
1903
1904 rc = efx_probe_port(efx);
1905 if (rc) {
1906 netif_err(efx, probe, efx->net_dev, "failed to create port\n");
1907 goto fail2;
1908 }
1909
1910 BUILD_BUG_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_RXQ_MIN_ENT);
1911 if (WARN_ON(EFX_DEFAULT_DMAQ_SIZE < EFX_TXQ_MIN_ENT(efx))) {
1912 rc = -EINVAL;
1913 goto fail3;
1914 }
1915 efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
1916
1917 #ifdef CONFIG_SFC_SRIOV
1918 rc = efx->type->vswitching_probe(efx);
1919 if (rc) /* not fatal; the PF will still work fine */
1920 netif_warn(efx, probe, efx->net_dev,
1921 "failed to setup vswitching rc=%d;"
1922 " VFs may not function\n", rc);
1923 #endif
1924
1925 rc = efx_probe_filters(efx);
1926 if (rc) {
1927 netif_err(efx, probe, efx->net_dev,
1928 "failed to create filter tables\n");
1929 goto fail4;
1930 }
1931
1932 rc = efx_probe_channels(efx);
1933 if (rc)
1934 goto fail5;
1935
1936 return 0;
1937
1938 fail5:
1939 efx_remove_filters(efx);
1940 fail4:
1941 #ifdef CONFIG_SFC_SRIOV
1942 efx->type->vswitching_remove(efx);
1943 #endif
1944 fail3:
1945 efx_remove_port(efx);
1946 fail2:
1947 efx_remove_nic(efx);
1948 fail1:
1949 return rc;
1950 }
1951
1952 /* If the interface is supposed to be running but is not, start
1953 * the hardware and software data path, regular activity for the port
1954 * (MAC statistics, link polling, etc.) and schedule the port to be
1955 * reconfigured. Interrupts must already be enabled. This function
1956 * is safe to call multiple times, so long as the NIC is not disabled.
1957 * Requires the RTNL lock.
1958 */
1959 static void efx_start_all(struct efx_nic *efx)
1960 {
1961 EFX_ASSERT_RESET_SERIALISED(efx);
1962 BUG_ON(efx->state == STATE_DISABLED);
1963
1964 /* Check that it is appropriate to restart the interface. All
1965 * of these flags are safe to read under just the rtnl lock */
1966 if (efx->port_enabled || !netif_running(efx->net_dev) ||
1967 efx->reset_pending)
1968 return;
1969
1970 efx_start_port(efx);
1971 efx_start_datapath(efx);
1972
1973 /* Start the hardware monitor if there is one */
1974 if (efx->type->monitor != NULL)
1975 queue_delayed_work(efx->workqueue, &efx->monitor_work,
1976 efx_monitor_interval);
1977
1978 /* Link state detection is normally event-driven; we have
1979 * to poll now because we could have missed a change
1980 */
1981 mutex_lock(&efx->mac_lock);
1982 if (efx->phy_op->poll(efx))
1983 efx_link_status_changed(efx);
1984 mutex_unlock(&efx->mac_lock);
1985
1986 efx->type->start_stats(efx);
1987 efx->type->pull_stats(efx);
1988 spin_lock_bh(&efx->stats_lock);
1989 efx->type->update_stats(efx, NULL, NULL);
1990 spin_unlock_bh(&efx->stats_lock);
1991 }
1992
1993 /* Quiesce the hardware and software data path, and regular activity
1994 * for the port without bringing the link down. Safe to call multiple
1995 * times with the NIC in almost any state, but interrupts should be
1996 * enabled. Requires the RTNL lock.
1997 */
1998 static void efx_stop_all(struct efx_nic *efx)
1999 {
2000 EFX_ASSERT_RESET_SERIALISED(efx);
2001
2002 /* port_enabled can be read safely under the rtnl lock */
2003 if (!efx->port_enabled)
2004 return;
2005
2006 /* update stats before we go down so we can accurately count
2007 * rx_nodesc_drops
2008 */
2009 efx->type->pull_stats(efx);
2010 spin_lock_bh(&efx->stats_lock);
2011 efx->type->update_stats(efx, NULL, NULL);
2012 spin_unlock_bh(&efx->stats_lock);
2013 efx->type->stop_stats(efx);
2014 efx_stop_port(efx);
2015
2016 /* Stop the kernel transmit interface. This is only valid if
2017 * the device is stopped or detached; otherwise the watchdog
2018 * may fire immediately.
2019 */
2020 WARN_ON(netif_running(efx->net_dev) &&
2021 netif_device_present(efx->net_dev));
2022 netif_tx_disable(efx->net_dev);
2023
2024 efx_stop_datapath(efx);
2025 }
2026
2027 static void efx_remove_all(struct efx_nic *efx)
2028 {
2029 efx_remove_channels(efx);
2030 efx_remove_filters(efx);
2031 #ifdef CONFIG_SFC_SRIOV
2032 efx->type->vswitching_remove(efx);
2033 #endif
2034 efx_remove_port(efx);
2035 efx_remove_nic(efx);
2036 }
2037
2038 /**************************************************************************
2039 *
2040 * Interrupt moderation
2041 *
2042 **************************************************************************/
2043 unsigned int efx_usecs_to_ticks(struct efx_nic *efx, unsigned int usecs)
2044 {
2045 if (usecs == 0)
2046 return 0;
2047 if (usecs * 1000 < efx->timer_quantum_ns)
2048 return 1; /* never round down to 0 */
2049 return usecs * 1000 / efx->timer_quantum_ns;
2050 }
2051
2052 unsigned int efx_ticks_to_usecs(struct efx_nic *efx, unsigned int ticks)
2053 {
2054 /* We must round up when converting ticks to microseconds
2055 * because we round down when converting the other way.
2056 */
2057 return DIV_ROUND_UP(ticks * efx->timer_quantum_ns, 1000);
2058 }
2059
2060 /* Set interrupt moderation parameters */
2061 int efx_init_irq_moderation(struct efx_nic *efx, unsigned int tx_usecs,
2062 unsigned int rx_usecs, bool rx_adaptive,
2063 bool rx_may_override_tx)
2064 {
2065 struct efx_channel *channel;
2066 unsigned int timer_max_us;
2067
2068 EFX_ASSERT_RESET_SERIALISED(efx);
2069
2070 timer_max_us = efx->timer_max_ns / 1000;
2071
2072 if (tx_usecs > timer_max_us || rx_usecs > timer_max_us)
2073 return -EINVAL;
2074
2075 if (tx_usecs != rx_usecs && efx->tx_channel_offset == 0 &&
2076 !rx_may_override_tx) {
2077 netif_err(efx, drv, efx->net_dev, "Channels are shared. "
2078 "RX and TX IRQ moderation must be equal\n");
2079 return -EINVAL;
2080 }
2081
2082 efx->irq_rx_adaptive = rx_adaptive;
2083 efx->irq_rx_moderation_us = rx_usecs;
2084 efx_for_each_channel(channel, efx) {
2085 if (efx_channel_has_rx_queue(channel))
2086 channel->irq_moderation_us = rx_usecs;
2087 else if (efx_channel_has_tx_queues(channel))
2088 channel->irq_moderation_us = tx_usecs;
2089 }
2090
2091 return 0;
2092 }
2093
2094 void efx_get_irq_moderation(struct efx_nic *efx, unsigned int *tx_usecs,
2095 unsigned int *rx_usecs, bool *rx_adaptive)
2096 {
2097 *rx_adaptive = efx->irq_rx_adaptive;
2098 *rx_usecs = efx->irq_rx_moderation_us;
2099
2100 /* If channels are shared between RX and TX, so is IRQ
2101 * moderation. Otherwise, IRQ moderation is the same for all
2102 * TX channels and is not adaptive.
2103 */
2104 if (efx->tx_channel_offset == 0) {
2105 *tx_usecs = *rx_usecs;
2106 } else {
2107 struct efx_channel *tx_channel;
2108
2109 tx_channel = efx->channel[efx->tx_channel_offset];
2110 *tx_usecs = tx_channel->irq_moderation_us;
2111 }
2112 }
2113
2114 /**************************************************************************
2115 *
2116 * Hardware monitor
2117 *
2118 **************************************************************************/
2119
2120 /* Run periodically off the general workqueue */
2121 static void efx_monitor(struct work_struct *data)
2122 {
2123 struct efx_nic *efx = container_of(data, struct efx_nic,
2124 monitor_work.work);
2125
2126 netif_vdbg(efx, timer, efx->net_dev,
2127 "hardware monitor executing on CPU %d\n",
2128 raw_smp_processor_id());
2129 BUG_ON(efx->type->monitor == NULL);
2130
2131 /* If the mac_lock is already held then it is likely a port
2132 * reconfiguration is already in place, which will likely do
2133 * most of the work of monitor() anyway. */
2134 if (mutex_trylock(&efx->mac_lock)) {
2135 if (efx->port_enabled)
2136 efx->type->monitor(efx);
2137 mutex_unlock(&efx->mac_lock);
2138 }
2139
2140 queue_delayed_work(efx->workqueue, &efx->monitor_work,
2141 efx_monitor_interval);
2142 }
2143
2144 /**************************************************************************
2145 *
2146 * ioctls
2147 *
2148 *************************************************************************/
2149
2150 /* Net device ioctl
2151 * Context: process, rtnl_lock() held.
2152 */
2153 static int efx_ioctl(struct net_device *net_dev, struct ifreq *ifr, int cmd)
2154 {
2155 struct efx_nic *efx = netdev_priv(net_dev);
2156 struct mii_ioctl_data *data = if_mii(ifr);
2157
2158 if (cmd == SIOCSHWTSTAMP)
2159 return efx_ptp_set_ts_config(efx, ifr);
2160 if (cmd == SIOCGHWTSTAMP)
2161 return efx_ptp_get_ts_config(efx, ifr);
2162
2163 /* Convert phy_id from older PRTAD/DEVAD format */
2164 if ((cmd == SIOCGMIIREG || cmd == SIOCSMIIREG) &&
2165 (data->phy_id & 0xfc00) == 0x0400)
2166 data->phy_id ^= MDIO_PHY_ID_C45 | 0x0400;
2167
2168 return mdio_mii_ioctl(&efx->mdio, data, cmd);
2169 }
2170
2171 /**************************************************************************
2172 *
2173 * NAPI interface
2174 *
2175 **************************************************************************/
2176
2177 static void efx_init_napi_channel(struct efx_channel *channel)
2178 {
2179 struct efx_nic *efx = channel->efx;
2180
2181 channel->napi_dev = efx->net_dev;
2182 netif_napi_add(channel->napi_dev, &channel->napi_str,
2183 efx_poll, napi_weight);
2184 }
2185
2186 static void efx_init_napi(struct efx_nic *efx)
2187 {
2188 struct efx_channel *channel;
2189
2190 efx_for_each_channel(channel, efx)
2191 efx_init_napi_channel(channel);
2192 }
2193
2194 static void efx_fini_napi_channel(struct efx_channel *channel)
2195 {
2196 if (channel->napi_dev)
2197 netif_napi_del(&channel->napi_str);
2198
2199 channel->napi_dev = NULL;
2200 }
2201
2202 static void efx_fini_napi(struct efx_nic *efx)
2203 {
2204 struct efx_channel *channel;
2205
2206 efx_for_each_channel(channel, efx)
2207 efx_fini_napi_channel(channel);
2208 }
2209
2210 /**************************************************************************
2211 *
2212 * Kernel net device interface
2213 *
2214 *************************************************************************/
2215
2216 /* Context: process, rtnl_lock() held. */
2217 int efx_net_open(struct net_device *net_dev)
2218 {
2219 struct efx_nic *efx = netdev_priv(net_dev);
2220 int rc;
2221
2222 netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
2223 raw_smp_processor_id());
2224
2225 rc = efx_check_disabled(efx);
2226 if (rc)
2227 return rc;
2228 if (efx->phy_mode & PHY_MODE_SPECIAL)
2229 return -EBUSY;
2230 if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
2231 return -EIO;
2232
2233 /* Notify the kernel of the link state polled during driver load,
2234 * before the monitor starts running */
2235 efx_link_status_changed(efx);
2236
2237 efx_start_all(efx);
2238 if (efx->state == STATE_DISABLED || efx->reset_pending)
2239 netif_device_detach(efx->net_dev);
2240 efx_selftest_async_start(efx);
2241 return 0;
2242 }
2243
2244 /* Context: process, rtnl_lock() held.
2245 * Note that the kernel will ignore our return code; this method
2246 * should really be a void.
2247 */
2248 int efx_net_stop(struct net_device *net_dev)
2249 {
2250 struct efx_nic *efx = netdev_priv(net_dev);
2251
2252 netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
2253 raw_smp_processor_id());
2254
2255 /* Stop the device and flush all the channels */
2256 efx_stop_all(efx);
2257
2258 return 0;
2259 }
2260
2261 /* Context: process, dev_base_lock or RTNL held, non-blocking. */
2262 static void efx_net_stats(struct net_device *net_dev,
2263 struct rtnl_link_stats64 *stats)
2264 {
2265 struct efx_nic *efx = netdev_priv(net_dev);
2266
2267 spin_lock_bh(&efx->stats_lock);
2268 efx->type->update_stats(efx, NULL, stats);
2269 spin_unlock_bh(&efx->stats_lock);
2270 }
2271
2272 /* Context: netif_tx_lock held, BHs disabled. */
2273 static void efx_watchdog(struct net_device *net_dev)
2274 {
2275 struct efx_nic *efx = netdev_priv(net_dev);
2276
2277 netif_err(efx, tx_err, efx->net_dev,
2278 "TX stuck with port_enabled=%d: resetting channels\n",
2279 efx->port_enabled);
2280
2281 efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
2282 }
2283
2284
2285 /* Context: process, rtnl_lock() held. */
2286 static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
2287 {
2288 struct efx_nic *efx = netdev_priv(net_dev);
2289 int rc;
2290
2291 rc = efx_check_disabled(efx);
2292 if (rc)
2293 return rc;
2294
2295 netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
2296
2297 efx_device_detach_sync(efx);
2298 efx_stop_all(efx);
2299
2300 mutex_lock(&efx->mac_lock);
2301 net_dev->mtu = new_mtu;
2302 efx_mac_reconfigure(efx);
2303 mutex_unlock(&efx->mac_lock);
2304
2305 efx_start_all(efx);
2306 efx_device_attach_if_not_resetting(efx);
2307 return 0;
2308 }
2309
2310 static int efx_set_mac_address(struct net_device *net_dev, void *data)
2311 {
2312 struct efx_nic *efx = netdev_priv(net_dev);
2313 struct sockaddr *addr = data;
2314 u8 *new_addr = addr->sa_data;
2315 u8 old_addr[6];
2316 int rc;
2317
2318 if (!is_valid_ether_addr(new_addr)) {
2319 netif_err(efx, drv, efx->net_dev,
2320 "invalid ethernet MAC address requested: %pM\n",
2321 new_addr);
2322 return -EADDRNOTAVAIL;
2323 }
2324
2325 /* save old address */
2326 ether_addr_copy(old_addr, net_dev->dev_addr);
2327 ether_addr_copy(net_dev->dev_addr, new_addr);
2328 if (efx->type->set_mac_address) {
2329 rc = efx->type->set_mac_address(efx);
2330 if (rc) {
2331 ether_addr_copy(net_dev->dev_addr, old_addr);
2332 return rc;
2333 }
2334 }
2335
2336 /* Reconfigure the MAC */
2337 mutex_lock(&efx->mac_lock);
2338 efx_mac_reconfigure(efx);
2339 mutex_unlock(&efx->mac_lock);
2340
2341 return 0;
2342 }
2343
2344 /* Context: netif_addr_lock held, BHs disabled. */
2345 static void efx_set_rx_mode(struct net_device *net_dev)
2346 {
2347 struct efx_nic *efx = netdev_priv(net_dev);
2348
2349 if (efx->port_enabled)
2350 queue_work(efx->workqueue, &efx->mac_work);
2351 /* Otherwise efx_start_port() will do this */
2352 }
2353
2354 static int efx_set_features(struct net_device *net_dev, netdev_features_t data)
2355 {
2356 struct efx_nic *efx = netdev_priv(net_dev);
2357 int rc;
2358
2359 /* If disabling RX n-tuple filtering, clear existing filters */
2360 if (net_dev->features & ~data & NETIF_F_NTUPLE) {
2361 rc = efx->type->filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
2362 if (rc)
2363 return rc;
2364 }
2365
2366 /* If Rx VLAN filter is changed, update filters via mac_reconfigure.
2367 * If rx-fcs is changed, mac_reconfigure updates that too.
2368 */
2369 if ((net_dev->features ^ data) & (NETIF_F_HW_VLAN_CTAG_FILTER |
2370 NETIF_F_RXFCS)) {
2371 /* efx_set_rx_mode() will schedule MAC work to update filters
2372 * when a new features are finally set in net_dev.
2373 */
2374 efx_set_rx_mode(net_dev);
2375 }
2376
2377 return 0;
2378 }
2379
2380 static int efx_get_phys_port_id(struct net_device *net_dev,
2381 struct netdev_phys_item_id *ppid)
2382 {
2383 struct efx_nic *efx = netdev_priv(net_dev);
2384
2385 if (efx->type->get_phys_port_id)
2386 return efx->type->get_phys_port_id(efx, ppid);
2387 else
2388 return -EOPNOTSUPP;
2389 }
2390
2391 static int efx_get_phys_port_name(struct net_device *net_dev,
2392 char *name, size_t len)
2393 {
2394 struct efx_nic *efx = netdev_priv(net_dev);
2395
2396 if (snprintf(name, len, "p%u", efx->port_num) >= len)
2397 return -EINVAL;
2398 return 0;
2399 }
2400
2401 static int efx_vlan_rx_add_vid(struct net_device *net_dev, __be16 proto, u16 vid)
2402 {
2403 struct efx_nic *efx = netdev_priv(net_dev);
2404
2405 if (efx->type->vlan_rx_add_vid)
2406 return efx->type->vlan_rx_add_vid(efx, proto, vid);
2407 else
2408 return -EOPNOTSUPP;
2409 }
2410
2411 static int efx_vlan_rx_kill_vid(struct net_device *net_dev, __be16 proto, u16 vid)
2412 {
2413 struct efx_nic *efx = netdev_priv(net_dev);
2414
2415 if (efx->type->vlan_rx_kill_vid)
2416 return efx->type->vlan_rx_kill_vid(efx, proto, vid);
2417 else
2418 return -EOPNOTSUPP;
2419 }
2420
2421 static int efx_udp_tunnel_type_map(enum udp_parsable_tunnel_type in)
2422 {
2423 switch (in) {
2424 case UDP_TUNNEL_TYPE_VXLAN:
2425 return TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN;
2426 case UDP_TUNNEL_TYPE_GENEVE:
2427 return TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE;
2428 default:
2429 return -1;
2430 }
2431 }
2432
2433 static void efx_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti)
2434 {
2435 struct efx_nic *efx = netdev_priv(dev);
2436 struct efx_udp_tunnel tnl;
2437 int efx_tunnel_type;
2438
2439 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2440 if (efx_tunnel_type < 0)
2441 return;
2442
2443 tnl.type = (u16)efx_tunnel_type;
2444 tnl.port = ti->port;
2445
2446 if (efx->type->udp_tnl_add_port)
2447 (void)efx->type->udp_tnl_add_port(efx, tnl);
2448 }
2449
2450 static void efx_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti)
2451 {
2452 struct efx_nic *efx = netdev_priv(dev);
2453 struct efx_udp_tunnel tnl;
2454 int efx_tunnel_type;
2455
2456 efx_tunnel_type = efx_udp_tunnel_type_map(ti->type);
2457 if (efx_tunnel_type < 0)
2458 return;
2459
2460 tnl.type = (u16)efx_tunnel_type;
2461 tnl.port = ti->port;
2462
2463 if (efx->type->udp_tnl_del_port)
2464 (void)efx->type->udp_tnl_del_port(efx, tnl);
2465 }
2466
2467 static const struct net_device_ops efx_netdev_ops = {
2468 .ndo_open = efx_net_open,
2469 .ndo_stop = efx_net_stop,
2470 .ndo_get_stats64 = efx_net_stats,
2471 .ndo_tx_timeout = efx_watchdog,
2472 .ndo_start_xmit = efx_hard_start_xmit,
2473 .ndo_validate_addr = eth_validate_addr,
2474 .ndo_do_ioctl = efx_ioctl,
2475 .ndo_change_mtu = efx_change_mtu,
2476 .ndo_set_mac_address = efx_set_mac_address,
2477 .ndo_set_rx_mode = efx_set_rx_mode,
2478 .ndo_set_features = efx_set_features,
2479 .ndo_vlan_rx_add_vid = efx_vlan_rx_add_vid,
2480 .ndo_vlan_rx_kill_vid = efx_vlan_rx_kill_vid,
2481 #ifdef CONFIG_SFC_SRIOV
2482 .ndo_set_vf_mac = efx_sriov_set_vf_mac,
2483 .ndo_set_vf_vlan = efx_sriov_set_vf_vlan,
2484 .ndo_set_vf_spoofchk = efx_sriov_set_vf_spoofchk,
2485 .ndo_get_vf_config = efx_sriov_get_vf_config,
2486 .ndo_set_vf_link_state = efx_sriov_set_vf_link_state,
2487 #endif
2488 .ndo_get_phys_port_id = efx_get_phys_port_id,
2489 .ndo_get_phys_port_name = efx_get_phys_port_name,
2490 .ndo_setup_tc = efx_setup_tc,
2491 #ifdef CONFIG_RFS_ACCEL
2492 .ndo_rx_flow_steer = efx_filter_rfs,
2493 #endif
2494 .ndo_udp_tunnel_add = efx_udp_tunnel_add,
2495 .ndo_udp_tunnel_del = efx_udp_tunnel_del,
2496 };
2497
2498 static void efx_update_name(struct efx_nic *efx)
2499 {
2500 strcpy(efx->name, efx->net_dev->name);
2501 efx_mtd_rename(efx);
2502 efx_set_channel_names(efx);
2503 }
2504
2505 static int efx_netdev_event(struct notifier_block *this,
2506 unsigned long event, void *ptr)
2507 {
2508 struct net_device *net_dev = netdev_notifier_info_to_dev(ptr);
2509
2510 if ((net_dev->netdev_ops == &efx_netdev_ops) &&
2511 event == NETDEV_CHANGENAME)
2512 efx_update_name(netdev_priv(net_dev));
2513
2514 return NOTIFY_DONE;
2515 }
2516
2517 static struct notifier_block efx_netdev_notifier = {
2518 .notifier_call = efx_netdev_event,
2519 };
2520
2521 static ssize_t
2522 show_phy_type(struct device *dev, struct device_attribute *attr, char *buf)
2523 {
2524 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2525 return sprintf(buf, "%d\n", efx->phy_type);
2526 }
2527 static DEVICE_ATTR(phy_type, 0444, show_phy_type, NULL);
2528
2529 #ifdef CONFIG_SFC_MCDI_LOGGING
2530 static ssize_t show_mcdi_log(struct device *dev, struct device_attribute *attr,
2531 char *buf)
2532 {
2533 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2534 struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2535
2536 return scnprintf(buf, PAGE_SIZE, "%d\n", mcdi->logging_enabled);
2537 }
2538 static ssize_t set_mcdi_log(struct device *dev, struct device_attribute *attr,
2539 const char *buf, size_t count)
2540 {
2541 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2542 struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
2543 bool enable = count > 0 && *buf != '0';
2544
2545 mcdi->logging_enabled = enable;
2546 return count;
2547 }
2548 static DEVICE_ATTR(mcdi_logging, 0644, show_mcdi_log, set_mcdi_log);
2549 #endif
2550
2551 static int efx_register_netdev(struct efx_nic *efx)
2552 {
2553 struct net_device *net_dev = efx->net_dev;
2554 struct efx_channel *channel;
2555 int rc;
2556
2557 net_dev->watchdog_timeo = 5 * HZ;
2558 net_dev->irq = efx->pci_dev->irq;
2559 net_dev->netdev_ops = &efx_netdev_ops;
2560 if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
2561 net_dev->priv_flags |= IFF_UNICAST_FLT;
2562 net_dev->ethtool_ops = &efx_ethtool_ops;
2563 net_dev->gso_max_segs = EFX_TSO_MAX_SEGS;
2564 net_dev->min_mtu = EFX_MIN_MTU;
2565 net_dev->max_mtu = EFX_MAX_MTU;
2566
2567 rtnl_lock();
2568
2569 /* Enable resets to be scheduled and check whether any were
2570 * already requested. If so, the NIC is probably hosed so we
2571 * abort.
2572 */
2573 efx->state = STATE_READY;
2574 smp_mb(); /* ensure we change state before checking reset_pending */
2575 if (efx->reset_pending) {
2576 netif_err(efx, probe, efx->net_dev,
2577 "aborting probe due to scheduled reset\n");
2578 rc = -EIO;
2579 goto fail_locked;
2580 }
2581
2582 rc = dev_alloc_name(net_dev, net_dev->name);
2583 if (rc < 0)
2584 goto fail_locked;
2585 efx_update_name(efx);
2586
2587 /* Always start with carrier off; PHY events will detect the link */
2588 netif_carrier_off(net_dev);
2589
2590 rc = register_netdevice(net_dev);
2591 if (rc)
2592 goto fail_locked;
2593
2594 efx_for_each_channel(channel, efx) {
2595 struct efx_tx_queue *tx_queue;
2596 efx_for_each_channel_tx_queue(tx_queue, channel)
2597 efx_init_tx_queue_core_txq(tx_queue);
2598 }
2599
2600 efx_associate(efx);
2601
2602 rtnl_unlock();
2603
2604 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2605 if (rc) {
2606 netif_err(efx, drv, efx->net_dev,
2607 "failed to init net dev attributes\n");
2608 goto fail_registered;
2609 }
2610 #ifdef CONFIG_SFC_MCDI_LOGGING
2611 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2612 if (rc) {
2613 netif_err(efx, drv, efx->net_dev,
2614 "failed to init net dev attributes\n");
2615 goto fail_attr_mcdi_logging;
2616 }
2617 #endif
2618
2619 return 0;
2620
2621 #ifdef CONFIG_SFC_MCDI_LOGGING
2622 fail_attr_mcdi_logging:
2623 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2624 #endif
2625 fail_registered:
2626 rtnl_lock();
2627 efx_dissociate(efx);
2628 unregister_netdevice(net_dev);
2629 fail_locked:
2630 efx->state = STATE_UNINIT;
2631 rtnl_unlock();
2632 netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
2633 return rc;
2634 }
2635
2636 static void efx_unregister_netdev(struct efx_nic *efx)
2637 {
2638 if (!efx->net_dev)
2639 return;
2640
2641 BUG_ON(netdev_priv(efx->net_dev) != efx);
2642
2643 if (efx_dev_registered(efx)) {
2644 strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2645 #ifdef CONFIG_SFC_MCDI_LOGGING
2646 device_remove_file(&efx->pci_dev->dev, &dev_attr_mcdi_logging);
2647 #endif
2648 device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2649 unregister_netdev(efx->net_dev);
2650 }
2651 }
2652
2653 /**************************************************************************
2654 *
2655 * Device reset and suspend
2656 *
2657 **************************************************************************/
2658
2659 /* Tears down the entire software state and most of the hardware state
2660 * before reset. */
2661 void efx_reset_down(struct efx_nic *efx, enum reset_type method)
2662 {
2663 EFX_ASSERT_RESET_SERIALISED(efx);
2664
2665 if (method == RESET_TYPE_MCDI_TIMEOUT)
2666 efx->type->prepare_flr(efx);
2667
2668 efx_stop_all(efx);
2669 efx_disable_interrupts(efx);
2670
2671 mutex_lock(&efx->mac_lock);
2672 down_write(&efx->filter_sem);
2673 mutex_lock(&efx->rss_lock);
2674 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2675 method != RESET_TYPE_DATAPATH)
2676 efx->phy_op->fini(efx);
2677 efx->type->fini(efx);
2678 }
2679
2680 /* This function will always ensure that the locks acquired in
2681 * efx_reset_down() are released. A failure return code indicates
2682 * that we were unable to reinitialise the hardware, and the
2683 * driver should be disabled. If ok is false, then the rx and tx
2684 * engines are not restarted, pending a RESET_DISABLE. */
2685 int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
2686 {
2687 int rc;
2688
2689 EFX_ASSERT_RESET_SERIALISED(efx);
2690
2691 if (method == RESET_TYPE_MCDI_TIMEOUT)
2692 efx->type->finish_flr(efx);
2693
2694 /* Ensure that SRAM is initialised even if we're disabling the device */
2695 rc = efx->type->init(efx);
2696 if (rc) {
2697 netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
2698 goto fail;
2699 }
2700
2701 if (!ok)
2702 goto fail;
2703
2704 if (efx->port_initialized && method != RESET_TYPE_INVISIBLE &&
2705 method != RESET_TYPE_DATAPATH) {
2706 rc = efx->phy_op->init(efx);
2707 if (rc)
2708 goto fail;
2709 rc = efx->phy_op->reconfigure(efx);
2710 if (rc && rc != -EPERM)
2711 netif_err(efx, drv, efx->net_dev,
2712 "could not restore PHY settings\n");
2713 }
2714
2715 rc = efx_enable_interrupts(efx);
2716 if (rc)
2717 goto fail;
2718
2719 #ifdef CONFIG_SFC_SRIOV
2720 rc = efx->type->vswitching_restore(efx);
2721 if (rc) /* not fatal; the PF will still work fine */
2722 netif_warn(efx, probe, efx->net_dev,
2723 "failed to restore vswitching rc=%d;"
2724 " VFs may not function\n", rc);
2725 #endif
2726
2727 if (efx->type->rx_restore_rss_contexts)
2728 efx->type->rx_restore_rss_contexts(efx);
2729 mutex_unlock(&efx->rss_lock);
2730 efx->type->filter_table_restore(efx);
2731 up_write(&efx->filter_sem);
2732 if (efx->type->sriov_reset)
2733 efx->type->sriov_reset(efx);
2734
2735 mutex_unlock(&efx->mac_lock);
2736
2737 efx_start_all(efx);
2738
2739 if (efx->type->udp_tnl_push_ports)
2740 efx->type->udp_tnl_push_ports(efx);
2741
2742 return 0;
2743
2744 fail:
2745 efx->port_initialized = false;
2746
2747 mutex_unlock(&efx->rss_lock);
2748 up_write(&efx->filter_sem);
2749 mutex_unlock(&efx->mac_lock);
2750
2751 return rc;
2752 }
2753
2754 /* Reset the NIC using the specified method. Note that the reset may
2755 * fail, in which case the card will be left in an unusable state.
2756 *
2757 * Caller must hold the rtnl_lock.
2758 */
2759 int efx_reset(struct efx_nic *efx, enum reset_type method)
2760 {
2761 int rc, rc2;
2762 bool disabled;
2763
2764 netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
2765 RESET_TYPE(method));
2766
2767 efx_device_detach_sync(efx);
2768 efx_reset_down(efx, method);
2769
2770 rc = efx->type->reset(efx, method);
2771 if (rc) {
2772 netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
2773 goto out;
2774 }
2775
2776 /* Clear flags for the scopes we covered. We assume the NIC and
2777 * driver are now quiescent so that there is no race here.
2778 */
2779 if (method < RESET_TYPE_MAX_METHOD)
2780 efx->reset_pending &= -(1 << (method + 1));
2781 else /* it doesn't fit into the well-ordered scope hierarchy */
2782 __clear_bit(method, &efx->reset_pending);
2783
2784 /* Reinitialise bus-mastering, which may have been turned off before
2785 * the reset was scheduled. This is still appropriate, even in the
2786 * RESET_TYPE_DISABLE since this driver generally assumes the hardware
2787 * can respond to requests. */
2788 pci_set_master(efx->pci_dev);
2789
2790 out:
2791 /* Leave device stopped if necessary */
2792 disabled = rc ||
2793 method == RESET_TYPE_DISABLE ||
2794 method == RESET_TYPE_RECOVER_OR_DISABLE;
2795 rc2 = efx_reset_up(efx, method, !disabled);
2796 if (rc2) {
2797 disabled = true;
2798 if (!rc)
2799 rc = rc2;
2800 }
2801
2802 if (disabled) {
2803 dev_close(efx->net_dev);
2804 netif_err(efx, drv, efx->net_dev, "has been disabled\n");
2805 efx->state = STATE_DISABLED;
2806 } else {
2807 netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
2808 efx_device_attach_if_not_resetting(efx);
2809 }
2810 return rc;
2811 }
2812
2813 /* Try recovery mechanisms.
2814 * For now only EEH is supported.
2815 * Returns 0 if the recovery mechanisms are unsuccessful.
2816 * Returns a non-zero value otherwise.
2817 */
2818 int efx_try_recovery(struct efx_nic *efx)
2819 {
2820 #ifdef CONFIG_EEH
2821 /* A PCI error can occur and not be seen by EEH because nothing
2822 * happens on the PCI bus. In this case the driver may fail and
2823 * schedule a 'recover or reset', leading to this recovery handler.
2824 * Manually call the eeh failure check function.
2825 */
2826 struct eeh_dev *eehdev = pci_dev_to_eeh_dev(efx->pci_dev);
2827 if (eeh_dev_check_failure(eehdev)) {
2828 /* The EEH mechanisms will handle the error and reset the
2829 * device if necessary.
2830 */
2831 return 1;
2832 }
2833 #endif
2834 return 0;
2835 }
2836
2837 static void efx_wait_for_bist_end(struct efx_nic *efx)
2838 {
2839 int i;
2840
2841 for (i = 0; i < BIST_WAIT_DELAY_COUNT; ++i) {
2842 if (efx_mcdi_poll_reboot(efx))
2843 goto out;
2844 msleep(BIST_WAIT_DELAY_MS);
2845 }
2846
2847 netif_err(efx, drv, efx->net_dev, "Warning: No MC reboot after BIST mode\n");
2848 out:
2849 /* Either way unset the BIST flag. If we found no reboot we probably
2850 * won't recover, but we should try.
2851 */
2852 efx->mc_bist_for_other_fn = false;
2853 }
2854
2855 /* The worker thread exists so that code that cannot sleep can
2856 * schedule a reset for later.
2857 */
2858 static void efx_reset_work(struct work_struct *data)
2859 {
2860 struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
2861 unsigned long pending;
2862 enum reset_type method;
2863
2864 pending = READ_ONCE(efx->reset_pending);
2865 method = fls(pending) - 1;
2866
2867 if (method == RESET_TYPE_MC_BIST)
2868 efx_wait_for_bist_end(efx);
2869
2870 if ((method == RESET_TYPE_RECOVER_OR_DISABLE ||
2871 method == RESET_TYPE_RECOVER_OR_ALL) &&
2872 efx_try_recovery(efx))
2873 return;
2874
2875 if (!pending)
2876 return;
2877
2878 rtnl_lock();
2879
2880 /* We checked the state in efx_schedule_reset() but it may
2881 * have changed by now. Now that we have the RTNL lock,
2882 * it cannot change again.
2883 */
2884 if (efx->state == STATE_READY)
2885 (void)efx_reset(efx, method);
2886
2887 rtnl_unlock();
2888 }
2889
2890 void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
2891 {
2892 enum reset_type method;
2893
2894 if (efx->state == STATE_RECOVERY) {
2895 netif_dbg(efx, drv, efx->net_dev,
2896 "recovering: skip scheduling %s reset\n",
2897 RESET_TYPE(type));
2898 return;
2899 }
2900
2901 switch (type) {
2902 case RESET_TYPE_INVISIBLE:
2903 case RESET_TYPE_ALL:
2904 case RESET_TYPE_RECOVER_OR_ALL:
2905 case RESET_TYPE_WORLD:
2906 case RESET_TYPE_DISABLE:
2907 case RESET_TYPE_RECOVER_OR_DISABLE:
2908 case RESET_TYPE_DATAPATH:
2909 case RESET_TYPE_MC_BIST:
2910 case RESET_TYPE_MCDI_TIMEOUT:
2911 method = type;
2912 netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
2913 RESET_TYPE(method));
2914 break;
2915 default:
2916 method = efx->type->map_reset_reason(type);
2917 netif_dbg(efx, drv, efx->net_dev,
2918 "scheduling %s reset for %s\n",
2919 RESET_TYPE(method), RESET_TYPE(type));
2920 break;
2921 }
2922
2923 set_bit(method, &efx->reset_pending);
2924 smp_mb(); /* ensure we change reset_pending before checking state */
2925
2926 /* If we're not READY then just leave the flags set as the cue
2927 * to abort probing or reschedule the reset later.
2928 */
2929 if (READ_ONCE(efx->state) != STATE_READY)
2930 return;
2931
2932 /* efx_process_channel() will no longer read events once a
2933 * reset is scheduled. So switch back to poll'd MCDI completions. */
2934 efx_mcdi_mode_poll(efx);
2935
2936 queue_work(reset_workqueue, &efx->reset_work);
2937 }
2938
2939 /**************************************************************************
2940 *
2941 * List of NICs we support
2942 *
2943 **************************************************************************/
2944
2945 /* PCI device ID table */
2946 static const struct pci_device_id efx_pci_table[] = {
2947 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0803), /* SFC9020 */
2948 .driver_data = (unsigned long) &siena_a0_nic_type},
2949 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0813), /* SFL9021 */
2950 .driver_data = (unsigned long) &siena_a0_nic_type},
2951 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0903), /* SFC9120 PF */
2952 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2953 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1903), /* SFC9120 VF */
2954 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2955 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0923), /* SFC9140 PF */
2956 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2957 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1923), /* SFC9140 VF */
2958 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2959 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0a03), /* SFC9220 PF */
2960 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2961 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1a03), /* SFC9220 VF */
2962 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2963 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x0b03), /* SFC9250 PF */
2964 .driver_data = (unsigned long) &efx_hunt_a0_nic_type},
2965 {PCI_DEVICE(PCI_VENDOR_ID_SOLARFLARE, 0x1b03), /* SFC9250 VF */
2966 .driver_data = (unsigned long) &efx_hunt_a0_vf_nic_type},
2967 {0} /* end of list */
2968 };
2969
2970 /**************************************************************************
2971 *
2972 * Dummy PHY/MAC operations
2973 *
2974 * Can be used for some unimplemented operations
2975 * Needed so all function pointers are valid and do not have to be tested
2976 * before use
2977 *
2978 **************************************************************************/
2979 int efx_port_dummy_op_int(struct efx_nic *efx)
2980 {
2981 return 0;
2982 }
2983 void efx_port_dummy_op_void(struct efx_nic *efx) {}
2984
2985 static bool efx_port_dummy_op_poll(struct efx_nic *efx)
2986 {
2987 return false;
2988 }
2989
2990 static const struct efx_phy_operations efx_dummy_phy_operations = {
2991 .init = efx_port_dummy_op_int,
2992 .reconfigure = efx_port_dummy_op_int,
2993 .poll = efx_port_dummy_op_poll,
2994 .fini = efx_port_dummy_op_void,
2995 };
2996
2997 /**************************************************************************
2998 *
2999 * Data housekeeping
3000 *
3001 **************************************************************************/
3002
3003 /* This zeroes out and then fills in the invariants in a struct
3004 * efx_nic (including all sub-structures).
3005 */
3006 static int efx_init_struct(struct efx_nic *efx,
3007 struct pci_dev *pci_dev, struct net_device *net_dev)
3008 {
3009 int rc = -ENOMEM, i;
3010
3011 /* Initialise common structures */
3012 INIT_LIST_HEAD(&efx->node);
3013 INIT_LIST_HEAD(&efx->secondary_list);
3014 spin_lock_init(&efx->biu_lock);
3015 #ifdef CONFIG_SFC_MTD
3016 INIT_LIST_HEAD(&efx->mtd_list);
3017 #endif
3018 INIT_WORK(&efx->reset_work, efx_reset_work);
3019 INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
3020 INIT_DELAYED_WORK(&efx->selftest_work, efx_selftest_async_work);
3021 efx->pci_dev = pci_dev;
3022 efx->msg_enable = debug;
3023 efx->state = STATE_UNINIT;
3024 strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
3025
3026 efx->net_dev = net_dev;
3027 efx->rx_prefix_size = efx->type->rx_prefix_size;
3028 efx->rx_ip_align =
3029 NET_IP_ALIGN ? (efx->rx_prefix_size + NET_IP_ALIGN) % 4 : 0;
3030 efx->rx_packet_hash_offset =
3031 efx->type->rx_hash_offset - efx->type->rx_prefix_size;
3032 efx->rx_packet_ts_offset =
3033 efx->type->rx_ts_offset - efx->type->rx_prefix_size;
3034 INIT_LIST_HEAD(&efx->rss_context.list);
3035 mutex_init(&efx->rss_lock);
3036 spin_lock_init(&efx->stats_lock);
3037 efx->vi_stride = EFX_DEFAULT_VI_STRIDE;
3038 efx->num_mac_stats = MC_CMD_MAC_NSTATS;
3039 BUILD_BUG_ON(MC_CMD_MAC_NSTATS - 1 != MC_CMD_MAC_GENERATION_END);
3040 mutex_init(&efx->mac_lock);
3041 #ifdef CONFIG_RFS_ACCEL
3042 mutex_init(&efx->rps_mutex);
3043 spin_lock_init(&efx->rps_hash_lock);
3044 /* Failure to allocate is not fatal, but may degrade ARFS performance */
3045 efx->rps_hash_table = kcalloc(EFX_ARFS_HASH_TABLE_SIZE,
3046 sizeof(*efx->rps_hash_table), GFP_KERNEL);
3047 #endif
3048 efx->phy_op = &efx_dummy_phy_operations;
3049 efx->mdio.dev = net_dev;
3050 INIT_WORK(&efx->mac_work, efx_mac_work);
3051 init_waitqueue_head(&efx->flush_wq);
3052
3053 for (i = 0; i < EFX_MAX_CHANNELS; i++) {
3054 efx->channel[i] = efx_alloc_channel(efx, i, NULL);
3055 if (!efx->channel[i])
3056 goto fail;
3057 efx->msi_context[i].efx = efx;
3058 efx->msi_context[i].index = i;
3059 }
3060
3061 /* Higher numbered interrupt modes are less capable! */
3062 if (WARN_ON_ONCE(efx->type->max_interrupt_mode >
3063 efx->type->min_interrupt_mode)) {
3064 rc = -EIO;
3065 goto fail;
3066 }
3067 efx->interrupt_mode = max(efx->type->max_interrupt_mode,
3068 interrupt_mode);
3069 efx->interrupt_mode = min(efx->type->min_interrupt_mode,
3070 interrupt_mode);
3071
3072 /* Would be good to use the net_dev name, but we're too early */
3073 snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
3074 pci_name(pci_dev));
3075 efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
3076 if (!efx->workqueue)
3077 goto fail;
3078
3079 return 0;
3080
3081 fail:
3082 efx_fini_struct(efx);
3083 return rc;
3084 }
3085
3086 static void efx_fini_struct(struct efx_nic *efx)
3087 {
3088 int i;
3089
3090 #ifdef CONFIG_RFS_ACCEL
3091 kfree(efx->rps_hash_table);
3092 #endif
3093
3094 for (i = 0; i < EFX_MAX_CHANNELS; i++)
3095 kfree(efx->channel[i]);
3096
3097 kfree(efx->vpd_sn);
3098
3099 if (efx->workqueue) {
3100 destroy_workqueue(efx->workqueue);
3101 efx->workqueue = NULL;
3102 }
3103 }
3104
3105 void efx_update_sw_stats(struct efx_nic *efx, u64 *stats)
3106 {
3107 u64 n_rx_nodesc_trunc = 0;
3108 struct efx_channel *channel;
3109
3110 efx_for_each_channel(channel, efx)
3111 n_rx_nodesc_trunc += channel->n_rx_nodesc_trunc;
3112 stats[GENERIC_STAT_rx_nodesc_trunc] = n_rx_nodesc_trunc;
3113 stats[GENERIC_STAT_rx_noskb_drops] = atomic_read(&efx->n_rx_noskb_drops);
3114 }
3115
3116 bool efx_filter_spec_equal(const struct efx_filter_spec *left,
3117 const struct efx_filter_spec *right)
3118 {
3119 if ((left->match_flags ^ right->match_flags) |
3120 ((left->flags ^ right->flags) &
3121 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
3122 return false;
3123
3124 return memcmp(&left->outer_vid, &right->outer_vid,
3125 sizeof(struct efx_filter_spec) -
3126 offsetof(struct efx_filter_spec, outer_vid)) == 0;
3127 }
3128
3129 u32 efx_filter_spec_hash(const struct efx_filter_spec *spec)
3130 {
3131 BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);
3132 return jhash2((const u32 *)&spec->outer_vid,
3133 (sizeof(struct efx_filter_spec) -
3134 offsetof(struct efx_filter_spec, outer_vid)) / 4,
3135 0);
3136 }
3137
3138 #ifdef CONFIG_RFS_ACCEL
3139 bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx,
3140 bool *force)
3141 {
3142 if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {
3143 /* ARFS is currently updating this entry, leave it */
3144 return false;
3145 }
3146 if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {
3147 /* ARFS tried and failed to update this, so it's probably out
3148 * of date. Remove the filter and the ARFS rule entry.
3149 */
3150 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
3151 *force = true;
3152 return true;
3153 } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */
3154 /* ARFS has moved on, so old filter is not needed. Since we did
3155 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
3156 * not be removed by efx_rps_hash_del() subsequently.
3157 */
3158 *force = true;
3159 return true;
3160 }
3161 /* Remove it iff ARFS wants to. */
3162 return true;
3163 }
3164
3165 static
3166 struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,
3167 const struct efx_filter_spec *spec)
3168 {
3169 u32 hash = efx_filter_spec_hash(spec);
3170
3171 WARN_ON(!spin_is_locked(&efx->rps_hash_lock));
3172 if (!efx->rps_hash_table)
3173 return NULL;
3174 return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];
3175 }
3176
3177 struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx,
3178 const struct efx_filter_spec *spec)
3179 {
3180 struct efx_arfs_rule *rule;
3181 struct hlist_head *head;
3182 struct hlist_node *node;
3183
3184 head = efx_rps_hash_bucket(efx, spec);
3185 if (!head)
3186 return NULL;
3187 hlist_for_each(node, head) {
3188 rule = container_of(node, struct efx_arfs_rule, node);
3189 if (efx_filter_spec_equal(spec, &rule->spec))
3190 return rule;
3191 }
3192 return NULL;
3193 }
3194
3195 struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,
3196 const struct efx_filter_spec *spec,
3197 bool *new)
3198 {
3199 struct efx_arfs_rule *rule;
3200 struct hlist_head *head;
3201 struct hlist_node *node;
3202
3203 head = efx_rps_hash_bucket(efx, spec);
3204 if (!head)
3205 return NULL;
3206 hlist_for_each(node, head) {
3207 rule = container_of(node, struct efx_arfs_rule, node);
3208 if (efx_filter_spec_equal(spec, &rule->spec)) {
3209 *new = false;
3210 return rule;
3211 }
3212 }
3213 rule = kmalloc(sizeof(*rule), GFP_ATOMIC);
3214 *new = true;
3215 if (rule) {
3216 memcpy(&rule->spec, spec, sizeof(rule->spec));
3217 hlist_add_head(&rule->node, head);
3218 }
3219 return rule;
3220 }
3221
3222 void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec)
3223 {
3224 struct efx_arfs_rule *rule;
3225 struct hlist_head *head;
3226 struct hlist_node *node;
3227
3228 head = efx_rps_hash_bucket(efx, spec);
3229 if (WARN_ON(!head))
3230 return;
3231 hlist_for_each(node, head) {
3232 rule = container_of(node, struct efx_arfs_rule, node);
3233 if (efx_filter_spec_equal(spec, &rule->spec)) {
3234 /* Someone already reused the entry. We know that if
3235 * this check doesn't fire (i.e. filter_id == REMOVING)
3236 * then the REMOVING mark was put there by our caller,
3237 * because caller is holding a lock on filter table and
3238 * only holders of that lock set REMOVING.
3239 */
3240 if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)
3241 return;
3242 hlist_del(node);
3243 kfree(rule);
3244 return;
3245 }
3246 }
3247 /* We didn't find it. */
3248 WARN_ON(1);
3249 }
3250 #endif
3251
3252 /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because
3253 * (a) this is an infrequent control-plane operation and (b) n is small (max 64)
3254 */
3255 struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx)
3256 {
3257 struct list_head *head = &efx->rss_context.list;
3258 struct efx_rss_context *ctx, *new;
3259 u32 id = 1; /* Don't use zero, that refers to the master RSS context */
3260
3261 WARN_ON(!mutex_is_locked(&efx->rss_lock));
3262
3263 /* Search for first gap in the numbering */
3264 list_for_each_entry(ctx, head, list) {
3265 if (ctx->user_id != id)
3266 break;
3267 id++;
3268 /* Check for wrap. If this happens, we have nearly 2^32
3269 * allocated RSS contexts, which seems unlikely.
3270 */
3271 if (WARN_ON_ONCE(!id))
3272 return NULL;
3273 }
3274
3275 /* Create the new entry */
3276 new = kmalloc(sizeof(struct efx_rss_context), GFP_KERNEL);
3277 if (!new)
3278 return NULL;
3279 new->context_id = EFX_EF10_RSS_CONTEXT_INVALID;
3280 new->rx_hash_udp_4tuple = false;
3281
3282 /* Insert the new entry into the gap */
3283 new->user_id = id;
3284 list_add_tail(&new->list, &ctx->list);
3285 return new;
3286 }
3287
3288 struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id)
3289 {
3290 struct list_head *head = &efx->rss_context.list;
3291 struct efx_rss_context *ctx;
3292
3293 WARN_ON(!mutex_is_locked(&efx->rss_lock));
3294
3295 list_for_each_entry(ctx, head, list)
3296 if (ctx->user_id == id)
3297 return ctx;
3298 return NULL;
3299 }
3300
3301 void efx_free_rss_context_entry(struct efx_rss_context *ctx)
3302 {
3303 list_del(&ctx->list);
3304 kfree(ctx);
3305 }
3306
3307 /**************************************************************************
3308 *
3309 * PCI interface
3310 *
3311 **************************************************************************/
3312
3313 /* Main body of final NIC shutdown code
3314 * This is called only at module unload (or hotplug removal).
3315 */
3316 static void efx_pci_remove_main(struct efx_nic *efx)
3317 {
3318 /* Flush reset_work. It can no longer be scheduled since we
3319 * are not READY.
3320 */
3321 BUG_ON(efx->state == STATE_READY);
3322 cancel_work_sync(&efx->reset_work);
3323
3324 efx_disable_interrupts(efx);
3325 efx_clear_interrupt_affinity(efx);
3326 efx_nic_fini_interrupt(efx);
3327 efx_fini_port(efx);
3328 efx->type->fini(efx);
3329 efx_fini_napi(efx);
3330 efx_remove_all(efx);
3331 }
3332
3333 /* Final NIC shutdown
3334 * This is called only at module unload (or hotplug removal). A PF can call
3335 * this on its VFs to ensure they are unbound first.
3336 */
3337 static void efx_pci_remove(struct pci_dev *pci_dev)
3338 {
3339 struct efx_nic *efx;
3340
3341 efx = pci_get_drvdata(pci_dev);
3342 if (!efx)
3343 return;
3344
3345 /* Mark the NIC as fini, then stop the interface */
3346 rtnl_lock();
3347 efx_dissociate(efx);
3348 dev_close(efx->net_dev);
3349 efx_disable_interrupts(efx);
3350 efx->state = STATE_UNINIT;
3351 rtnl_unlock();
3352
3353 if (efx->type->sriov_fini)
3354 efx->type->sriov_fini(efx);
3355
3356 efx_unregister_netdev(efx);
3357
3358 efx_mtd_remove(efx);
3359
3360 efx_pci_remove_main(efx);
3361
3362 efx_fini_io(efx);
3363 netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
3364
3365 efx_fini_struct(efx);
3366 free_netdev(efx->net_dev);
3367
3368 pci_disable_pcie_error_reporting(pci_dev);
3369 };
3370
3371 /* NIC VPD information
3372 * Called during probe to display the part number of the
3373 * installed NIC. VPD is potentially very large but this should
3374 * always appear within the first 512 bytes.
3375 */
3376 #define SFC_VPD_LEN 512
3377 static void efx_probe_vpd_strings(struct efx_nic *efx)
3378 {
3379 struct pci_dev *dev = efx->pci_dev;
3380 char vpd_data[SFC_VPD_LEN];
3381 ssize_t vpd_size;
3382 int ro_start, ro_size, i, j;
3383
3384 /* Get the vpd data from the device */
3385 vpd_size = pci_read_vpd(dev, 0, sizeof(vpd_data), vpd_data);
3386 if (vpd_size <= 0) {
3387 netif_err(efx, drv, efx->net_dev, "Unable to read VPD\n");
3388 return;
3389 }
3390
3391 /* Get the Read only section */
3392 ro_start = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
3393 if (ro_start < 0) {
3394 netif_err(efx, drv, efx->net_dev, "VPD Read-only not found\n");
3395 return;
3396 }
3397
3398 ro_size = pci_vpd_lrdt_size(&vpd_data[ro_start]);
3399 j = ro_size;
3400 i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
3401 if (i + j > vpd_size)
3402 j = vpd_size - i;
3403
3404 /* Get the Part number */
3405 i = pci_vpd_find_info_keyword(vpd_data, i, j, "PN");
3406 if (i < 0) {
3407 netif_err(efx, drv, efx->net_dev, "Part number not found\n");
3408 return;
3409 }
3410
3411 j = pci_vpd_info_field_size(&vpd_data[i]);
3412 i += PCI_VPD_INFO_FLD_HDR_SIZE;
3413 if (i + j > vpd_size) {
3414 netif_err(efx, drv, efx->net_dev, "Incomplete part number\n");
3415 return;
3416 }
3417
3418 netif_info(efx, drv, efx->net_dev,
3419 "Part Number : %.*s\n", j, &vpd_data[i]);
3420
3421 i = ro_start + PCI_VPD_LRDT_TAG_SIZE;
3422 j = ro_size;
3423 i = pci_vpd_find_info_keyword(vpd_data, i, j, "SN");
3424 if (i < 0) {
3425 netif_err(efx, drv, efx->net_dev, "Serial number not found\n");
3426 return;
3427 }
3428
3429 j = pci_vpd_info_field_size(&vpd_data[i]);
3430 i += PCI_VPD_INFO_FLD_HDR_SIZE;
3431 if (i + j > vpd_size) {
3432 netif_err(efx, drv, efx->net_dev, "Incomplete serial number\n");
3433 return;
3434 }
3435
3436 efx->vpd_sn = kmalloc(j + 1, GFP_KERNEL);
3437 if (!efx->vpd_sn)
3438 return;
3439
3440 snprintf(efx->vpd_sn, j + 1, "%s", &vpd_data[i]);
3441 }
3442
3443
3444 /* Main body of NIC initialisation
3445 * This is called at module load (or hotplug insertion, theoretically).
3446 */
3447 static int efx_pci_probe_main(struct efx_nic *efx)
3448 {
3449 int rc;
3450
3451 /* Do start-of-day initialisation */
3452 rc = efx_probe_all(efx);
3453 if (rc)
3454 goto fail1;
3455
3456 efx_init_napi(efx);
3457
3458 down_write(&efx->filter_sem);
3459 rc = efx->type->init(efx);
3460 up_write(&efx->filter_sem);
3461 if (rc) {
3462 netif_err(efx, probe, efx->net_dev,
3463 "failed to initialise NIC\n");
3464 goto fail3;
3465 }
3466
3467 rc = efx_init_port(efx);
3468 if (rc) {
3469 netif_err(efx, probe, efx->net_dev,
3470 "failed to initialise port\n");
3471 goto fail4;
3472 }
3473
3474 rc = efx_nic_init_interrupt(efx);
3475 if (rc)
3476 goto fail5;
3477
3478 efx_set_interrupt_affinity(efx);
3479 rc = efx_enable_interrupts(efx);
3480 if (rc)
3481 goto fail6;
3482
3483 return 0;
3484
3485 fail6:
3486 efx_clear_interrupt_affinity(efx);
3487 efx_nic_fini_interrupt(efx);
3488 fail5:
3489 efx_fini_port(efx);
3490 fail4:
3491 efx->type->fini(efx);
3492 fail3:
3493 efx_fini_napi(efx);
3494 efx_remove_all(efx);
3495 fail1:
3496 return rc;
3497 }
3498
3499 static int efx_pci_probe_post_io(struct efx_nic *efx)
3500 {
3501 struct net_device *net_dev = efx->net_dev;
3502 int rc = efx_pci_probe_main(efx);
3503
3504 if (rc)
3505 return rc;
3506
3507 if (efx->type->sriov_init) {
3508 rc = efx->type->sriov_init(efx);
3509 if (rc)
3510 netif_err(efx, probe, efx->net_dev,
3511 "SR-IOV can't be enabled rc %d\n", rc);
3512 }
3513
3514 /* Determine netdevice features */
3515 net_dev->features |= (efx->type->offload_features | NETIF_F_SG |
3516 NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_RXALL);
3517 if (efx->type->offload_features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
3518 net_dev->features |= NETIF_F_TSO6;
3519 /* Check whether device supports TSO */
3520 if (!efx->type->tso_versions || !efx->type->tso_versions(efx))
3521 net_dev->features &= ~NETIF_F_ALL_TSO;
3522 /* Mask for features that also apply to VLAN devices */
3523 net_dev->vlan_features |= (NETIF_F_HW_CSUM | NETIF_F_SG |
3524 NETIF_F_HIGHDMA | NETIF_F_ALL_TSO |
3525 NETIF_F_RXCSUM);
3526
3527 net_dev->hw_features |= net_dev->features & ~efx->fixed_features;
3528
3529 /* Disable receiving frames with bad FCS, by default. */
3530 net_dev->features &= ~NETIF_F_RXALL;
3531
3532 /* Disable VLAN filtering by default. It may be enforced if
3533 * the feature is fixed (i.e. VLAN filters are required to
3534 * receive VLAN tagged packets due to vPort restrictions).
3535 */
3536 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
3537 net_dev->features |= efx->fixed_features;
3538
3539 rc = efx_register_netdev(efx);
3540 if (!rc)
3541 return 0;
3542
3543 efx_pci_remove_main(efx);
3544 return rc;
3545 }
3546
3547 /* NIC initialisation
3548 *
3549 * This is called at module load (or hotplug insertion,
3550 * theoretically). It sets up PCI mappings, resets the NIC,
3551 * sets up and registers the network devices with the kernel and hooks
3552 * the interrupt service routine. It does not prepare the device for
3553 * transmission; this is left to the first time one of the network
3554 * interfaces is brought up (i.e. efx_net_open).
3555 */
3556 static int efx_pci_probe(struct pci_dev *pci_dev,
3557 const struct pci_device_id *entry)
3558 {
3559 struct net_device *net_dev;
3560 struct efx_nic *efx;
3561 int rc;
3562
3563 /* Allocate and initialise a struct net_device and struct efx_nic */
3564 net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
3565 EFX_MAX_RX_QUEUES);
3566 if (!net_dev)
3567 return -ENOMEM;
3568 efx = netdev_priv(net_dev);
3569 efx->type = (const struct efx_nic_type *) entry->driver_data;
3570 efx->fixed_features |= NETIF_F_HIGHDMA;
3571
3572 pci_set_drvdata(pci_dev, efx);
3573 SET_NETDEV_DEV(net_dev, &pci_dev->dev);
3574 rc = efx_init_struct(efx, pci_dev, net_dev);
3575 if (rc)
3576 goto fail1;
3577
3578 netif_info(efx, probe, efx->net_dev,
3579 "Solarflare NIC detected\n");
3580
3581 if (!efx->type->is_vf)
3582 efx_probe_vpd_strings(efx);
3583
3584 /* Set up basic I/O (BAR mappings etc) */
3585 rc = efx_init_io(efx);
3586 if (rc)
3587 goto fail2;
3588
3589 rc = efx_pci_probe_post_io(efx);
3590 if (rc) {
3591 /* On failure, retry once immediately.
3592 * If we aborted probe due to a scheduled reset, dismiss it.
3593 */
3594 efx->reset_pending = 0;
3595 rc = efx_pci_probe_post_io(efx);
3596 if (rc) {
3597 /* On another failure, retry once more
3598 * after a 50-305ms delay.
3599 */
3600 unsigned char r;
3601
3602 get_random_bytes(&r, 1);
3603 msleep((unsigned int)r + 50);
3604 efx->reset_pending = 0;
3605 rc = efx_pci_probe_post_io(efx);
3606 }
3607 }
3608 if (rc)
3609 goto fail3;
3610
3611 netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
3612
3613 /* Try to create MTDs, but allow this to fail */
3614 rtnl_lock();
3615 rc = efx_mtd_probe(efx);
3616 rtnl_unlock();
3617 if (rc && rc != -EPERM)
3618 netif_warn(efx, probe, efx->net_dev,
3619 "failed to create MTDs (%d)\n", rc);
3620
3621 rc = pci_enable_pcie_error_reporting(pci_dev);
3622 if (rc && rc != -EINVAL)
3623 netif_notice(efx, probe, efx->net_dev,
3624 "PCIE error reporting unavailable (%d).\n",
3625 rc);
3626
3627 if (efx->type->udp_tnl_push_ports)
3628 efx->type->udp_tnl_push_ports(efx);
3629
3630 return 0;
3631
3632 fail3:
3633 efx_fini_io(efx);
3634 fail2:
3635 efx_fini_struct(efx);
3636 fail1:
3637 WARN_ON(rc > 0);
3638 netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
3639 free_netdev(net_dev);
3640 return rc;
3641 }
3642
3643 /* efx_pci_sriov_configure returns the actual number of Virtual Functions
3644 * enabled on success
3645 */
3646 #ifdef CONFIG_SFC_SRIOV
3647 static int efx_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
3648 {
3649 int rc;
3650 struct efx_nic *efx = pci_get_drvdata(dev);
3651
3652 if (efx->type->sriov_configure) {
3653 rc = efx->type->sriov_configure(efx, num_vfs);
3654 if (rc)
3655 return rc;
3656 else
3657 return num_vfs;
3658 } else
3659 return -EOPNOTSUPP;
3660 }
3661 #endif
3662
3663 static int efx_pm_freeze(struct device *dev)
3664 {
3665 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
3666
3667 rtnl_lock();
3668
3669 if (efx->state != STATE_DISABLED) {
3670 efx->state = STATE_UNINIT;
3671
3672 efx_device_detach_sync(efx);
3673
3674 efx_stop_all(efx);
3675 efx_disable_interrupts(efx);
3676 }
3677
3678 rtnl_unlock();
3679
3680 return 0;
3681 }
3682
3683 static int efx_pm_thaw(struct device *dev)
3684 {
3685 int rc;
3686 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
3687
3688 rtnl_lock();
3689
3690 if (efx->state != STATE_DISABLED) {
3691 rc = efx_enable_interrupts(efx);
3692 if (rc)
3693 goto fail;
3694
3695 mutex_lock(&efx->mac_lock);
3696 efx->phy_op->reconfigure(efx);
3697 mutex_unlock(&efx->mac_lock);
3698
3699 efx_start_all(efx);
3700
3701 efx_device_attach_if_not_resetting(efx);
3702
3703 efx->state = STATE_READY;
3704
3705 efx->type->resume_wol(efx);
3706 }
3707
3708 rtnl_unlock();
3709
3710 /* Reschedule any quenched resets scheduled during efx_pm_freeze() */
3711 queue_work(reset_workqueue, &efx->reset_work);
3712
3713 return 0;
3714
3715 fail:
3716 rtnl_unlock();
3717
3718 return rc;
3719 }
3720
3721 static int efx_pm_poweroff(struct device *dev)
3722 {
3723 struct pci_dev *pci_dev = to_pci_dev(dev);
3724 struct efx_nic *efx = pci_get_drvdata(pci_dev);
3725
3726 efx->type->fini(efx);
3727
3728 efx->reset_pending = 0;
3729
3730 pci_save_state(pci_dev);
3731 return pci_set_power_state(pci_dev, PCI_D3hot);
3732 }
3733
3734 /* Used for both resume and restore */
3735 static int efx_pm_resume(struct device *dev)
3736 {
3737 struct pci_dev *pci_dev = to_pci_dev(dev);
3738 struct efx_nic *efx = pci_get_drvdata(pci_dev);
3739 int rc;
3740
3741 rc = pci_set_power_state(pci_dev, PCI_D0);
3742 if (rc)
3743 return rc;
3744 pci_restore_state(pci_dev);
3745 rc = pci_enable_device(pci_dev);
3746 if (rc)
3747 return rc;
3748 pci_set_master(efx->pci_dev);
3749 rc = efx->type->reset(efx, RESET_TYPE_ALL);
3750 if (rc)
3751 return rc;
3752 down_write(&efx->filter_sem);
3753 rc = efx->type->init(efx);
3754 up_write(&efx->filter_sem);
3755 if (rc)
3756 return rc;
3757 rc = efx_pm_thaw(dev);
3758 return rc;
3759 }
3760
3761 static int efx_pm_suspend(struct device *dev)
3762 {
3763 int rc;
3764
3765 efx_pm_freeze(dev);
3766 rc = efx_pm_poweroff(dev);
3767 if (rc)
3768 efx_pm_resume(dev);
3769 return rc;
3770 }
3771
3772 static const struct dev_pm_ops efx_pm_ops = {
3773 .suspend = efx_pm_suspend,
3774 .resume = efx_pm_resume,
3775 .freeze = efx_pm_freeze,
3776 .thaw = efx_pm_thaw,
3777 .poweroff = efx_pm_poweroff,
3778 .restore = efx_pm_resume,
3779 };
3780
3781 /* A PCI error affecting this device was detected.
3782 * At this point MMIO and DMA may be disabled.
3783 * Stop the software path and request a slot reset.
3784 */
3785 static pci_ers_result_t efx_io_error_detected(struct pci_dev *pdev,
3786 enum pci_channel_state state)
3787 {
3788 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3789 struct efx_nic *efx = pci_get_drvdata(pdev);
3790
3791 if (state == pci_channel_io_perm_failure)
3792 return PCI_ERS_RESULT_DISCONNECT;
3793
3794 rtnl_lock();
3795
3796 if (efx->state != STATE_DISABLED) {
3797 efx->state = STATE_RECOVERY;
3798 efx->reset_pending = 0;
3799
3800 efx_device_detach_sync(efx);
3801
3802 efx_stop_all(efx);
3803 efx_disable_interrupts(efx);
3804
3805 status = PCI_ERS_RESULT_NEED_RESET;
3806 } else {
3807 /* If the interface is disabled we don't want to do anything
3808 * with it.
3809 */
3810 status = PCI_ERS_RESULT_RECOVERED;
3811 }
3812
3813 rtnl_unlock();
3814
3815 pci_disable_device(pdev);
3816
3817 return status;
3818 }
3819
3820 /* Fake a successful reset, which will be performed later in efx_io_resume. */
3821 static pci_ers_result_t efx_io_slot_reset(struct pci_dev *pdev)
3822 {
3823 struct efx_nic *efx = pci_get_drvdata(pdev);
3824 pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED;
3825 int rc;
3826
3827 if (pci_enable_device(pdev)) {
3828 netif_err(efx, hw, efx->net_dev,
3829 "Cannot re-enable PCI device after reset.\n");
3830 status = PCI_ERS_RESULT_DISCONNECT;
3831 }
3832
3833 rc = pci_cleanup_aer_uncorrect_error_status(pdev);
3834 if (rc) {
3835 netif_err(efx, hw, efx->net_dev,
3836 "pci_cleanup_aer_uncorrect_error_status failed (%d)\n", rc);
3837 /* Non-fatal error. Continue. */
3838 }
3839
3840 return status;
3841 }
3842
3843 /* Perform the actual reset and resume I/O operations. */
3844 static void efx_io_resume(struct pci_dev *pdev)
3845 {
3846 struct efx_nic *efx = pci_get_drvdata(pdev);
3847 int rc;
3848
3849 rtnl_lock();
3850
3851 if (efx->state == STATE_DISABLED)
3852 goto out;
3853
3854 rc = efx_reset(efx, RESET_TYPE_ALL);
3855 if (rc) {
3856 netif_err(efx, hw, efx->net_dev,
3857 "efx_reset failed after PCI error (%d)\n", rc);
3858 } else {
3859 efx->state = STATE_READY;
3860 netif_dbg(efx, hw, efx->net_dev,
3861 "Done resetting and resuming IO after PCI error.\n");
3862 }
3863
3864 out:
3865 rtnl_unlock();
3866 }
3867
3868 /* For simplicity and reliability, we always require a slot reset and try to
3869 * reset the hardware when a pci error affecting the device is detected.
3870 * We leave both the link_reset and mmio_enabled callback unimplemented:
3871 * with our request for slot reset the mmio_enabled callback will never be
3872 * called, and the link_reset callback is not used by AER or EEH mechanisms.
3873 */
3874 static const struct pci_error_handlers efx_err_handlers = {
3875 .error_detected = efx_io_error_detected,
3876 .slot_reset = efx_io_slot_reset,
3877 .resume = efx_io_resume,
3878 };
3879
3880 static struct pci_driver efx_pci_driver = {
3881 .name = KBUILD_MODNAME,
3882 .id_table = efx_pci_table,
3883 .probe = efx_pci_probe,
3884 .remove = efx_pci_remove,
3885 .driver.pm = &efx_pm_ops,
3886 .err_handler = &efx_err_handlers,
3887 #ifdef CONFIG_SFC_SRIOV
3888 .sriov_configure = efx_pci_sriov_configure,
3889 #endif
3890 };
3891
3892 /**************************************************************************
3893 *
3894 * Kernel module interface
3895 *
3896 *************************************************************************/
3897
3898 module_param(interrupt_mode, uint, 0444);
3899 MODULE_PARM_DESC(interrupt_mode,
3900 "Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
3901
3902 static int __init efx_init_module(void)
3903 {
3904 int rc;
3905
3906 printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
3907
3908 rc = register_netdevice_notifier(&efx_netdev_notifier);
3909 if (rc)
3910 goto err_notifier;
3911
3912 #ifdef CONFIG_SFC_SRIOV
3913 rc = efx_init_sriov();
3914 if (rc)
3915 goto err_sriov;
3916 #endif
3917
3918 reset_workqueue = create_singlethread_workqueue("sfc_reset");
3919 if (!reset_workqueue) {
3920 rc = -ENOMEM;
3921 goto err_reset;
3922 }
3923
3924 rc = pci_register_driver(&efx_pci_driver);
3925 if (rc < 0)
3926 goto err_pci;
3927
3928 return 0;
3929
3930 err_pci:
3931 destroy_workqueue(reset_workqueue);
3932 err_reset:
3933 #ifdef CONFIG_SFC_SRIOV
3934 efx_fini_sriov();
3935 err_sriov:
3936 #endif
3937 unregister_netdevice_notifier(&efx_netdev_notifier);
3938 err_notifier:
3939 return rc;
3940 }
3941
3942 static void __exit efx_exit_module(void)
3943 {
3944 printk(KERN_INFO "Solarflare NET driver unloading\n");
3945
3946 pci_unregister_driver(&efx_pci_driver);
3947 destroy_workqueue(reset_workqueue);
3948 #ifdef CONFIG_SFC_SRIOV
3949 efx_fini_sriov();
3950 #endif
3951 unregister_netdevice_notifier(&efx_netdev_notifier);
3952
3953 }
3954
3955 module_init(efx_init_module);
3956 module_exit(efx_exit_module);
3957
3958 MODULE_AUTHOR("Solarflare Communications and "
3959 "Michael Brown <mbrown@fensystems.co.uk>");
3960 MODULE_DESCRIPTION("Solarflare network driver");
3961 MODULE_LICENSE("GPL");
3962 MODULE_DEVICE_TABLE(pci, efx_pci_table);
3963 MODULE_VERSION(EFX_DRIVER_VERSION);
Linux with added FPC-III support