?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / net / ethernet / cadence / macb_main.c
blob7a2fe63d1136877b1ab0fad1297c0b2e3259b107
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cadence MACB/GEM Ethernet Controller driver
4 *
5 * Copyright (C) 2004-2006 Atmel Corporation
6 */
7
8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/clk.h>
10 #include <linux/clk-provider.h>
11 #include <linux/crc32.h>
12 #include <linux/module.h>
13 #include <linux/moduleparam.h>
14 #include <linux/kernel.h>
15 #include <linux/types.h>
16 #include <linux/circ_buf.h>
17 #include <linux/slab.h>
18 #include <linux/init.h>
19 #include <linux/io.h>
20 #include <linux/gpio.h>
21 #include <linux/gpio/consumer.h>
22 #include <linux/interrupt.h>
23 #include <linux/netdevice.h>
24 #include <linux/etherdevice.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/platform_data/macb.h>
27 #include <linux/platform_device.h>
28 #include <linux/phylink.h>
29 #include <linux/of.h>
30 #include <linux/of_device.h>
31 #include <linux/of_gpio.h>
32 #include <linux/of_mdio.h>
33 #include <linux/of_net.h>
34 #include <linux/ip.h>
35 #include <linux/udp.h>
36 #include <linux/tcp.h>
37 #include <linux/iopoll.h>
38 #include <linux/pm_runtime.h>
39 #include "macb.h"
40
41 /* This structure is only used for MACB on SiFive FU540 devices */
42 struct sifive_fu540_macb_mgmt {
43 void __iomem *reg;
44 unsigned long rate;
45 struct clk_hw hw;
46 };
47
48 #define MACB_RX_BUFFER_SIZE 128
49 #define RX_BUFFER_MULTIPLE 64 /* bytes */
50
51 #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */
52 #define MIN_RX_RING_SIZE 64
53 #define MAX_RX_RING_SIZE 8192
54 #define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \
55 * (bp)->rx_ring_size)
56
57 #define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */
58 #define MIN_TX_RING_SIZE 64
59 #define MAX_TX_RING_SIZE 4096
60 #define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \
61 * (bp)->tx_ring_size)
62
63 /* level of occupied TX descriptors under which we wake up TX process */
64 #define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4)
65
66 #define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR))
67 #define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \
68 | MACB_BIT(ISR_RLE) \
69 | MACB_BIT(TXERR))
70 #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \
71 | MACB_BIT(TXUBR))
72
73 /* Max length of transmit frame must be a multiple of 8 bytes */
74 #define MACB_TX_LEN_ALIGN 8
75 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))
76 #define GEM_MAX_TX_LEN ((unsigned int)((1 << GEM_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))
77
78 #define GEM_MTU_MIN_SIZE ETH_MIN_MTU
79 #define MACB_NETIF_LSO NETIF_F_TSO
80
81 #define MACB_WOL_HAS_MAGIC_PACKET (0x1 << 0)
82 #define MACB_WOL_ENABLED (0x1 << 1)
83
84 /* Graceful stop timeouts in us. We should allow up to
85 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
86 */
87 #define MACB_HALT_TIMEOUT 1230
88
89 #define MACB_PM_TIMEOUT 100 /* ms */
90
91 #define MACB_MDIO_TIMEOUT 1000000 /* in usecs */
92
93 /* DMA buffer descriptor might be different size
94 * depends on hardware configuration:
95 *
96 * 1. dma address width 32 bits:
97 * word 1: 32 bit address of Data Buffer
98 * word 2: control
99 *
100 * 2. dma address width 64 bits:
101 * word 1: 32 bit address of Data Buffer
102 * word 2: control
103 * word 3: upper 32 bit address of Data Buffer
104 * word 4: unused
105 *
106 * 3. dma address width 32 bits with hardware timestamping:
107 * word 1: 32 bit address of Data Buffer
108 * word 2: control
109 * word 3: timestamp word 1
110 * word 4: timestamp word 2
111 *
112 * 4. dma address width 64 bits with hardware timestamping:
113 * word 1: 32 bit address of Data Buffer
114 * word 2: control
115 * word 3: upper 32 bit address of Data Buffer
116 * word 4: unused
117 * word 5: timestamp word 1
118 * word 6: timestamp word 2
119 */
120 static unsigned int macb_dma_desc_get_size(struct macb *bp)
121 {
122 #ifdef MACB_EXT_DESC
123 unsigned int desc_size;
124
125 switch (bp->hw_dma_cap) {
126 case HW_DMA_CAP_64B:
127 desc_size = sizeof(struct macb_dma_desc)
128 + sizeof(struct macb_dma_desc_64);
129 break;
130 case HW_DMA_CAP_PTP:
131 desc_size = sizeof(struct macb_dma_desc)
132 + sizeof(struct macb_dma_desc_ptp);
133 break;
134 case HW_DMA_CAP_64B_PTP:
135 desc_size = sizeof(struct macb_dma_desc)
136 + sizeof(struct macb_dma_desc_64)
137 + sizeof(struct macb_dma_desc_ptp);
138 break;
139 default:
140 desc_size = sizeof(struct macb_dma_desc);
141 }
142 return desc_size;
143 #endif
144 return sizeof(struct macb_dma_desc);
145 }
146
147 static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx)
148 {
149 #ifdef MACB_EXT_DESC
150 switch (bp->hw_dma_cap) {
151 case HW_DMA_CAP_64B:
152 case HW_DMA_CAP_PTP:
153 desc_idx <<= 1;
154 break;
155 case HW_DMA_CAP_64B_PTP:
156 desc_idx *= 3;
157 break;
158 default:
159 break;
160 }
161 #endif
162 return desc_idx;
163 }
164
165 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
166 static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc)
167 {
168 return (struct macb_dma_desc_64 *)((void *)desc
169 + sizeof(struct macb_dma_desc));
170 }
171 #endif
172
173 /* Ring buffer accessors */
174 static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index)
175 {
176 return index & (bp->tx_ring_size - 1);
177 }
178
179 static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue,
180 unsigned int index)
181 {
182 index = macb_tx_ring_wrap(queue->bp, index);
183 index = macb_adj_dma_desc_idx(queue->bp, index);
184 return &queue->tx_ring[index];
185 }
186
187 static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue,
188 unsigned int index)
189 {
190 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];
191 }
192
193 static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index)
194 {
195 dma_addr_t offset;
196
197 offset = macb_tx_ring_wrap(queue->bp, index) *
198 macb_dma_desc_get_size(queue->bp);
199
200 return queue->tx_ring_dma + offset;
201 }
202
203 static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index)
204 {
205 return index & (bp->rx_ring_size - 1);
206 }
207
208 static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index)
209 {
210 index = macb_rx_ring_wrap(queue->bp, index);
211 index = macb_adj_dma_desc_idx(queue->bp, index);
212 return &queue->rx_ring[index];
213 }
214
215 static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index)
216 {
217 return queue->rx_buffers + queue->bp->rx_buffer_size *
218 macb_rx_ring_wrap(queue->bp, index);
219 }
220
221 /* I/O accessors */
222 static u32 hw_readl_native(struct macb *bp, int offset)
223 {
224 return __raw_readl(bp->regs + offset);
225 }
226
227 static void hw_writel_native(struct macb *bp, int offset, u32 value)
228 {
229 __raw_writel(value, bp->regs + offset);
230 }
231
232 static u32 hw_readl(struct macb *bp, int offset)
233 {
234 return readl_relaxed(bp->regs + offset);
235 }
236
237 static void hw_writel(struct macb *bp, int offset, u32 value)
238 {
239 writel_relaxed(value, bp->regs + offset);
240 }
241
242 /* Find the CPU endianness by using the loopback bit of NCR register. When the
243 * CPU is in big endian we need to program swapped mode for management
244 * descriptor access.
245 */
246 static bool hw_is_native_io(void __iomem *addr)
247 {
248 u32 value = MACB_BIT(LLB);
249
250 __raw_writel(value, addr + MACB_NCR);
251 value = __raw_readl(addr + MACB_NCR);
252
253 /* Write 0 back to disable everything */
254 __raw_writel(0, addr + MACB_NCR);
255
256 return value == MACB_BIT(LLB);
257 }
258
259 static bool hw_is_gem(void __iomem *addr, bool native_io)
260 {
261 u32 id;
262
263 if (native_io)
264 id = __raw_readl(addr + MACB_MID);
265 else
266 id = readl_relaxed(addr + MACB_MID);
267
268 return MACB_BFEXT(IDNUM, id) >= 0x2;
269 }
270
271 static void macb_set_hwaddr(struct macb *bp)
272 {
273 u32 bottom;
274 u16 top;
275
276 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));
277 macb_or_gem_writel(bp, SA1B, bottom);
278 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));
279 macb_or_gem_writel(bp, SA1T, top);
280
281 /* Clear unused address register sets */
282 macb_or_gem_writel(bp, SA2B, 0);
283 macb_or_gem_writel(bp, SA2T, 0);
284 macb_or_gem_writel(bp, SA3B, 0);
285 macb_or_gem_writel(bp, SA3T, 0);
286 macb_or_gem_writel(bp, SA4B, 0);
287 macb_or_gem_writel(bp, SA4T, 0);
288 }
289
290 static void macb_get_hwaddr(struct macb *bp)
291 {
292 u32 bottom;
293 u16 top;
294 u8 addr[6];
295 int i;
296
297 /* Check all 4 address register for valid address */
298 for (i = 0; i < 4; i++) {
299 bottom = macb_or_gem_readl(bp, SA1B + i * 8);
300 top = macb_or_gem_readl(bp, SA1T + i * 8);
301
302 addr[0] = bottom & 0xff;
303 addr[1] = (bottom >> 8) & 0xff;
304 addr[2] = (bottom >> 16) & 0xff;
305 addr[3] = (bottom >> 24) & 0xff;
306 addr[4] = top & 0xff;
307 addr[5] = (top >> 8) & 0xff;
308
309 if (is_valid_ether_addr(addr)) {
310 memcpy(bp->dev->dev_addr, addr, sizeof(addr));
311 return;
312 }
313 }
314
315 dev_info(&bp->pdev->dev, "invalid hw address, using random\n");
316 eth_hw_addr_random(bp->dev);
317 }
318
319 static int macb_mdio_wait_for_idle(struct macb *bp)
320 {
321 u32 val;
322
323 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE),
324 1, MACB_MDIO_TIMEOUT);
325 }
326
327 static int macb_mdio_read(struct mii_bus *bus, int mii_id, int regnum)
328 {
329 struct macb *bp = bus->priv;
330 int status;
331
332 status = pm_runtime_get_sync(&bp->pdev->dev);
333 if (status < 0)
334 goto mdio_pm_exit;
335
336 status = macb_mdio_wait_for_idle(bp);
337 if (status < 0)
338 goto mdio_read_exit;
339
340 if (regnum & MII_ADDR_C45) {
341 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
342 | MACB_BF(RW, MACB_MAN_C45_ADDR)
343 | MACB_BF(PHYA, mii_id)
344 | MACB_BF(REGA, (regnum >> 16) & 0x1F)
345 | MACB_BF(DATA, regnum & 0xFFFF)
346 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
347
348 status = macb_mdio_wait_for_idle(bp);
349 if (status < 0)
350 goto mdio_read_exit;
351
352 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
353 | MACB_BF(RW, MACB_MAN_C45_READ)
354 | MACB_BF(PHYA, mii_id)
355 | MACB_BF(REGA, (regnum >> 16) & 0x1F)
356 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
357 } else {
358 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
359 | MACB_BF(RW, MACB_MAN_C22_READ)
360 | MACB_BF(PHYA, mii_id)
361 | MACB_BF(REGA, regnum)
362 | MACB_BF(CODE, MACB_MAN_C22_CODE)));
363 }
364
365 status = macb_mdio_wait_for_idle(bp);
366 if (status < 0)
367 goto mdio_read_exit;
368
369 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));
370
371 mdio_read_exit:
372 pm_runtime_mark_last_busy(&bp->pdev->dev);
373 pm_runtime_put_autosuspend(&bp->pdev->dev);
374 mdio_pm_exit:
375 return status;
376 }
377
378 static int macb_mdio_write(struct mii_bus *bus, int mii_id, int regnum,
379 u16 value)
380 {
381 struct macb *bp = bus->priv;
382 int status;
383
384 status = pm_runtime_get_sync(&bp->pdev->dev);
385 if (status < 0)
386 goto mdio_pm_exit;
387
388 status = macb_mdio_wait_for_idle(bp);
389 if (status < 0)
390 goto mdio_write_exit;
391
392 if (regnum & MII_ADDR_C45) {
393 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
394 | MACB_BF(RW, MACB_MAN_C45_ADDR)
395 | MACB_BF(PHYA, mii_id)
396 | MACB_BF(REGA, (regnum >> 16) & 0x1F)
397 | MACB_BF(DATA, regnum & 0xFFFF)
398 | MACB_BF(CODE, MACB_MAN_C45_CODE)));
399
400 status = macb_mdio_wait_for_idle(bp);
401 if (status < 0)
402 goto mdio_write_exit;
403
404 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)
405 | MACB_BF(RW, MACB_MAN_C45_WRITE)
406 | MACB_BF(PHYA, mii_id)
407 | MACB_BF(REGA, (regnum >> 16) & 0x1F)
408 | MACB_BF(CODE, MACB_MAN_C45_CODE)
409 | MACB_BF(DATA, value)));
410 } else {
411 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)
412 | MACB_BF(RW, MACB_MAN_C22_WRITE)
413 | MACB_BF(PHYA, mii_id)
414 | MACB_BF(REGA, regnum)
415 | MACB_BF(CODE, MACB_MAN_C22_CODE)
416 | MACB_BF(DATA, value)));
417 }
418
419 status = macb_mdio_wait_for_idle(bp);
420 if (status < 0)
421 goto mdio_write_exit;
422
423 mdio_write_exit:
424 pm_runtime_mark_last_busy(&bp->pdev->dev);
425 pm_runtime_put_autosuspend(&bp->pdev->dev);
426 mdio_pm_exit:
427 return status;
428 }
429
430 static void macb_init_buffers(struct macb *bp)
431 {
432 struct macb_queue *queue;
433 unsigned int q;
434
435 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
436 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));
437 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
438 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
439 queue_writel(queue, RBQPH,
440 upper_32_bits(queue->rx_ring_dma));
441 #endif
442 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
443 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
444 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
445 queue_writel(queue, TBQPH,
446 upper_32_bits(queue->tx_ring_dma));
447 #endif
448 }
449 }
450
451 /**
452 * macb_set_tx_clk() - Set a clock to a new frequency
453 * @clk Pointer to the clock to change
454 * @rate New frequency in Hz
455 * @dev Pointer to the struct net_device
456 */
457 static void macb_set_tx_clk(struct clk *clk, int speed, struct net_device *dev)
458 {
459 long ferr, rate, rate_rounded;
460
461 if (!clk)
462 return;
463
464 switch (speed) {
465 case SPEED_10:
466 rate = 2500000;
467 break;
468 case SPEED_100:
469 rate = 25000000;
470 break;
471 case SPEED_1000:
472 rate = 125000000;
473 break;
474 default:
475 return;
476 }
477
478 rate_rounded = clk_round_rate(clk, rate);
479 if (rate_rounded < 0)
480 return;
481
482 /* RGMII allows 50 ppm frequency error. Test and warn if this limit
483 * is not satisfied.
484 */
485 ferr = abs(rate_rounded - rate);
486 ferr = DIV_ROUND_UP(ferr, rate / 100000);
487 if (ferr > 5)
488 netdev_warn(dev, "unable to generate target frequency: %ld Hz\n",
489 rate);
490
491 if (clk_set_rate(clk, rate_rounded))
492 netdev_err(dev, "adjusting tx_clk failed.\n");
493 }
494
495 static void macb_validate(struct phylink_config *config,
496 unsigned long *supported,
497 struct phylink_link_state *state)
498 {
499 struct net_device *ndev = to_net_dev(config->dev);
500 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
501 struct macb *bp = netdev_priv(ndev);
502
503 /* We only support MII, RMII, GMII, RGMII & SGMII. */
504 if (state->interface != PHY_INTERFACE_MODE_NA &&
505 state->interface != PHY_INTERFACE_MODE_MII &&
506 state->interface != PHY_INTERFACE_MODE_RMII &&
507 state->interface != PHY_INTERFACE_MODE_GMII &&
508 state->interface != PHY_INTERFACE_MODE_SGMII &&
509 !phy_interface_mode_is_rgmii(state->interface)) {
510 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
511 return;
512 }
513
514 if (!macb_is_gem(bp) &&
515 (state->interface == PHY_INTERFACE_MODE_GMII ||
516 phy_interface_mode_is_rgmii(state->interface))) {
517 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
518 return;
519 }
520
521 phylink_set_port_modes(mask);
522 phylink_set(mask, Autoneg);
523 phylink_set(mask, Asym_Pause);
524
525 phylink_set(mask, 10baseT_Half);
526 phylink_set(mask, 10baseT_Full);
527 phylink_set(mask, 100baseT_Half);
528 phylink_set(mask, 100baseT_Full);
529
530 if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE &&
531 (state->interface == PHY_INTERFACE_MODE_NA ||
532 state->interface == PHY_INTERFACE_MODE_GMII ||
533 state->interface == PHY_INTERFACE_MODE_SGMII ||
534 phy_interface_mode_is_rgmii(state->interface))) {
535 phylink_set(mask, 1000baseT_Full);
536 phylink_set(mask, 1000baseX_Full);
537
538 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))
539 phylink_set(mask, 1000baseT_Half);
540 }
541
542 bitmap_and(supported, supported, mask, __ETHTOOL_LINK_MODE_MASK_NBITS);
543 bitmap_and(state->advertising, state->advertising, mask,
544 __ETHTOOL_LINK_MODE_MASK_NBITS);
545 }
546
547 static void macb_mac_pcs_get_state(struct phylink_config *config,
548 struct phylink_link_state *state)
549 {
550 state->link = 0;
551 }
552
553 static void macb_mac_an_restart(struct phylink_config *config)
554 {
555 /* Not supported */
556 }
557
558 static void macb_mac_config(struct phylink_config *config, unsigned int mode,
559 const struct phylink_link_state *state)
560 {
561 struct net_device *ndev = to_net_dev(config->dev);
562 struct macb *bp = netdev_priv(ndev);
563 unsigned long flags;
564 u32 old_ctrl, ctrl;
565
566 spin_lock_irqsave(&bp->lock, flags);
567
568 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR);
569
570 /* Clear all the bits we might set later */
571 ctrl &= ~(GEM_BIT(GBE) | MACB_BIT(SPD) | MACB_BIT(FD) | MACB_BIT(PAE) |
572 GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL));
573
574 if (state->speed == SPEED_1000)
575 ctrl |= GEM_BIT(GBE);
576 else if (state->speed == SPEED_100)
577 ctrl |= MACB_BIT(SPD);
578
579 if (state->duplex)
580 ctrl |= MACB_BIT(FD);
581
582 /* We do not support MLO_PAUSE_RX yet */
583 if (state->pause & MLO_PAUSE_TX)
584 ctrl |= MACB_BIT(PAE);
585
586 if (state->interface == PHY_INTERFACE_MODE_SGMII)
587 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
588
589 /* Apply the new configuration, if any */
590 if (old_ctrl ^ ctrl)
591 macb_or_gem_writel(bp, NCFGR, ctrl);
592
593 bp->speed = state->speed;
594
595 spin_unlock_irqrestore(&bp->lock, flags);
596 }
597
598 static void macb_mac_link_down(struct phylink_config *config, unsigned int mode,
599 phy_interface_t interface)
600 {
601 struct net_device *ndev = to_net_dev(config->dev);
602 struct macb *bp = netdev_priv(ndev);
603 struct macb_queue *queue;
604 unsigned int q;
605 u32 ctrl;
606
607 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
608 queue_writel(queue, IDR,
609 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
610
611 /* Disable Rx and Tx */
612 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE));
613 macb_writel(bp, NCR, ctrl);
614
615 netif_tx_stop_all_queues(ndev);
616 }
617
618 static void macb_mac_link_up(struct phylink_config *config, unsigned int mode,
619 phy_interface_t interface, struct phy_device *phy)
620 {
621 struct net_device *ndev = to_net_dev(config->dev);
622 struct macb *bp = netdev_priv(ndev);
623 struct macb_queue *queue;
624 unsigned int q;
625
626 macb_set_tx_clk(bp->tx_clk, bp->speed, ndev);
627
628 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down
629 * cleared the pipeline and control registers.
630 */
631 bp->macbgem_ops.mog_init_rings(bp);
632 macb_init_buffers(bp);
633
634 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
635 queue_writel(queue, IER,
636 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));
637
638 /* Enable Rx and Tx */
639 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(RE) | MACB_BIT(TE));
640
641 netif_tx_wake_all_queues(ndev);
642 }
643
644 static const struct phylink_mac_ops macb_phylink_ops = {
645 .validate = macb_validate,
646 .mac_pcs_get_state = macb_mac_pcs_get_state,
647 .mac_an_restart = macb_mac_an_restart,
648 .mac_config = macb_mac_config,
649 .mac_link_down = macb_mac_link_down,
650 .mac_link_up = macb_mac_link_up,
651 };
652
653 static bool macb_phy_handle_exists(struct device_node *dn)
654 {
655 dn = of_parse_phandle(dn, "phy-handle", 0);
656 of_node_put(dn);
657 return dn != NULL;
658 }
659
660 static int macb_phylink_connect(struct macb *bp)
661 {
662 struct device_node *dn = bp->pdev->dev.of_node;
663 struct net_device *dev = bp->dev;
664 struct phy_device *phydev;
665 int ret;
666
667 if (dn)
668 ret = phylink_of_phy_connect(bp->phylink, dn, 0);
669
670 if (!dn || (ret && !macb_phy_handle_exists(dn))) {
671 phydev = phy_find_first(bp->mii_bus);
672 if (!phydev) {
673 netdev_err(dev, "no PHY found\n");
674 return -ENXIO;
675 }
676
677 /* attach the mac to the phy */
678 ret = phylink_connect_phy(bp->phylink, phydev);
679 }
680
681 if (ret) {
682 netdev_err(dev, "Could not attach PHY (%d)\n", ret);
683 return ret;
684 }
685
686 phylink_start(bp->phylink);
687
688 return 0;
689 }
690
691 /* based on au1000_eth. c*/
692 static int macb_mii_probe(struct net_device *dev)
693 {
694 struct macb *bp = netdev_priv(dev);
695
696 bp->phylink_config.dev = &dev->dev;
697 bp->phylink_config.type = PHYLINK_NETDEV;
698
699 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,
700 bp->phy_interface, &macb_phylink_ops);
701 if (IS_ERR(bp->phylink)) {
702 netdev_err(dev, "Could not create a phylink instance (%ld)\n",
703 PTR_ERR(bp->phylink));
704 return PTR_ERR(bp->phylink);
705 }
706
707 return 0;
708 }
709
710 static int macb_mdiobus_register(struct macb *bp)
711 {
712 struct device_node *child, *np = bp->pdev->dev.of_node;
713
714 /* Only create the PHY from the device tree if at least one PHY is
715 * described. Otherwise scan the entire MDIO bus. We do this to support
716 * old device tree that did not follow the best practices and did not
717 * describe their network PHYs.
718 */
719 for_each_available_child_of_node(np, child)
720 if (of_mdiobus_child_is_phy(child)) {
721 /* The loop increments the child refcount,
722 * decrement it before returning.
723 */
724 of_node_put(child);
725
726 return of_mdiobus_register(bp->mii_bus, np);
727 }
728
729 return mdiobus_register(bp->mii_bus);
730 }
731
732 static int macb_mii_init(struct macb *bp)
733 {
734 int err = -ENXIO;
735
736 /* Enable management port */
737 macb_writel(bp, NCR, MACB_BIT(MPE));
738
739 bp->mii_bus = mdiobus_alloc();
740 if (!bp->mii_bus) {
741 err = -ENOMEM;
742 goto err_out;
743 }
744
745 bp->mii_bus->name = "MACB_mii_bus";
746 bp->mii_bus->read = &macb_mdio_read;
747 bp->mii_bus->write = &macb_mdio_write;
748 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
749 bp->pdev->name, bp->pdev->id);
750 bp->mii_bus->priv = bp;
751 bp->mii_bus->parent = &bp->pdev->dev;
752
753 dev_set_drvdata(&bp->dev->dev, bp->mii_bus);
754
755 err = macb_mdiobus_register(bp);
756 if (err)
757 goto err_out_free_mdiobus;
758
759 err = macb_mii_probe(bp->dev);
760 if (err)
761 goto err_out_unregister_bus;
762
763 return 0;
764
765 err_out_unregister_bus:
766 mdiobus_unregister(bp->mii_bus);
767 err_out_free_mdiobus:
768 mdiobus_free(bp->mii_bus);
769 err_out:
770 return err;
771 }
772
773 static void macb_update_stats(struct macb *bp)
774 {
775 u32 *p = &bp->hw_stats.macb.rx_pause_frames;
776 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;
777 int offset = MACB_PFR;
778
779 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);
780
781 for (; p < end; p++, offset += 4)
782 *p += bp->macb_reg_readl(bp, offset);
783 }
784
785 static int macb_halt_tx(struct macb *bp)
786 {
787 unsigned long halt_time, timeout;
788 u32 status;
789
790 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT));
791
792 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT);
793 do {
794 halt_time = jiffies;
795 status = macb_readl(bp, TSR);
796 if (!(status & MACB_BIT(TGO)))
797 return 0;
798
799 udelay(250);
800 } while (time_before(halt_time, timeout));
801
802 return -ETIMEDOUT;
803 }
804
805 static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb)
806 {
807 if (tx_skb->mapping) {
808 if (tx_skb->mapped_as_page)
809 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,
810 tx_skb->size, DMA_TO_DEVICE);
811 else
812 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,
813 tx_skb->size, DMA_TO_DEVICE);
814 tx_skb->mapping = 0;
815 }
816
817 if (tx_skb->skb) {
818 dev_kfree_skb_any(tx_skb->skb);
819 tx_skb->skb = NULL;
820 }
821 }
822
823 static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr)
824 {
825 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
826 struct macb_dma_desc_64 *desc_64;
827
828 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
829 desc_64 = macb_64b_desc(bp, desc);
830 desc_64->addrh = upper_32_bits(addr);
831 /* The low bits of RX address contain the RX_USED bit, clearing
832 * of which allows packet RX. Make sure the high bits are also
833 * visible to HW at that point.
834 */
835 dma_wmb();
836 }
837 #endif
838 desc->addr = lower_32_bits(addr);
839 }
840
841 static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc)
842 {
843 dma_addr_t addr = 0;
844 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
845 struct macb_dma_desc_64 *desc_64;
846
847 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
848 desc_64 = macb_64b_desc(bp, desc);
849 addr = ((u64)(desc_64->addrh) << 32);
850 }
851 #endif
852 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));
853 return addr;
854 }
855
856 static void macb_tx_error_task(struct work_struct *work)
857 {
858 struct macb_queue *queue = container_of(work, struct macb_queue,
859 tx_error_task);
860 struct macb *bp = queue->bp;
861 struct macb_tx_skb *tx_skb;
862 struct macb_dma_desc *desc;
863 struct sk_buff *skb;
864 unsigned int tail;
865 unsigned long flags;
866
867 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",
868 (unsigned int)(queue - bp->queues),
869 queue->tx_tail, queue->tx_head);
870
871 /* Prevent the queue IRQ handlers from running: each of them may call
872 * macb_tx_interrupt(), which in turn may call netif_wake_subqueue().
873 * As explained below, we have to halt the transmission before updating
874 * TBQP registers so we call netif_tx_stop_all_queues() to notify the
875 * network engine about the macb/gem being halted.
876 */
877 spin_lock_irqsave(&bp->lock, flags);
878
879 /* Make sure nobody is trying to queue up new packets */
880 netif_tx_stop_all_queues(bp->dev);
881
882 /* Stop transmission now
883 * (in case we have just queued new packets)
884 * macb/gem must be halted to write TBQP register
885 */
886 if (macb_halt_tx(bp))
887 /* Just complain for now, reinitializing TX path can be good */
888 netdev_err(bp->dev, "BUG: halt tx timed out\n");
889
890 /* Treat frames in TX queue including the ones that caused the error.
891 * Free transmit buffers in upper layer.
892 */
893 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {
894 u32 ctrl;
895
896 desc = macb_tx_desc(queue, tail);
897 ctrl = desc->ctrl;
898 tx_skb = macb_tx_skb(queue, tail);
899 skb = tx_skb->skb;
900
901 if (ctrl & MACB_BIT(TX_USED)) {
902 /* skb is set for the last buffer of the frame */
903 while (!skb) {
904 macb_tx_unmap(bp, tx_skb);
905 tail++;
906 tx_skb = macb_tx_skb(queue, tail);
907 skb = tx_skb->skb;
908 }
909
910 /* ctrl still refers to the first buffer descriptor
911 * since it's the only one written back by the hardware
912 */
913 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) {
914 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",
915 macb_tx_ring_wrap(bp, tail),
916 skb->data);
917 bp->dev->stats.tx_packets++;
918 queue->stats.tx_packets++;
919 bp->dev->stats.tx_bytes += skb->len;
920 queue->stats.tx_bytes += skb->len;
921 }
922 } else {
923 /* "Buffers exhausted mid-frame" errors may only happen
924 * if the driver is buggy, so complain loudly about
925 * those. Statistics are updated by hardware.
926 */
927 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED))
928 netdev_err(bp->dev,
929 "BUG: TX buffers exhausted mid-frame\n");
930
931 desc->ctrl = ctrl | MACB_BIT(TX_USED);
932 }
933
934 macb_tx_unmap(bp, tx_skb);
935 }
936
937 /* Set end of TX queue */
938 desc = macb_tx_desc(queue, 0);
939 macb_set_addr(bp, desc, 0);
940 desc->ctrl = MACB_BIT(TX_USED);
941
942 /* Make descriptor updates visible to hardware */
943 wmb();
944
945 /* Reinitialize the TX desc queue */
946 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));
947 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
948 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
949 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));
950 #endif
951 /* Make TX ring reflect state of hardware */
952 queue->tx_head = 0;
953 queue->tx_tail = 0;
954
955 /* Housework before enabling TX IRQ */
956 macb_writel(bp, TSR, macb_readl(bp, TSR));
957 queue_writel(queue, IER, MACB_TX_INT_FLAGS);
958
959 /* Now we are ready to start transmission again */
960 netif_tx_start_all_queues(bp->dev);
961 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
962
963 spin_unlock_irqrestore(&bp->lock, flags);
964 }
965
966 static void macb_tx_interrupt(struct macb_queue *queue)
967 {
968 unsigned int tail;
969 unsigned int head;
970 u32 status;
971 struct macb *bp = queue->bp;
972 u16 queue_index = queue - bp->queues;
973
974 status = macb_readl(bp, TSR);
975 macb_writel(bp, TSR, status);
976
977 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
978 queue_writel(queue, ISR, MACB_BIT(TCOMP));
979
980 netdev_vdbg(bp->dev, "macb_tx_interrupt status = 0x%03lx\n",
981 (unsigned long)status);
982
983 head = queue->tx_head;
984 for (tail = queue->tx_tail; tail != head; tail++) {
985 struct macb_tx_skb *tx_skb;
986 struct sk_buff *skb;
987 struct macb_dma_desc *desc;
988 u32 ctrl;
989
990 desc = macb_tx_desc(queue, tail);
991
992 /* Make hw descriptor updates visible to CPU */
993 rmb();
994
995 ctrl = desc->ctrl;
996
997 /* TX_USED bit is only set by hardware on the very first buffer
998 * descriptor of the transmitted frame.
999 */
1000 if (!(ctrl & MACB_BIT(TX_USED)))
1001 break;
1002
1003 /* Process all buffers of the current transmitted frame */
1004 for (;; tail++) {
1005 tx_skb = macb_tx_skb(queue, tail);
1006 skb = tx_skb->skb;
1007
1008 /* First, update TX stats if needed */
1009 if (skb) {
1010 if (unlikely(skb_shinfo(skb)->tx_flags &
1011 SKBTX_HW_TSTAMP) &&
1012 gem_ptp_do_txstamp(queue, skb, desc) == 0) {
1013 /* skb now belongs to timestamp buffer
1014 * and will be removed later
1015 */
1016 tx_skb->skb = NULL;
1017 }
1018 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",
1019 macb_tx_ring_wrap(bp, tail),
1020 skb->data);
1021 bp->dev->stats.tx_packets++;
1022 queue->stats.tx_packets++;
1023 bp->dev->stats.tx_bytes += skb->len;
1024 queue->stats.tx_bytes += skb->len;
1025 }
1026
1027 /* Now we can safely release resources */
1028 macb_tx_unmap(bp, tx_skb);
1029
1030 /* skb is set only for the last buffer of the frame.
1031 * WARNING: at this point skb has been freed by
1032 * macb_tx_unmap().
1033 */
1034 if (skb)
1035 break;
1036 }
1037 }
1038
1039 queue->tx_tail = tail;
1040 if (__netif_subqueue_stopped(bp->dev, queue_index) &&
1041 CIRC_CNT(queue->tx_head, queue->tx_tail,
1042 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))
1043 netif_wake_subqueue(bp->dev, queue_index);
1044 }
1045
1046 static void gem_rx_refill(struct macb_queue *queue)
1047 {
1048 unsigned int entry;
1049 struct sk_buff *skb;
1050 dma_addr_t paddr;
1051 struct macb *bp = queue->bp;
1052 struct macb_dma_desc *desc;
1053
1054 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,
1055 bp->rx_ring_size) > 0) {
1056 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);
1057
1058 /* Make hw descriptor updates visible to CPU */
1059 rmb();
1060
1061 queue->rx_prepared_head++;
1062 desc = macb_rx_desc(queue, entry);
1063
1064 if (!queue->rx_skbuff[entry]) {
1065 /* allocate sk_buff for this free entry in ring */
1066 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);
1067 if (unlikely(!skb)) {
1068 netdev_err(bp->dev,
1069 "Unable to allocate sk_buff\n");
1070 break;
1071 }
1072
1073 /* now fill corresponding descriptor entry */
1074 paddr = dma_map_single(&bp->pdev->dev, skb->data,
1075 bp->rx_buffer_size,
1076 DMA_FROM_DEVICE);
1077 if (dma_mapping_error(&bp->pdev->dev, paddr)) {
1078 dev_kfree_skb(skb);
1079 break;
1080 }
1081
1082 queue->rx_skbuff[entry] = skb;
1083
1084 if (entry == bp->rx_ring_size - 1)
1085 paddr |= MACB_BIT(RX_WRAP);
1086 desc->ctrl = 0;
1087 /* Setting addr clears RX_USED and allows reception,
1088 * make sure ctrl is cleared first to avoid a race.
1089 */
1090 dma_wmb();
1091 macb_set_addr(bp, desc, paddr);
1092
1093 /* properly align Ethernet header */
1094 skb_reserve(skb, NET_IP_ALIGN);
1095 } else {
1096 desc->ctrl = 0;
1097 dma_wmb();
1098 desc->addr &= ~MACB_BIT(RX_USED);
1099 }
1100 }
1101
1102 /* Make descriptor updates visible to hardware */
1103 wmb();
1104
1105 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",
1106 queue, queue->rx_prepared_head, queue->rx_tail);
1107 }
1108
1109 /* Mark DMA descriptors from begin up to and not including end as unused */
1110 static void discard_partial_frame(struct macb_queue *queue, unsigned int begin,
1111 unsigned int end)
1112 {
1113 unsigned int frag;
1114
1115 for (frag = begin; frag != end; frag++) {
1116 struct macb_dma_desc *desc = macb_rx_desc(queue, frag);
1117
1118 desc->addr &= ~MACB_BIT(RX_USED);
1119 }
1120
1121 /* Make descriptor updates visible to hardware */
1122 wmb();
1123
1124 /* When this happens, the hardware stats registers for
1125 * whatever caused this is updated, so we don't have to record
1126 * anything.
1127 */
1128 }
1129
1130 static int gem_rx(struct macb_queue *queue, struct napi_struct *napi,
1131 int budget)
1132 {
1133 struct macb *bp = queue->bp;
1134 unsigned int len;
1135 unsigned int entry;
1136 struct sk_buff *skb;
1137 struct macb_dma_desc *desc;
1138 int count = 0;
1139
1140 while (count < budget) {
1141 u32 ctrl;
1142 dma_addr_t addr;
1143 bool rxused;
1144
1145 entry = macb_rx_ring_wrap(bp, queue->rx_tail);
1146 desc = macb_rx_desc(queue, entry);
1147
1148 /* Make hw descriptor updates visible to CPU */
1149 rmb();
1150
1151 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;
1152 addr = macb_get_addr(bp, desc);
1153
1154 if (!rxused)
1155 break;
1156
1157 /* Ensure ctrl is at least as up-to-date as rxused */
1158 dma_rmb();
1159
1160 ctrl = desc->ctrl;
1161
1162 queue->rx_tail++;
1163 count++;
1164
1165 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) {
1166 netdev_err(bp->dev,
1167 "not whole frame pointed by descriptor\n");
1168 bp->dev->stats.rx_dropped++;
1169 queue->stats.rx_dropped++;
1170 break;
1171 }
1172 skb = queue->rx_skbuff[entry];
1173 if (unlikely(!skb)) {
1174 netdev_err(bp->dev,
1175 "inconsistent Rx descriptor chain\n");
1176 bp->dev->stats.rx_dropped++;
1177 queue->stats.rx_dropped++;
1178 break;
1179 }
1180 /* now everything is ready for receiving packet */
1181 queue->rx_skbuff[entry] = NULL;
1182 len = ctrl & bp->rx_frm_len_mask;
1183
1184 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);
1185
1186 skb_put(skb, len);
1187 dma_unmap_single(&bp->pdev->dev, addr,
1188 bp->rx_buffer_size, DMA_FROM_DEVICE);
1189
1190 skb->protocol = eth_type_trans(skb, bp->dev);
1191 skb_checksum_none_assert(skb);
1192 if (bp->dev->features & NETIF_F_RXCSUM &&
1193 !(bp->dev->flags & IFF_PROMISC) &&
1194 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK)
1195 skb->ip_summed = CHECKSUM_UNNECESSARY;
1196
1197 bp->dev->stats.rx_packets++;
1198 queue->stats.rx_packets++;
1199 bp->dev->stats.rx_bytes += skb->len;
1200 queue->stats.rx_bytes += skb->len;
1201
1202 gem_ptp_do_rxstamp(bp, skb, desc);
1203
1204 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
1205 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1206 skb->len, skb->csum);
1207 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1,
1208 skb_mac_header(skb), 16, true);
1209 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1,
1210 skb->data, 32, true);
1211 #endif
1212
1213 napi_gro_receive(napi, skb);
1214 }
1215
1216 gem_rx_refill(queue);
1217
1218 return count;
1219 }
1220
1221 static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi,
1222 unsigned int first_frag, unsigned int last_frag)
1223 {
1224 unsigned int len;
1225 unsigned int frag;
1226 unsigned int offset;
1227 struct sk_buff *skb;
1228 struct macb_dma_desc *desc;
1229 struct macb *bp = queue->bp;
1230
1231 desc = macb_rx_desc(queue, last_frag);
1232 len = desc->ctrl & bp->rx_frm_len_mask;
1233
1234 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",
1235 macb_rx_ring_wrap(bp, first_frag),
1236 macb_rx_ring_wrap(bp, last_frag), len);
1237
1238 /* The ethernet header starts NET_IP_ALIGN bytes into the
1239 * first buffer. Since the header is 14 bytes, this makes the
1240 * payload word-aligned.
1241 *
1242 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy
1243 * the two padding bytes into the skb so that we avoid hitting
1244 * the slowpath in memcpy(), and pull them off afterwards.
1245 */
1246 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);
1247 if (!skb) {
1248 bp->dev->stats.rx_dropped++;
1249 for (frag = first_frag; ; frag++) {
1250 desc = macb_rx_desc(queue, frag);
1251 desc->addr &= ~MACB_BIT(RX_USED);
1252 if (frag == last_frag)
1253 break;
1254 }
1255
1256 /* Make descriptor updates visible to hardware */
1257 wmb();
1258
1259 return 1;
1260 }
1261
1262 offset = 0;
1263 len += NET_IP_ALIGN;
1264 skb_checksum_none_assert(skb);
1265 skb_put(skb, len);
1266
1267 for (frag = first_frag; ; frag++) {
1268 unsigned int frag_len = bp->rx_buffer_size;
1269
1270 if (offset + frag_len > len) {
1271 if (unlikely(frag != last_frag)) {
1272 dev_kfree_skb_any(skb);
1273 return -1;
1274 }
1275 frag_len = len - offset;
1276 }
1277 skb_copy_to_linear_data_offset(skb, offset,
1278 macb_rx_buffer(queue, frag),
1279 frag_len);
1280 offset += bp->rx_buffer_size;
1281 desc = macb_rx_desc(queue, frag);
1282 desc->addr &= ~MACB_BIT(RX_USED);
1283
1284 if (frag == last_frag)
1285 break;
1286 }
1287
1288 /* Make descriptor updates visible to hardware */
1289 wmb();
1290
1291 __skb_pull(skb, NET_IP_ALIGN);
1292 skb->protocol = eth_type_trans(skb, bp->dev);
1293
1294 bp->dev->stats.rx_packets++;
1295 bp->dev->stats.rx_bytes += skb->len;
1296 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",
1297 skb->len, skb->csum);
1298 napi_gro_receive(napi, skb);
1299
1300 return 0;
1301 }
1302
1303 static inline void macb_init_rx_ring(struct macb_queue *queue)
1304 {
1305 struct macb *bp = queue->bp;
1306 dma_addr_t addr;
1307 struct macb_dma_desc *desc = NULL;
1308 int i;
1309
1310 addr = queue->rx_buffers_dma;
1311 for (i = 0; i < bp->rx_ring_size; i++) {
1312 desc = macb_rx_desc(queue, i);
1313 macb_set_addr(bp, desc, addr);
1314 desc->ctrl = 0;
1315 addr += bp->rx_buffer_size;
1316 }
1317 desc->addr |= MACB_BIT(RX_WRAP);
1318 queue->rx_tail = 0;
1319 }
1320
1321 static int macb_rx(struct macb_queue *queue, struct napi_struct *napi,
1322 int budget)
1323 {
1324 struct macb *bp = queue->bp;
1325 bool reset_rx_queue = false;
1326 int received = 0;
1327 unsigned int tail;
1328 int first_frag = -1;
1329
1330 for (tail = queue->rx_tail; budget > 0; tail++) {
1331 struct macb_dma_desc *desc = macb_rx_desc(queue, tail);
1332 u32 ctrl;
1333
1334 /* Make hw descriptor updates visible to CPU */
1335 rmb();
1336
1337 if (!(desc->addr & MACB_BIT(RX_USED)))
1338 break;
1339
1340 /* Ensure ctrl is at least as up-to-date as addr */
1341 dma_rmb();
1342
1343 ctrl = desc->ctrl;
1344
1345 if (ctrl & MACB_BIT(RX_SOF)) {
1346 if (first_frag != -1)
1347 discard_partial_frame(queue, first_frag, tail);
1348 first_frag = tail;
1349 }
1350
1351 if (ctrl & MACB_BIT(RX_EOF)) {
1352 int dropped;
1353
1354 if (unlikely(first_frag == -1)) {
1355 reset_rx_queue = true;
1356 continue;
1357 }
1358
1359 dropped = macb_rx_frame(queue, napi, first_frag, tail);
1360 first_frag = -1;
1361 if (unlikely(dropped < 0)) {
1362 reset_rx_queue = true;
1363 continue;
1364 }
1365 if (!dropped) {
1366 received++;
1367 budget--;
1368 }
1369 }
1370 }
1371
1372 if (unlikely(reset_rx_queue)) {
1373 unsigned long flags;
1374 u32 ctrl;
1375
1376 netdev_err(bp->dev, "RX queue corruption: reset it\n");
1377
1378 spin_lock_irqsave(&bp->lock, flags);
1379
1380 ctrl = macb_readl(bp, NCR);
1381 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1382
1383 macb_init_rx_ring(queue);
1384 queue_writel(queue, RBQP, queue->rx_ring_dma);
1385
1386 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1387
1388 spin_unlock_irqrestore(&bp->lock, flags);
1389 return received;
1390 }
1391
1392 if (first_frag != -1)
1393 queue->rx_tail = first_frag;
1394 else
1395 queue->rx_tail = tail;
1396
1397 return received;
1398 }
1399
1400 static int macb_poll(struct napi_struct *napi, int budget)
1401 {
1402 struct macb_queue *queue = container_of(napi, struct macb_queue, napi);
1403 struct macb *bp = queue->bp;
1404 int work_done;
1405 u32 status;
1406
1407 status = macb_readl(bp, RSR);
1408 macb_writel(bp, RSR, status);
1409
1410 netdev_vdbg(bp->dev, "poll: status = %08lx, budget = %d\n",
1411 (unsigned long)status, budget);
1412
1413 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);
1414 if (work_done < budget) {
1415 napi_complete_done(napi, work_done);
1416
1417 /* Packets received while interrupts were disabled */
1418 status = macb_readl(bp, RSR);
1419 if (status) {
1420 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1421 queue_writel(queue, ISR, MACB_BIT(RCOMP));
1422 napi_reschedule(napi);
1423 } else {
1424 queue_writel(queue, IER, bp->rx_intr_mask);
1425 }
1426 }
1427
1428 /* TODO: Handle errors */
1429
1430 return work_done;
1431 }
1432
1433 static void macb_hresp_error_task(unsigned long data)
1434 {
1435 struct macb *bp = (struct macb *)data;
1436 struct net_device *dev = bp->dev;
1437 struct macb_queue *queue = bp->queues;
1438 unsigned int q;
1439 u32 ctrl;
1440
1441 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
1442 queue_writel(queue, IDR, bp->rx_intr_mask |
1443 MACB_TX_INT_FLAGS |
1444 MACB_BIT(HRESP));
1445 }
1446 ctrl = macb_readl(bp, NCR);
1447 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
1448 macb_writel(bp, NCR, ctrl);
1449
1450 netif_tx_stop_all_queues(dev);
1451 netif_carrier_off(dev);
1452
1453 bp->macbgem_ops.mog_init_rings(bp);
1454
1455 /* Initialize TX and RX buffers */
1456 macb_init_buffers(bp);
1457
1458 /* Enable interrupts */
1459 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
1460 queue_writel(queue, IER,
1461 bp->rx_intr_mask |
1462 MACB_TX_INT_FLAGS |
1463 MACB_BIT(HRESP));
1464
1465 ctrl |= MACB_BIT(RE) | MACB_BIT(TE);
1466 macb_writel(bp, NCR, ctrl);
1467
1468 netif_carrier_on(dev);
1469 netif_tx_start_all_queues(dev);
1470 }
1471
1472 static void macb_tx_restart(struct macb_queue *queue)
1473 {
1474 unsigned int head = queue->tx_head;
1475 unsigned int tail = queue->tx_tail;
1476 struct macb *bp = queue->bp;
1477
1478 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1479 queue_writel(queue, ISR, MACB_BIT(TXUBR));
1480
1481 if (head == tail)
1482 return;
1483
1484 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1485 }
1486
1487 static irqreturn_t macb_interrupt(int irq, void *dev_id)
1488 {
1489 struct macb_queue *queue = dev_id;
1490 struct macb *bp = queue->bp;
1491 struct net_device *dev = bp->dev;
1492 u32 status, ctrl;
1493
1494 status = queue_readl(queue, ISR);
1495
1496 if (unlikely(!status))
1497 return IRQ_NONE;
1498
1499 spin_lock(&bp->lock);
1500
1501 while (status) {
1502 /* close possible race with dev_close */
1503 if (unlikely(!netif_running(dev))) {
1504 queue_writel(queue, IDR, -1);
1505 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1506 queue_writel(queue, ISR, -1);
1507 break;
1508 }
1509
1510 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",
1511 (unsigned int)(queue - bp->queues),
1512 (unsigned long)status);
1513
1514 if (status & bp->rx_intr_mask) {
1515 /* There's no point taking any more interrupts
1516 * until we have processed the buffers. The
1517 * scheduling call may fail if the poll routine
1518 * is already scheduled, so disable interrupts
1519 * now.
1520 */
1521 queue_writel(queue, IDR, bp->rx_intr_mask);
1522 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1523 queue_writel(queue, ISR, MACB_BIT(RCOMP));
1524
1525 if (napi_schedule_prep(&queue->napi)) {
1526 netdev_vdbg(bp->dev, "scheduling RX softirq\n");
1527 __napi_schedule(&queue->napi);
1528 }
1529 }
1530
1531 if (unlikely(status & (MACB_TX_ERR_FLAGS))) {
1532 queue_writel(queue, IDR, MACB_TX_INT_FLAGS);
1533 schedule_work(&queue->tx_error_task);
1534
1535 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1536 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS);
1537
1538 break;
1539 }
1540
1541 if (status & MACB_BIT(TCOMP))
1542 macb_tx_interrupt(queue);
1543
1544 if (status & MACB_BIT(TXUBR))
1545 macb_tx_restart(queue);
1546
1547 /* Link change detection isn't possible with RMII, so we'll
1548 * add that if/when we get our hands on a full-blown MII PHY.
1549 */
1550
1551 /* There is a hardware issue under heavy load where DMA can
1552 * stop, this causes endless "used buffer descriptor read"
1553 * interrupts but it can be cleared by re-enabling RX. See
1554 * the at91rm9200 manual, section 41.3.1 or the Zynq manual
1555 * section 16.7.4 for details. RXUBR is only enabled for
1556 * these two versions.
1557 */
1558 if (status & MACB_BIT(RXUBR)) {
1559 ctrl = macb_readl(bp, NCR);
1560 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));
1561 wmb();
1562 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));
1563
1564 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1565 queue_writel(queue, ISR, MACB_BIT(RXUBR));
1566 }
1567
1568 if (status & MACB_BIT(ISR_ROVR)) {
1569 /* We missed at least one packet */
1570 if (macb_is_gem(bp))
1571 bp->hw_stats.gem.rx_overruns++;
1572 else
1573 bp->hw_stats.macb.rx_overruns++;
1574
1575 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1576 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR));
1577 }
1578
1579 if (status & MACB_BIT(HRESP)) {
1580 tasklet_schedule(&bp->hresp_err_tasklet);
1581 netdev_err(dev, "DMA bus error: HRESP not OK\n");
1582
1583 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
1584 queue_writel(queue, ISR, MACB_BIT(HRESP));
1585 }
1586 status = queue_readl(queue, ISR);
1587 }
1588
1589 spin_unlock(&bp->lock);
1590
1591 return IRQ_HANDLED;
1592 }
1593
1594 #ifdef CONFIG_NET_POLL_CONTROLLER
1595 /* Polling receive - used by netconsole and other diagnostic tools
1596 * to allow network i/o with interrupts disabled.
1597 */
1598 static void macb_poll_controller(struct net_device *dev)
1599 {
1600 struct macb *bp = netdev_priv(dev);
1601 struct macb_queue *queue;
1602 unsigned long flags;
1603 unsigned int q;
1604
1605 local_irq_save(flags);
1606 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
1607 macb_interrupt(dev->irq, queue);
1608 local_irq_restore(flags);
1609 }
1610 #endif
1611
1612 static unsigned int macb_tx_map(struct macb *bp,
1613 struct macb_queue *queue,
1614 struct sk_buff *skb,
1615 unsigned int hdrlen)
1616 {
1617 dma_addr_t mapping;
1618 unsigned int len, entry, i, tx_head = queue->tx_head;
1619 struct macb_tx_skb *tx_skb = NULL;
1620 struct macb_dma_desc *desc;
1621 unsigned int offset, size, count = 0;
1622 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;
1623 unsigned int eof = 1, mss_mfs = 0;
1624 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0;
1625
1626 /* LSO */
1627 if (skb_shinfo(skb)->gso_size != 0) {
1628 if (ip_hdr(skb)->protocol == IPPROTO_UDP)
1629 /* UDP - UFO */
1630 lso_ctrl = MACB_LSO_UFO_ENABLE;
1631 else
1632 /* TCP - TSO */
1633 lso_ctrl = MACB_LSO_TSO_ENABLE;
1634 }
1635
1636 /* First, map non-paged data */
1637 len = skb_headlen(skb);
1638
1639 /* first buffer length */
1640 size = hdrlen;
1641
1642 offset = 0;
1643 while (len) {
1644 entry = macb_tx_ring_wrap(bp, tx_head);
1645 tx_skb = &queue->tx_skb[entry];
1646
1647 mapping = dma_map_single(&bp->pdev->dev,
1648 skb->data + offset,
1649 size, DMA_TO_DEVICE);
1650 if (dma_mapping_error(&bp->pdev->dev, mapping))
1651 goto dma_error;
1652
1653 /* Save info to properly release resources */
1654 tx_skb->skb = NULL;
1655 tx_skb->mapping = mapping;
1656 tx_skb->size = size;
1657 tx_skb->mapped_as_page = false;
1658
1659 len -= size;
1660 offset += size;
1661 count++;
1662 tx_head++;
1663
1664 size = min(len, bp->max_tx_length);
1665 }
1666
1667 /* Then, map paged data from fragments */
1668 for (f = 0; f < nr_frags; f++) {
1669 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
1670
1671 len = skb_frag_size(frag);
1672 offset = 0;
1673 while (len) {
1674 size = min(len, bp->max_tx_length);
1675 entry = macb_tx_ring_wrap(bp, tx_head);
1676 tx_skb = &queue->tx_skb[entry];
1677
1678 mapping = skb_frag_dma_map(&bp->pdev->dev, frag,
1679 offset, size, DMA_TO_DEVICE);
1680 if (dma_mapping_error(&bp->pdev->dev, mapping))
1681 goto dma_error;
1682
1683 /* Save info to properly release resources */
1684 tx_skb->skb = NULL;
1685 tx_skb->mapping = mapping;
1686 tx_skb->size = size;
1687 tx_skb->mapped_as_page = true;
1688
1689 len -= size;
1690 offset += size;
1691 count++;
1692 tx_head++;
1693 }
1694 }
1695
1696 /* Should never happen */
1697 if (unlikely(!tx_skb)) {
1698 netdev_err(bp->dev, "BUG! empty skb!\n");
1699 return 0;
1700 }
1701
1702 /* This is the last buffer of the frame: save socket buffer */
1703 tx_skb->skb = skb;
1704
1705 /* Update TX ring: update buffer descriptors in reverse order
1706 * to avoid race condition
1707 */
1708
1709 /* Set 'TX_USED' bit in buffer descriptor at tx_head position
1710 * to set the end of TX queue
1711 */
1712 i = tx_head;
1713 entry = macb_tx_ring_wrap(bp, i);
1714 ctrl = MACB_BIT(TX_USED);
1715 desc = macb_tx_desc(queue, entry);
1716 desc->ctrl = ctrl;
1717
1718 if (lso_ctrl) {
1719 if (lso_ctrl == MACB_LSO_UFO_ENABLE)
1720 /* include header and FCS in value given to h/w */
1721 mss_mfs = skb_shinfo(skb)->gso_size +
1722 skb_transport_offset(skb) +
1723 ETH_FCS_LEN;
1724 else /* TSO */ {
1725 mss_mfs = skb_shinfo(skb)->gso_size;
1726 /* TCP Sequence Number Source Select
1727 * can be set only for TSO
1728 */
1729 seq_ctrl = 0;
1730 }
1731 }
1732
1733 do {
1734 i--;
1735 entry = macb_tx_ring_wrap(bp, i);
1736 tx_skb = &queue->tx_skb[entry];
1737 desc = macb_tx_desc(queue, entry);
1738
1739 ctrl = (u32)tx_skb->size;
1740 if (eof) {
1741 ctrl |= MACB_BIT(TX_LAST);
1742 eof = 0;
1743 }
1744 if (unlikely(entry == (bp->tx_ring_size - 1)))
1745 ctrl |= MACB_BIT(TX_WRAP);
1746
1747 /* First descriptor is header descriptor */
1748 if (i == queue->tx_head) {
1749 ctrl |= MACB_BF(TX_LSO, lso_ctrl);
1750 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl);
1751 if ((bp->dev->features & NETIF_F_HW_CSUM) &&
1752 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl)
1753 ctrl |= MACB_BIT(TX_NOCRC);
1754 } else
1755 /* Only set MSS/MFS on payload descriptors
1756 * (second or later descriptor)
1757 */
1758 ctrl |= MACB_BF(MSS_MFS, mss_mfs);
1759
1760 /* Set TX buffer descriptor */
1761 macb_set_addr(bp, desc, tx_skb->mapping);
1762 /* desc->addr must be visible to hardware before clearing
1763 * 'TX_USED' bit in desc->ctrl.
1764 */
1765 wmb();
1766 desc->ctrl = ctrl;
1767 } while (i != queue->tx_head);
1768
1769 queue->tx_head = tx_head;
1770
1771 return count;
1772
1773 dma_error:
1774 netdev_err(bp->dev, "TX DMA map failed\n");
1775
1776 for (i = queue->tx_head; i != tx_head; i++) {
1777 tx_skb = macb_tx_skb(queue, i);
1778
1779 macb_tx_unmap(bp, tx_skb);
1780 }
1781
1782 return 0;
1783 }
1784
1785 static netdev_features_t macb_features_check(struct sk_buff *skb,
1786 struct net_device *dev,
1787 netdev_features_t features)
1788 {
1789 unsigned int nr_frags, f;
1790 unsigned int hdrlen;
1791
1792 /* Validate LSO compatibility */
1793
1794 /* there is only one buffer */
1795 if (!skb_is_nonlinear(skb))
1796 return features;
1797
1798 /* length of header */
1799 hdrlen = skb_transport_offset(skb);
1800 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1801 hdrlen += tcp_hdrlen(skb);
1802
1803 /* For LSO:
1804 * When software supplies two or more payload buffers all payload buffers
1805 * apart from the last must be a multiple of 8 bytes in size.
1806 */
1807 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))
1808 return features & ~MACB_NETIF_LSO;
1809
1810 nr_frags = skb_shinfo(skb)->nr_frags;
1811 /* No need to check last fragment */
1812 nr_frags--;
1813 for (f = 0; f < nr_frags; f++) {
1814 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
1815
1816 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN))
1817 return features & ~MACB_NETIF_LSO;
1818 }
1819 return features;
1820 }
1821
1822 static inline int macb_clear_csum(struct sk_buff *skb)
1823 {
1824 /* no change for packets without checksum offloading */
1825 if (skb->ip_summed != CHECKSUM_PARTIAL)
1826 return 0;
1827
1828 /* make sure we can modify the header */
1829 if (unlikely(skb_cow_head(skb, 0)))
1830 return -1;
1831
1832 /* initialize checksum field
1833 * This is required - at least for Zynq, which otherwise calculates
1834 * wrong UDP header checksums for UDP packets with UDP data len <=2
1835 */
1836 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;
1837 return 0;
1838 }
1839
1840 static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev)
1841 {
1842 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb);
1843 int padlen = ETH_ZLEN - (*skb)->len;
1844 int headroom = skb_headroom(*skb);
1845 int tailroom = skb_tailroom(*skb);
1846 struct sk_buff *nskb;
1847 u32 fcs;
1848
1849 if (!(ndev->features & NETIF_F_HW_CSUM) ||
1850 !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||
1851 skb_shinfo(*skb)->gso_size) /* Not available for GSO */
1852 return 0;
1853
1854 if (padlen <= 0) {
1855 /* FCS could be appeded to tailroom. */
1856 if (tailroom >= ETH_FCS_LEN)
1857 goto add_fcs;
1858 /* FCS could be appeded by moving data to headroom. */
1859 else if (!cloned && headroom + tailroom >= ETH_FCS_LEN)
1860 padlen = 0;
1861 /* No room for FCS, need to reallocate skb. */
1862 else
1863 padlen = ETH_FCS_LEN;
1864 } else {
1865 /* Add room for FCS. */
1866 padlen += ETH_FCS_LEN;
1867 }
1868
1869 if (!cloned && headroom + tailroom >= padlen) {
1870 (*skb)->data = memmove((*skb)->head, (*skb)->data, (*skb)->len);
1871 skb_set_tail_pointer(*skb, (*skb)->len);
1872 } else {
1873 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC);
1874 if (!nskb)
1875 return -ENOMEM;
1876
1877 dev_consume_skb_any(*skb);
1878 *skb = nskb;
1879 }
1880
1881 if (padlen > ETH_FCS_LEN)
1882 skb_put_zero(*skb, padlen - ETH_FCS_LEN);
1883
1884 add_fcs:
1885 /* set FCS to packet */
1886 fcs = crc32_le(~0, (*skb)->data, (*skb)->len);
1887 fcs = ~fcs;
1888
1889 skb_put_u8(*skb, fcs & 0xff);
1890 skb_put_u8(*skb, (fcs >> 8) & 0xff);
1891 skb_put_u8(*skb, (fcs >> 16) & 0xff);
1892 skb_put_u8(*skb, (fcs >> 24) & 0xff);
1893
1894 return 0;
1895 }
1896
1897 static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev)
1898 {
1899 u16 queue_index = skb_get_queue_mapping(skb);
1900 struct macb *bp = netdev_priv(dev);
1901 struct macb_queue *queue = &bp->queues[queue_index];
1902 unsigned long flags;
1903 unsigned int desc_cnt, nr_frags, frag_size, f;
1904 unsigned int hdrlen;
1905 bool is_lso, is_udp = 0;
1906 netdev_tx_t ret = NETDEV_TX_OK;
1907
1908 if (macb_clear_csum(skb)) {
1909 dev_kfree_skb_any(skb);
1910 return ret;
1911 }
1912
1913 if (macb_pad_and_fcs(&skb, dev)) {
1914 dev_kfree_skb_any(skb);
1915 return ret;
1916 }
1917
1918 is_lso = (skb_shinfo(skb)->gso_size != 0);
1919
1920 if (is_lso) {
1921 is_udp = !!(ip_hdr(skb)->protocol == IPPROTO_UDP);
1922
1923 /* length of headers */
1924 if (is_udp)
1925 /* only queue eth + ip headers separately for UDP */
1926 hdrlen = skb_transport_offset(skb);
1927 else
1928 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
1929 if (skb_headlen(skb) < hdrlen) {
1930 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");
1931 /* if this is required, would need to copy to single buffer */
1932 return NETDEV_TX_BUSY;
1933 }
1934 } else
1935 hdrlen = min(skb_headlen(skb), bp->max_tx_length);
1936
1937 #if defined(DEBUG) && defined(VERBOSE_DEBUG)
1938 netdev_vdbg(bp->dev,
1939 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n",
1940 queue_index, skb->len, skb->head, skb->data,
1941 skb_tail_pointer(skb), skb_end_pointer(skb));
1942 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1,
1943 skb->data, 16, true);
1944 #endif
1945
1946 /* Count how many TX buffer descriptors are needed to send this
1947 * socket buffer: skb fragments of jumbo frames may need to be
1948 * split into many buffer descriptors.
1949 */
1950 if (is_lso && (skb_headlen(skb) > hdrlen))
1951 /* extra header descriptor if also payload in first buffer */
1952 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;
1953 else
1954 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);
1955 nr_frags = skb_shinfo(skb)->nr_frags;
1956 for (f = 0; f < nr_frags; f++) {
1957 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);
1958 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);
1959 }
1960
1961 spin_lock_irqsave(&bp->lock, flags);
1962
1963 /* This is a hard error, log it. */
1964 if (CIRC_SPACE(queue->tx_head, queue->tx_tail,
1965 bp->tx_ring_size) < desc_cnt) {
1966 netif_stop_subqueue(dev, queue_index);
1967 spin_unlock_irqrestore(&bp->lock, flags);
1968 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",
1969 queue->tx_head, queue->tx_tail);
1970 return NETDEV_TX_BUSY;
1971 }
1972
1973 /* Map socket buffer for DMA transfer */
1974 if (!macb_tx_map(bp, queue, skb, hdrlen)) {
1975 dev_kfree_skb_any(skb);
1976 goto unlock;
1977 }
1978
1979 /* Make newly initialized descriptor visible to hardware */
1980 wmb();
1981 skb_tx_timestamp(skb);
1982
1983 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));
1984
1985 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)
1986 netif_stop_subqueue(dev, queue_index);
1987
1988 unlock:
1989 spin_unlock_irqrestore(&bp->lock, flags);
1990
1991 return ret;
1992 }
1993
1994 static void macb_init_rx_buffer_size(struct macb *bp, size_t size)
1995 {
1996 if (!macb_is_gem(bp)) {
1997 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;
1998 } else {
1999 bp->rx_buffer_size = size;
2000
2001 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {
2002 netdev_dbg(bp->dev,
2003 "RX buffer must be multiple of %d bytes, expanding\n",
2004 RX_BUFFER_MULTIPLE);
2005 bp->rx_buffer_size =
2006 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);
2007 }
2008 }
2009
2010 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",
2011 bp->dev->mtu, bp->rx_buffer_size);
2012 }
2013
2014 static void gem_free_rx_buffers(struct macb *bp)
2015 {
2016 struct sk_buff *skb;
2017 struct macb_dma_desc *desc;
2018 struct macb_queue *queue;
2019 dma_addr_t addr;
2020 unsigned int q;
2021 int i;
2022
2023 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2024 if (!queue->rx_skbuff)
2025 continue;
2026
2027 for (i = 0; i < bp->rx_ring_size; i++) {
2028 skb = queue->rx_skbuff[i];
2029
2030 if (!skb)
2031 continue;
2032
2033 desc = macb_rx_desc(queue, i);
2034 addr = macb_get_addr(bp, desc);
2035
2036 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,
2037 DMA_FROM_DEVICE);
2038 dev_kfree_skb_any(skb);
2039 skb = NULL;
2040 }
2041
2042 kfree(queue->rx_skbuff);
2043 queue->rx_skbuff = NULL;
2044 }
2045 }
2046
2047 static void macb_free_rx_buffers(struct macb *bp)
2048 {
2049 struct macb_queue *queue = &bp->queues[0];
2050
2051 if (queue->rx_buffers) {
2052 dma_free_coherent(&bp->pdev->dev,
2053 bp->rx_ring_size * bp->rx_buffer_size,
2054 queue->rx_buffers, queue->rx_buffers_dma);
2055 queue->rx_buffers = NULL;
2056 }
2057 }
2058
2059 static void macb_free_consistent(struct macb *bp)
2060 {
2061 struct macb_queue *queue;
2062 unsigned int q;
2063 int size;
2064
2065 bp->macbgem_ops.mog_free_rx_buffers(bp);
2066
2067 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2068 kfree(queue->tx_skb);
2069 queue->tx_skb = NULL;
2070 if (queue->tx_ring) {
2071 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2072 dma_free_coherent(&bp->pdev->dev, size,
2073 queue->tx_ring, queue->tx_ring_dma);
2074 queue->tx_ring = NULL;
2075 }
2076 if (queue->rx_ring) {
2077 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2078 dma_free_coherent(&bp->pdev->dev, size,
2079 queue->rx_ring, queue->rx_ring_dma);
2080 queue->rx_ring = NULL;
2081 }
2082 }
2083 }
2084
2085 static int gem_alloc_rx_buffers(struct macb *bp)
2086 {
2087 struct macb_queue *queue;
2088 unsigned int q;
2089 int size;
2090
2091 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2092 size = bp->rx_ring_size * sizeof(struct sk_buff *);
2093 queue->rx_skbuff = kzalloc(size, GFP_KERNEL);
2094 if (!queue->rx_skbuff)
2095 return -ENOMEM;
2096 else
2097 netdev_dbg(bp->dev,
2098 "Allocated %d RX struct sk_buff entries at %p\n",
2099 bp->rx_ring_size, queue->rx_skbuff);
2100 }
2101 return 0;
2102 }
2103
2104 static int macb_alloc_rx_buffers(struct macb *bp)
2105 {
2106 struct macb_queue *queue = &bp->queues[0];
2107 int size;
2108
2109 size = bp->rx_ring_size * bp->rx_buffer_size;
2110 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,
2111 &queue->rx_buffers_dma, GFP_KERNEL);
2112 if (!queue->rx_buffers)
2113 return -ENOMEM;
2114
2115 netdev_dbg(bp->dev,
2116 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n",
2117 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);
2118 return 0;
2119 }
2120
2121 static int macb_alloc_consistent(struct macb *bp)
2122 {
2123 struct macb_queue *queue;
2124 unsigned int q;
2125 int size;
2126
2127 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2128 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;
2129 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2130 &queue->tx_ring_dma,
2131 GFP_KERNEL);
2132 if (!queue->tx_ring)
2133 goto out_err;
2134 netdev_dbg(bp->dev,
2135 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n",
2136 q, size, (unsigned long)queue->tx_ring_dma,
2137 queue->tx_ring);
2138
2139 size = bp->tx_ring_size * sizeof(struct macb_tx_skb);
2140 queue->tx_skb = kmalloc(size, GFP_KERNEL);
2141 if (!queue->tx_skb)
2142 goto out_err;
2143
2144 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;
2145 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,
2146 &queue->rx_ring_dma, GFP_KERNEL);
2147 if (!queue->rx_ring)
2148 goto out_err;
2149 netdev_dbg(bp->dev,
2150 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",
2151 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);
2152 }
2153 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))
2154 goto out_err;
2155
2156 return 0;
2157
2158 out_err:
2159 macb_free_consistent(bp);
2160 return -ENOMEM;
2161 }
2162
2163 static void gem_init_rings(struct macb *bp)
2164 {
2165 struct macb_queue *queue;
2166 struct macb_dma_desc *desc = NULL;
2167 unsigned int q;
2168 int i;
2169
2170 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2171 for (i = 0; i < bp->tx_ring_size; i++) {
2172 desc = macb_tx_desc(queue, i);
2173 macb_set_addr(bp, desc, 0);
2174 desc->ctrl = MACB_BIT(TX_USED);
2175 }
2176 desc->ctrl |= MACB_BIT(TX_WRAP);
2177 queue->tx_head = 0;
2178 queue->tx_tail = 0;
2179
2180 queue->rx_tail = 0;
2181 queue->rx_prepared_head = 0;
2182
2183 gem_rx_refill(queue);
2184 }
2185
2186 }
2187
2188 static void macb_init_rings(struct macb *bp)
2189 {
2190 int i;
2191 struct macb_dma_desc *desc = NULL;
2192
2193 macb_init_rx_ring(&bp->queues[0]);
2194
2195 for (i = 0; i < bp->tx_ring_size; i++) {
2196 desc = macb_tx_desc(&bp->queues[0], i);
2197 macb_set_addr(bp, desc, 0);
2198 desc->ctrl = MACB_BIT(TX_USED);
2199 }
2200 bp->queues[0].tx_head = 0;
2201 bp->queues[0].tx_tail = 0;
2202 desc->ctrl |= MACB_BIT(TX_WRAP);
2203 }
2204
2205 static void macb_reset_hw(struct macb *bp)
2206 {
2207 struct macb_queue *queue;
2208 unsigned int q;
2209 u32 ctrl = macb_readl(bp, NCR);
2210
2211 /* Disable RX and TX (XXX: Should we halt the transmission
2212 * more gracefully?)
2213 */
2214 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));
2215
2216 /* Clear the stats registers (XXX: Update stats first?) */
2217 ctrl |= MACB_BIT(CLRSTAT);
2218
2219 macb_writel(bp, NCR, ctrl);
2220
2221 /* Clear all status flags */
2222 macb_writel(bp, TSR, -1);
2223 macb_writel(bp, RSR, -1);
2224
2225 /* Disable all interrupts */
2226 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2227 queue_writel(queue, IDR, -1);
2228 queue_readl(queue, ISR);
2229 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)
2230 queue_writel(queue, ISR, -1);
2231 }
2232 }
2233
2234 static u32 gem_mdc_clk_div(struct macb *bp)
2235 {
2236 u32 config;
2237 unsigned long pclk_hz = clk_get_rate(bp->pclk);
2238
2239 if (pclk_hz <= 20000000)
2240 config = GEM_BF(CLK, GEM_CLK_DIV8);
2241 else if (pclk_hz <= 40000000)
2242 config = GEM_BF(CLK, GEM_CLK_DIV16);
2243 else if (pclk_hz <= 80000000)
2244 config = GEM_BF(CLK, GEM_CLK_DIV32);
2245 else if (pclk_hz <= 120000000)
2246 config = GEM_BF(CLK, GEM_CLK_DIV48);
2247 else if (pclk_hz <= 160000000)
2248 config = GEM_BF(CLK, GEM_CLK_DIV64);
2249 else
2250 config = GEM_BF(CLK, GEM_CLK_DIV96);
2251
2252 return config;
2253 }
2254
2255 static u32 macb_mdc_clk_div(struct macb *bp)
2256 {
2257 u32 config;
2258 unsigned long pclk_hz;
2259
2260 if (macb_is_gem(bp))
2261 return gem_mdc_clk_div(bp);
2262
2263 pclk_hz = clk_get_rate(bp->pclk);
2264 if (pclk_hz <= 20000000)
2265 config = MACB_BF(CLK, MACB_CLK_DIV8);
2266 else if (pclk_hz <= 40000000)
2267 config = MACB_BF(CLK, MACB_CLK_DIV16);
2268 else if (pclk_hz <= 80000000)
2269 config = MACB_BF(CLK, MACB_CLK_DIV32);
2270 else
2271 config = MACB_BF(CLK, MACB_CLK_DIV64);
2272
2273 return config;
2274 }
2275
2276 /* Get the DMA bus width field of the network configuration register that we
2277 * should program. We find the width from decoding the design configuration
2278 * register to find the maximum supported data bus width.
2279 */
2280 static u32 macb_dbw(struct macb *bp)
2281 {
2282 if (!macb_is_gem(bp))
2283 return 0;
2284
2285 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) {
2286 case 4:
2287 return GEM_BF(DBW, GEM_DBW128);
2288 case 2:
2289 return GEM_BF(DBW, GEM_DBW64);
2290 case 1:
2291 default:
2292 return GEM_BF(DBW, GEM_DBW32);
2293 }
2294 }
2295
2296 /* Configure the receive DMA engine
2297 * - use the correct receive buffer size
2298 * - set best burst length for DMA operations
2299 * (if not supported by FIFO, it will fallback to default)
2300 * - set both rx/tx packet buffers to full memory size
2301 * These are configurable parameters for GEM.
2302 */
2303 static void macb_configure_dma(struct macb *bp)
2304 {
2305 struct macb_queue *queue;
2306 u32 buffer_size;
2307 unsigned int q;
2308 u32 dmacfg;
2309
2310 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;
2311 if (macb_is_gem(bp)) {
2312 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);
2313 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2314 if (q)
2315 queue_writel(queue, RBQS, buffer_size);
2316 else
2317 dmacfg |= GEM_BF(RXBS, buffer_size);
2318 }
2319 if (bp->dma_burst_length)
2320 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);
2321 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);
2322 dmacfg &= ~GEM_BIT(ENDIA_PKT);
2323
2324 if (bp->native_io)
2325 dmacfg &= ~GEM_BIT(ENDIA_DESC);
2326 else
2327 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */
2328
2329 if (bp->dev->features & NETIF_F_HW_CSUM)
2330 dmacfg |= GEM_BIT(TXCOEN);
2331 else
2332 dmacfg &= ~GEM_BIT(TXCOEN);
2333
2334 dmacfg &= ~GEM_BIT(ADDR64);
2335 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
2336 if (bp->hw_dma_cap & HW_DMA_CAP_64B)
2337 dmacfg |= GEM_BIT(ADDR64);
2338 #endif
2339 #ifdef CONFIG_MACB_USE_HWSTAMP
2340 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)
2341 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT);
2342 #endif
2343 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",
2344 dmacfg);
2345 gem_writel(bp, DMACFG, dmacfg);
2346 }
2347 }
2348
2349 static void macb_init_hw(struct macb *bp)
2350 {
2351 u32 config;
2352
2353 macb_reset_hw(bp);
2354 macb_set_hwaddr(bp);
2355
2356 config = macb_mdc_clk_div(bp);
2357 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */
2358 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */
2359 if (bp->caps & MACB_CAPS_JUMBO)
2360 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */
2361 else
2362 config |= MACB_BIT(BIG); /* Receive oversized frames */
2363 if (bp->dev->flags & IFF_PROMISC)
2364 config |= MACB_BIT(CAF); /* Copy All Frames */
2365 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)
2366 config |= GEM_BIT(RXCOEN);
2367 if (!(bp->dev->flags & IFF_BROADCAST))
2368 config |= MACB_BIT(NBC); /* No BroadCast */
2369 config |= macb_dbw(bp);
2370 macb_writel(bp, NCFGR, config);
2371 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)
2372 gem_writel(bp, JML, bp->jumbo_max_len);
2373 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;
2374 if (bp->caps & MACB_CAPS_JUMBO)
2375 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;
2376
2377 macb_configure_dma(bp);
2378 }
2379
2380 /* The hash address register is 64 bits long and takes up two
2381 * locations in the memory map. The least significant bits are stored
2382 * in EMAC_HSL and the most significant bits in EMAC_HSH.
2383 *
2384 * The unicast hash enable and the multicast hash enable bits in the
2385 * network configuration register enable the reception of hash matched
2386 * frames. The destination address is reduced to a 6 bit index into
2387 * the 64 bit hash register using the following hash function. The
2388 * hash function is an exclusive or of every sixth bit of the
2389 * destination address.
2390 *
2391 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47]
2392 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46]
2393 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45]
2394 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44]
2395 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43]
2396 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42]
2397 *
2398 * da[0] represents the least significant bit of the first byte
2399 * received, that is, the multicast/unicast indicator, and da[47]
2400 * represents the most significant bit of the last byte received. If
2401 * the hash index, hi[n], points to a bit that is set in the hash
2402 * register then the frame will be matched according to whether the
2403 * frame is multicast or unicast. A multicast match will be signalled
2404 * if the multicast hash enable bit is set, da[0] is 1 and the hash
2405 * index points to a bit set in the hash register. A unicast match
2406 * will be signalled if the unicast hash enable bit is set, da[0] is 0
2407 * and the hash index points to a bit set in the hash register. To
2408 * receive all multicast frames, the hash register should be set with
2409 * all ones and the multicast hash enable bit should be set in the
2410 * network configuration register.
2411 */
2412
2413 static inline int hash_bit_value(int bitnr, __u8 *addr)
2414 {
2415 if (addr[bitnr / 8] & (1 << (bitnr % 8)))
2416 return 1;
2417 return 0;
2418 }
2419
2420 /* Return the hash index value for the specified address. */
2421 static int hash_get_index(__u8 *addr)
2422 {
2423 int i, j, bitval;
2424 int hash_index = 0;
2425
2426 for (j = 0; j < 6; j++) {
2427 for (i = 0, bitval = 0; i < 8; i++)
2428 bitval ^= hash_bit_value(i * 6 + j, addr);
2429
2430 hash_index |= (bitval << j);
2431 }
2432
2433 return hash_index;
2434 }
2435
2436 /* Add multicast addresses to the internal multicast-hash table. */
2437 static void macb_sethashtable(struct net_device *dev)
2438 {
2439 struct netdev_hw_addr *ha;
2440 unsigned long mc_filter[2];
2441 unsigned int bitnr;
2442 struct macb *bp = netdev_priv(dev);
2443
2444 mc_filter[0] = 0;
2445 mc_filter[1] = 0;
2446
2447 netdev_for_each_mc_addr(ha, dev) {
2448 bitnr = hash_get_index(ha->addr);
2449 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31);
2450 }
2451
2452 macb_or_gem_writel(bp, HRB, mc_filter[0]);
2453 macb_or_gem_writel(bp, HRT, mc_filter[1]);
2454 }
2455
2456 /* Enable/Disable promiscuous and multicast modes. */
2457 static void macb_set_rx_mode(struct net_device *dev)
2458 {
2459 unsigned long cfg;
2460 struct macb *bp = netdev_priv(dev);
2461
2462 cfg = macb_readl(bp, NCFGR);
2463
2464 if (dev->flags & IFF_PROMISC) {
2465 /* Enable promiscuous mode */
2466 cfg |= MACB_BIT(CAF);
2467
2468 /* Disable RX checksum offload */
2469 if (macb_is_gem(bp))
2470 cfg &= ~GEM_BIT(RXCOEN);
2471 } else {
2472 /* Disable promiscuous mode */
2473 cfg &= ~MACB_BIT(CAF);
2474
2475 /* Enable RX checksum offload only if requested */
2476 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)
2477 cfg |= GEM_BIT(RXCOEN);
2478 }
2479
2480 if (dev->flags & IFF_ALLMULTI) {
2481 /* Enable all multicast mode */
2482 macb_or_gem_writel(bp, HRB, -1);
2483 macb_or_gem_writel(bp, HRT, -1);
2484 cfg |= MACB_BIT(NCFGR_MTI);
2485 } else if (!netdev_mc_empty(dev)) {
2486 /* Enable specific multicasts */
2487 macb_sethashtable(dev);
2488 cfg |= MACB_BIT(NCFGR_MTI);
2489 } else if (dev->flags & (~IFF_ALLMULTI)) {
2490 /* Disable all multicast mode */
2491 macb_or_gem_writel(bp, HRB, 0);
2492 macb_or_gem_writel(bp, HRT, 0);
2493 cfg &= ~MACB_BIT(NCFGR_MTI);
2494 }
2495
2496 macb_writel(bp, NCFGR, cfg);
2497 }
2498
2499 static int macb_open(struct net_device *dev)
2500 {
2501 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;
2502 struct macb *bp = netdev_priv(dev);
2503 struct macb_queue *queue;
2504 unsigned int q;
2505 int err;
2506
2507 netdev_dbg(bp->dev, "open\n");
2508
2509 err = pm_runtime_get_sync(&bp->pdev->dev);
2510 if (err < 0)
2511 goto pm_exit;
2512
2513 /* RX buffers initialization */
2514 macb_init_rx_buffer_size(bp, bufsz);
2515
2516 err = macb_alloc_consistent(bp);
2517 if (err) {
2518 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n",
2519 err);
2520 goto pm_exit;
2521 }
2522
2523 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2524 napi_enable(&queue->napi);
2525
2526 macb_init_hw(bp);
2527
2528 err = macb_phylink_connect(bp);
2529 if (err)
2530 goto pm_exit;
2531
2532 netif_tx_start_all_queues(dev);
2533
2534 if (bp->ptp_info)
2535 bp->ptp_info->ptp_init(dev);
2536
2537 pm_exit:
2538 if (err) {
2539 pm_runtime_put_sync(&bp->pdev->dev);
2540 return err;
2541 }
2542 return 0;
2543 }
2544
2545 static int macb_close(struct net_device *dev)
2546 {
2547 struct macb *bp = netdev_priv(dev);
2548 struct macb_queue *queue;
2549 unsigned long flags;
2550 unsigned int q;
2551
2552 netif_tx_stop_all_queues(dev);
2553
2554 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2555 napi_disable(&queue->napi);
2556
2557 phylink_stop(bp->phylink);
2558 phylink_disconnect_phy(bp->phylink);
2559
2560 spin_lock_irqsave(&bp->lock, flags);
2561 macb_reset_hw(bp);
2562 netif_carrier_off(dev);
2563 spin_unlock_irqrestore(&bp->lock, flags);
2564
2565 macb_free_consistent(bp);
2566
2567 if (bp->ptp_info)
2568 bp->ptp_info->ptp_remove(dev);
2569
2570 pm_runtime_put(&bp->pdev->dev);
2571
2572 return 0;
2573 }
2574
2575 static int macb_change_mtu(struct net_device *dev, int new_mtu)
2576 {
2577 if (netif_running(dev))
2578 return -EBUSY;
2579
2580 dev->mtu = new_mtu;
2581
2582 return 0;
2583 }
2584
2585 static void gem_update_stats(struct macb *bp)
2586 {
2587 struct macb_queue *queue;
2588 unsigned int i, q, idx;
2589 unsigned long *stat;
2590
2591 u32 *p = &bp->hw_stats.gem.tx_octets_31_0;
2592
2593 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) {
2594 u32 offset = gem_statistics[i].offset;
2595 u64 val = bp->macb_reg_readl(bp, offset);
2596
2597 bp->ethtool_stats[i] += val;
2598 *p += val;
2599
2600 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) {
2601 /* Add GEM_OCTTXH, GEM_OCTRXH */
2602 val = bp->macb_reg_readl(bp, offset + 4);
2603 bp->ethtool_stats[i] += ((u64)val) << 32;
2604 *(++p) += val;
2605 }
2606 }
2607
2608 idx = GEM_STATS_LEN;
2609 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)
2610 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)
2611 bp->ethtool_stats[idx++] = *stat;
2612 }
2613
2614 static struct net_device_stats *gem_get_stats(struct macb *bp)
2615 {
2616 struct gem_stats *hwstat = &bp->hw_stats.gem;
2617 struct net_device_stats *nstat = &bp->dev->stats;
2618
2619 gem_update_stats(bp);
2620
2621 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +
2622 hwstat->rx_alignment_errors +
2623 hwstat->rx_resource_errors +
2624 hwstat->rx_overruns +
2625 hwstat->rx_oversize_frames +
2626 hwstat->rx_jabbers +
2627 hwstat->rx_undersized_frames +
2628 hwstat->rx_length_field_frame_errors);
2629 nstat->tx_errors = (hwstat->tx_late_collisions +
2630 hwstat->tx_excessive_collisions +
2631 hwstat->tx_underrun +
2632 hwstat->tx_carrier_sense_errors);
2633 nstat->multicast = hwstat->rx_multicast_frames;
2634 nstat->collisions = (hwstat->tx_single_collision_frames +
2635 hwstat->tx_multiple_collision_frames +
2636 hwstat->tx_excessive_collisions);
2637 nstat->rx_length_errors = (hwstat->rx_oversize_frames +
2638 hwstat->rx_jabbers +
2639 hwstat->rx_undersized_frames +
2640 hwstat->rx_length_field_frame_errors);
2641 nstat->rx_over_errors = hwstat->rx_resource_errors;
2642 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;
2643 nstat->rx_frame_errors = hwstat->rx_alignment_errors;
2644 nstat->rx_fifo_errors = hwstat->rx_overruns;
2645 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;
2646 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;
2647 nstat->tx_fifo_errors = hwstat->tx_underrun;
2648
2649 return nstat;
2650 }
2651
2652 static void gem_get_ethtool_stats(struct net_device *dev,
2653 struct ethtool_stats *stats, u64 *data)
2654 {
2655 struct macb *bp;
2656
2657 bp = netdev_priv(dev);
2658 gem_update_stats(bp);
2659 memcpy(data, &bp->ethtool_stats, sizeof(u64)
2660 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES));
2661 }
2662
2663 static int gem_get_sset_count(struct net_device *dev, int sset)
2664 {
2665 struct macb *bp = netdev_priv(dev);
2666
2667 switch (sset) {
2668 case ETH_SS_STATS:
2669 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;
2670 default:
2671 return -EOPNOTSUPP;
2672 }
2673 }
2674
2675 static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)
2676 {
2677 char stat_string[ETH_GSTRING_LEN];
2678 struct macb *bp = netdev_priv(dev);
2679 struct macb_queue *queue;
2680 unsigned int i;
2681 unsigned int q;
2682
2683 switch (sset) {
2684 case ETH_SS_STATS:
2685 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN)
2686 memcpy(p, gem_statistics[i].stat_string,
2687 ETH_GSTRING_LEN);
2688
2689 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {
2690 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {
2691 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",
2692 q, queue_statistics[i].stat_string);
2693 memcpy(p, stat_string, ETH_GSTRING_LEN);
2694 }
2695 }
2696 break;
2697 }
2698 }
2699
2700 static struct net_device_stats *macb_get_stats(struct net_device *dev)
2701 {
2702 struct macb *bp = netdev_priv(dev);
2703 struct net_device_stats *nstat = &bp->dev->stats;
2704 struct macb_stats *hwstat = &bp->hw_stats.macb;
2705
2706 if (macb_is_gem(bp))
2707 return gem_get_stats(bp);
2708
2709 /* read stats from hardware */
2710 macb_update_stats(bp);
2711
2712 /* Convert HW stats into netdevice stats */
2713 nstat->rx_errors = (hwstat->rx_fcs_errors +
2714 hwstat->rx_align_errors +
2715 hwstat->rx_resource_errors +
2716 hwstat->rx_overruns +
2717 hwstat->rx_oversize_pkts +
2718 hwstat->rx_jabbers +
2719 hwstat->rx_undersize_pkts +
2720 hwstat->rx_length_mismatch);
2721 nstat->tx_errors = (hwstat->tx_late_cols +
2722 hwstat->tx_excessive_cols +
2723 hwstat->tx_underruns +
2724 hwstat->tx_carrier_errors +
2725 hwstat->sqe_test_errors);
2726 nstat->collisions = (hwstat->tx_single_cols +
2727 hwstat->tx_multiple_cols +
2728 hwstat->tx_excessive_cols);
2729 nstat->rx_length_errors = (hwstat->rx_oversize_pkts +
2730 hwstat->rx_jabbers +
2731 hwstat->rx_undersize_pkts +
2732 hwstat->rx_length_mismatch);
2733 nstat->rx_over_errors = hwstat->rx_resource_errors +
2734 hwstat->rx_overruns;
2735 nstat->rx_crc_errors = hwstat->rx_fcs_errors;
2736 nstat->rx_frame_errors = hwstat->rx_align_errors;
2737 nstat->rx_fifo_errors = hwstat->rx_overruns;
2738 /* XXX: What does "missed" mean? */
2739 nstat->tx_aborted_errors = hwstat->tx_excessive_cols;
2740 nstat->tx_carrier_errors = hwstat->tx_carrier_errors;
2741 nstat->tx_fifo_errors = hwstat->tx_underruns;
2742 /* Don't know about heartbeat or window errors... */
2743
2744 return nstat;
2745 }
2746
2747 static int macb_get_regs_len(struct net_device *netdev)
2748 {
2749 return MACB_GREGS_NBR * sizeof(u32);
2750 }
2751
2752 static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs,
2753 void *p)
2754 {
2755 struct macb *bp = netdev_priv(dev);
2756 unsigned int tail, head;
2757 u32 *regs_buff = p;
2758
2759 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))
2760 | MACB_GREGS_VERSION;
2761
2762 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);
2763 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);
2764
2765 regs_buff[0] = macb_readl(bp, NCR);
2766 regs_buff[1] = macb_or_gem_readl(bp, NCFGR);
2767 regs_buff[2] = macb_readl(bp, NSR);
2768 regs_buff[3] = macb_readl(bp, TSR);
2769 regs_buff[4] = macb_readl(bp, RBQP);
2770 regs_buff[5] = macb_readl(bp, TBQP);
2771 regs_buff[6] = macb_readl(bp, RSR);
2772 regs_buff[7] = macb_readl(bp, IMR);
2773
2774 regs_buff[8] = tail;
2775 regs_buff[9] = head;
2776 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);
2777 regs_buff[11] = macb_tx_dma(&bp->queues[0], head);
2778
2779 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
2780 regs_buff[12] = macb_or_gem_readl(bp, USRIO);
2781 if (macb_is_gem(bp))
2782 regs_buff[13] = gem_readl(bp, DMACFG);
2783 }
2784
2785 static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
2786 {
2787 struct macb *bp = netdev_priv(netdev);
2788
2789 wol->supported = 0;
2790 wol->wolopts = 0;
2791
2792 if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET)
2793 phylink_ethtool_get_wol(bp->phylink, wol);
2794 }
2795
2796 static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
2797 {
2798 struct macb *bp = netdev_priv(netdev);
2799 int ret;
2800
2801 ret = phylink_ethtool_set_wol(bp->phylink, wol);
2802 if (!ret)
2803 return 0;
2804
2805 if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) ||
2806 (wol->wolopts & ~WAKE_MAGIC))
2807 return -EOPNOTSUPP;
2808
2809 if (wol->wolopts & WAKE_MAGIC)
2810 bp->wol |= MACB_WOL_ENABLED;
2811 else
2812 bp->wol &= ~MACB_WOL_ENABLED;
2813
2814 device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED);
2815
2816 return 0;
2817 }
2818
2819 static int macb_get_link_ksettings(struct net_device *netdev,
2820 struct ethtool_link_ksettings *kset)
2821 {
2822 struct macb *bp = netdev_priv(netdev);
2823
2824 return phylink_ethtool_ksettings_get(bp->phylink, kset);
2825 }
2826
2827 static int macb_set_link_ksettings(struct net_device *netdev,
2828 const struct ethtool_link_ksettings *kset)
2829 {
2830 struct macb *bp = netdev_priv(netdev);
2831
2832 return phylink_ethtool_ksettings_set(bp->phylink, kset);
2833 }
2834
2835 static void macb_get_ringparam(struct net_device *netdev,
2836 struct ethtool_ringparam *ring)
2837 {
2838 struct macb *bp = netdev_priv(netdev);
2839
2840 ring->rx_max_pending = MAX_RX_RING_SIZE;
2841 ring->tx_max_pending = MAX_TX_RING_SIZE;
2842
2843 ring->rx_pending = bp->rx_ring_size;
2844 ring->tx_pending = bp->tx_ring_size;
2845 }
2846
2847 static int macb_set_ringparam(struct net_device *netdev,
2848 struct ethtool_ringparam *ring)
2849 {
2850 struct macb *bp = netdev_priv(netdev);
2851 u32 new_rx_size, new_tx_size;
2852 unsigned int reset = 0;
2853
2854 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))
2855 return -EINVAL;
2856
2857 new_rx_size = clamp_t(u32, ring->rx_pending,
2858 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE);
2859 new_rx_size = roundup_pow_of_two(new_rx_size);
2860
2861 new_tx_size = clamp_t(u32, ring->tx_pending,
2862 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE);
2863 new_tx_size = roundup_pow_of_two(new_tx_size);
2864
2865 if ((new_tx_size == bp->tx_ring_size) &&
2866 (new_rx_size == bp->rx_ring_size)) {
2867 /* nothing to do */
2868 return 0;
2869 }
2870
2871 if (netif_running(bp->dev)) {
2872 reset = 1;
2873 macb_close(bp->dev);
2874 }
2875
2876 bp->rx_ring_size = new_rx_size;
2877 bp->tx_ring_size = new_tx_size;
2878
2879 if (reset)
2880 macb_open(bp->dev);
2881
2882 return 0;
2883 }
2884
2885 #ifdef CONFIG_MACB_USE_HWSTAMP
2886 static unsigned int gem_get_tsu_rate(struct macb *bp)
2887 {
2888 struct clk *tsu_clk;
2889 unsigned int tsu_rate;
2890
2891 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");
2892 if (!IS_ERR(tsu_clk))
2893 tsu_rate = clk_get_rate(tsu_clk);
2894 /* try pclk instead */
2895 else if (!IS_ERR(bp->pclk)) {
2896 tsu_clk = bp->pclk;
2897 tsu_rate = clk_get_rate(tsu_clk);
2898 } else
2899 return -ENOTSUPP;
2900 return tsu_rate;
2901 }
2902
2903 static s32 gem_get_ptp_max_adj(void)
2904 {
2905 return 64000000;
2906 }
2907
2908 static int gem_get_ts_info(struct net_device *dev,
2909 struct ethtool_ts_info *info)
2910 {
2911 struct macb *bp = netdev_priv(dev);
2912
2913 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {
2914 ethtool_op_get_ts_info(dev, info);
2915 return 0;
2916 }
2917
2918 info->so_timestamping =
2919 SOF_TIMESTAMPING_TX_SOFTWARE |
2920 SOF_TIMESTAMPING_RX_SOFTWARE |
2921 SOF_TIMESTAMPING_SOFTWARE |
2922 SOF_TIMESTAMPING_TX_HARDWARE |
2923 SOF_TIMESTAMPING_RX_HARDWARE |
2924 SOF_TIMESTAMPING_RAW_HARDWARE;
2925 info->tx_types =
2926 (1 << HWTSTAMP_TX_ONESTEP_SYNC) |
2927 (1 << HWTSTAMP_TX_OFF) |
2928 (1 << HWTSTAMP_TX_ON);
2929 info->rx_filters =
2930 (1 << HWTSTAMP_FILTER_NONE) |
2931 (1 << HWTSTAMP_FILTER_ALL);
2932
2933 info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1;
2934
2935 return 0;
2936 }
2937
2938 static struct macb_ptp_info gem_ptp_info = {
2939 .ptp_init = gem_ptp_init,
2940 .ptp_remove = gem_ptp_remove,
2941 .get_ptp_max_adj = gem_get_ptp_max_adj,
2942 .get_tsu_rate = gem_get_tsu_rate,
2943 .get_ts_info = gem_get_ts_info,
2944 .get_hwtst = gem_get_hwtst,
2945 .set_hwtst = gem_set_hwtst,
2946 };
2947 #endif
2948
2949 static int macb_get_ts_info(struct net_device *netdev,
2950 struct ethtool_ts_info *info)
2951 {
2952 struct macb *bp = netdev_priv(netdev);
2953
2954 if (bp->ptp_info)
2955 return bp->ptp_info->get_ts_info(netdev, info);
2956
2957 return ethtool_op_get_ts_info(netdev, info);
2958 }
2959
2960 static void gem_enable_flow_filters(struct macb *bp, bool enable)
2961 {
2962 struct net_device *netdev = bp->dev;
2963 struct ethtool_rx_fs_item *item;
2964 u32 t2_scr;
2965 int num_t2_scr;
2966
2967 if (!(netdev->features & NETIF_F_NTUPLE))
2968 return;
2969
2970 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8));
2971
2972 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
2973 struct ethtool_rx_flow_spec *fs = &item->fs;
2974 struct ethtool_tcpip4_spec *tp4sp_m;
2975
2976 if (fs->location >= num_t2_scr)
2977 continue;
2978
2979 t2_scr = gem_readl_n(bp, SCRT2, fs->location);
2980
2981 /* enable/disable screener regs for the flow entry */
2982 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr);
2983
2984 /* only enable fields with no masking */
2985 tp4sp_m = &(fs->m_u.tcp_ip4_spec);
2986
2987 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))
2988 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr);
2989 else
2990 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr);
2991
2992 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))
2993 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr);
2994 else
2995 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr);
2996
2997 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))
2998 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr);
2999 else
3000 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr);
3001
3002 gem_writel_n(bp, SCRT2, fs->location, t2_scr);
3003 }
3004 }
3005
3006 static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs)
3007 {
3008 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m;
3009 uint16_t index = fs->location;
3010 u32 w0, w1, t2_scr;
3011 bool cmp_a = false;
3012 bool cmp_b = false;
3013 bool cmp_c = false;
3014
3015 tp4sp_v = &(fs->h_u.tcp_ip4_spec);
3016 tp4sp_m = &(fs->m_u.tcp_ip4_spec);
3017
3018 /* ignore field if any masking set */
3019 if (tp4sp_m->ip4src == 0xFFFFFFFF) {
3020 /* 1st compare reg - IP source address */
3021 w0 = 0;
3022 w1 = 0;
3023 w0 = tp4sp_v->ip4src;
3024 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3025 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3026 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1);
3027 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0);
3028 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1);
3029 cmp_a = true;
3030 }
3031
3032 /* ignore field if any masking set */
3033 if (tp4sp_m->ip4dst == 0xFFFFFFFF) {
3034 /* 2nd compare reg - IP destination address */
3035 w0 = 0;
3036 w1 = 0;
3037 w0 = tp4sp_v->ip4dst;
3038 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3039 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);
3040 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1);
3041 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0);
3042 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1);
3043 cmp_b = true;
3044 }
3045
3046 /* ignore both port fields if masking set in both */
3047 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {
3048 /* 3rd compare reg - source port, destination port */
3049 w0 = 0;
3050 w1 = 0;
3051 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1);
3052 if (tp4sp_m->psrc == tp4sp_m->pdst) {
3053 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);
3054 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3055 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */
3056 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3057 } else {
3058 /* only one port definition */
3059 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */
3060 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0);
3061 if (tp4sp_m->psrc == 0xFFFF) { /* src port */
3062 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);
3063 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);
3064 } else { /* dst port */
3065 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);
3066 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1);
3067 }
3068 }
3069 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0);
3070 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1);
3071 cmp_c = true;
3072 }
3073
3074 t2_scr = 0;
3075 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);
3076 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr);
3077 if (cmp_a)
3078 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr);
3079 if (cmp_b)
3080 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr);
3081 if (cmp_c)
3082 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr);
3083 gem_writel_n(bp, SCRT2, index, t2_scr);
3084 }
3085
3086 static int gem_add_flow_filter(struct net_device *netdev,
3087 struct ethtool_rxnfc *cmd)
3088 {
3089 struct macb *bp = netdev_priv(netdev);
3090 struct ethtool_rx_flow_spec *fs = &cmd->fs;
3091 struct ethtool_rx_fs_item *item, *newfs;
3092 unsigned long flags;
3093 int ret = -EINVAL;
3094 bool added = false;
3095
3096 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL);
3097 if (newfs == NULL)
3098 return -ENOMEM;
3099 memcpy(&newfs->fs, fs, sizeof(newfs->fs));
3100
3101 netdev_dbg(netdev,
3102 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3103 fs->flow_type, (int)fs->ring_cookie, fs->location,
3104 htonl(fs->h_u.tcp_ip4_spec.ip4src),
3105 htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3106 htons(fs->h_u.tcp_ip4_spec.psrc), htons(fs->h_u.tcp_ip4_spec.pdst));
3107
3108 spin_lock_irqsave(&bp->rx_fs_lock, flags);
3109
3110 /* find correct place to add in list */
3111 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3112 if (item->fs.location > newfs->fs.location) {
3113 list_add_tail(&newfs->list, &item->list);
3114 added = true;
3115 break;
3116 } else if (item->fs.location == fs->location) {
3117 netdev_err(netdev, "Rule not added: location %d not free!\n",
3118 fs->location);
3119 ret = -EBUSY;
3120 goto err;
3121 }
3122 }
3123 if (!added)
3124 list_add_tail(&newfs->list, &bp->rx_fs_list.list);
3125
3126 gem_prog_cmp_regs(bp, fs);
3127 bp->rx_fs_list.count++;
3128 /* enable filtering if NTUPLE on */
3129 gem_enable_flow_filters(bp, 1);
3130
3131 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3132 return 0;
3133
3134 err:
3135 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3136 kfree(newfs);
3137 return ret;
3138 }
3139
3140 static int gem_del_flow_filter(struct net_device *netdev,
3141 struct ethtool_rxnfc *cmd)
3142 {
3143 struct macb *bp = netdev_priv(netdev);
3144 struct ethtool_rx_fs_item *item;
3145 struct ethtool_rx_flow_spec *fs;
3146 unsigned long flags;
3147
3148 spin_lock_irqsave(&bp->rx_fs_lock, flags);
3149
3150 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3151 if (item->fs.location == cmd->fs.location) {
3152 /* disable screener regs for the flow entry */
3153 fs = &(item->fs);
3154 netdev_dbg(netdev,
3155 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",
3156 fs->flow_type, (int)fs->ring_cookie, fs->location,
3157 htonl(fs->h_u.tcp_ip4_spec.ip4src),
3158 htonl(fs->h_u.tcp_ip4_spec.ip4dst),
3159 htons(fs->h_u.tcp_ip4_spec.psrc),
3160 htons(fs->h_u.tcp_ip4_spec.pdst));
3161
3162 gem_writel_n(bp, SCRT2, fs->location, 0);
3163
3164 list_del(&item->list);
3165 bp->rx_fs_list.count--;
3166 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3167 kfree(item);
3168 return 0;
3169 }
3170 }
3171
3172 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);
3173 return -EINVAL;
3174 }
3175
3176 static int gem_get_flow_entry(struct net_device *netdev,
3177 struct ethtool_rxnfc *cmd)
3178 {
3179 struct macb *bp = netdev_priv(netdev);
3180 struct ethtool_rx_fs_item *item;
3181
3182 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3183 if (item->fs.location == cmd->fs.location) {
3184 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));
3185 return 0;
3186 }
3187 }
3188 return -EINVAL;
3189 }
3190
3191 static int gem_get_all_flow_entries(struct net_device *netdev,
3192 struct ethtool_rxnfc *cmd, u32 *rule_locs)
3193 {
3194 struct macb *bp = netdev_priv(netdev);
3195 struct ethtool_rx_fs_item *item;
3196 uint32_t cnt = 0;
3197
3198 list_for_each_entry(item, &bp->rx_fs_list.list, list) {
3199 if (cnt == cmd->rule_cnt)
3200 return -EMSGSIZE;
3201 rule_locs[cnt] = item->fs.location;
3202 cnt++;
3203 }
3204 cmd->data = bp->max_tuples;
3205 cmd->rule_cnt = cnt;
3206
3207 return 0;
3208 }
3209
3210 static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,
3211 u32 *rule_locs)
3212 {
3213 struct macb *bp = netdev_priv(netdev);
3214 int ret = 0;
3215
3216 switch (cmd->cmd) {
3217 case ETHTOOL_GRXRINGS:
3218 cmd->data = bp->num_queues;
3219 break;
3220 case ETHTOOL_GRXCLSRLCNT:
3221 cmd->rule_cnt = bp->rx_fs_list.count;
3222 break;
3223 case ETHTOOL_GRXCLSRULE:
3224 ret = gem_get_flow_entry(netdev, cmd);
3225 break;
3226 case ETHTOOL_GRXCLSRLALL:
3227 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs);
3228 break;
3229 default:
3230 netdev_err(netdev,
3231 "Command parameter %d is not supported\n", cmd->cmd);
3232 ret = -EOPNOTSUPP;
3233 }
3234
3235 return ret;
3236 }
3237
3238 static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
3239 {
3240 struct macb *bp = netdev_priv(netdev);
3241 int ret;
3242
3243 switch (cmd->cmd) {
3244 case ETHTOOL_SRXCLSRLINS:
3245 if ((cmd->fs.location >= bp->max_tuples)
3246 || (cmd->fs.ring_cookie >= bp->num_queues)) {
3247 ret = -EINVAL;
3248 break;
3249 }
3250 ret = gem_add_flow_filter(netdev, cmd);
3251 break;
3252 case ETHTOOL_SRXCLSRLDEL:
3253 ret = gem_del_flow_filter(netdev, cmd);
3254 break;
3255 default:
3256 netdev_err(netdev,
3257 "Command parameter %d is not supported\n", cmd->cmd);
3258 ret = -EOPNOTSUPP;
3259 }
3260
3261 return ret;
3262 }
3263
3264 static const struct ethtool_ops macb_ethtool_ops = {
3265 .get_regs_len = macb_get_regs_len,
3266 .get_regs = macb_get_regs,
3267 .get_link = ethtool_op_get_link,
3268 .get_ts_info = ethtool_op_get_ts_info,
3269 .get_wol = macb_get_wol,
3270 .set_wol = macb_set_wol,
3271 .get_link_ksettings = macb_get_link_ksettings,
3272 .set_link_ksettings = macb_set_link_ksettings,
3273 .get_ringparam = macb_get_ringparam,
3274 .set_ringparam = macb_set_ringparam,
3275 };
3276
3277 static const struct ethtool_ops gem_ethtool_ops = {
3278 .get_regs_len = macb_get_regs_len,
3279 .get_regs = macb_get_regs,
3280 .get_link = ethtool_op_get_link,
3281 .get_ts_info = macb_get_ts_info,
3282 .get_ethtool_stats = gem_get_ethtool_stats,
3283 .get_strings = gem_get_ethtool_strings,
3284 .get_sset_count = gem_get_sset_count,
3285 .get_link_ksettings = macb_get_link_ksettings,
3286 .set_link_ksettings = macb_set_link_ksettings,
3287 .get_ringparam = macb_get_ringparam,
3288 .set_ringparam = macb_set_ringparam,
3289 .get_rxnfc = gem_get_rxnfc,
3290 .set_rxnfc = gem_set_rxnfc,
3291 };
3292
3293 static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3294 {
3295 struct macb *bp = netdev_priv(dev);
3296
3297 if (!netif_running(dev))
3298 return -EINVAL;
3299
3300 if (bp->ptp_info) {
3301 switch (cmd) {
3302 case SIOCSHWTSTAMP:
3303 return bp->ptp_info->set_hwtst(dev, rq, cmd);
3304 case SIOCGHWTSTAMP:
3305 return bp->ptp_info->get_hwtst(dev, rq);
3306 }
3307 }
3308
3309 return phylink_mii_ioctl(bp->phylink, rq, cmd);
3310 }
3311
3312 static inline void macb_set_txcsum_feature(struct macb *bp,
3313 netdev_features_t features)
3314 {
3315 u32 val;
3316
3317 if (!macb_is_gem(bp))
3318 return;
3319
3320 val = gem_readl(bp, DMACFG);
3321 if (features & NETIF_F_HW_CSUM)
3322 val |= GEM_BIT(TXCOEN);
3323 else
3324 val &= ~GEM_BIT(TXCOEN);
3325
3326 gem_writel(bp, DMACFG, val);
3327 }
3328
3329 static inline void macb_set_rxcsum_feature(struct macb *bp,
3330 netdev_features_t features)
3331 {
3332 struct net_device *netdev = bp->dev;
3333 u32 val;
3334
3335 if (!macb_is_gem(bp))
3336 return;
3337
3338 val = gem_readl(bp, NCFGR);
3339 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))
3340 val |= GEM_BIT(RXCOEN);
3341 else
3342 val &= ~GEM_BIT(RXCOEN);
3343
3344 gem_writel(bp, NCFGR, val);
3345 }
3346
3347 static inline void macb_set_rxflow_feature(struct macb *bp,
3348 netdev_features_t features)
3349 {
3350 if (!macb_is_gem(bp))
3351 return;
3352
3353 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE));
3354 }
3355
3356 static int macb_set_features(struct net_device *netdev,
3357 netdev_features_t features)
3358 {
3359 struct macb *bp = netdev_priv(netdev);
3360 netdev_features_t changed = features ^ netdev->features;
3361
3362 /* TX checksum offload */
3363 if (changed & NETIF_F_HW_CSUM)
3364 macb_set_txcsum_feature(bp, features);
3365
3366 /* RX checksum offload */
3367 if (changed & NETIF_F_RXCSUM)
3368 macb_set_rxcsum_feature(bp, features);
3369
3370 /* RX Flow Filters */
3371 if (changed & NETIF_F_NTUPLE)
3372 macb_set_rxflow_feature(bp, features);
3373
3374 return 0;
3375 }
3376
3377 static void macb_restore_features(struct macb *bp)
3378 {
3379 struct net_device *netdev = bp->dev;
3380 netdev_features_t features = netdev->features;
3381
3382 /* TX checksum offload */
3383 macb_set_txcsum_feature(bp, features);
3384
3385 /* RX checksum offload */
3386 macb_set_rxcsum_feature(bp, features);
3387
3388 /* RX Flow Filters */
3389 macb_set_rxflow_feature(bp, features);
3390 }
3391
3392 static const struct net_device_ops macb_netdev_ops = {
3393 .ndo_open = macb_open,
3394 .ndo_stop = macb_close,
3395 .ndo_start_xmit = macb_start_xmit,
3396 .ndo_set_rx_mode = macb_set_rx_mode,
3397 .ndo_get_stats = macb_get_stats,
3398 .ndo_do_ioctl = macb_ioctl,
3399 .ndo_validate_addr = eth_validate_addr,
3400 .ndo_change_mtu = macb_change_mtu,
3401 .ndo_set_mac_address = eth_mac_addr,
3402 #ifdef CONFIG_NET_POLL_CONTROLLER
3403 .ndo_poll_controller = macb_poll_controller,
3404 #endif
3405 .ndo_set_features = macb_set_features,
3406 .ndo_features_check = macb_features_check,
3407 };
3408
3409 /* Configure peripheral capabilities according to device tree
3410 * and integration options used
3411 */
3412 static void macb_configure_caps(struct macb *bp,
3413 const struct macb_config *dt_conf)
3414 {
3415 u32 dcfg;
3416
3417 if (dt_conf)
3418 bp->caps = dt_conf->caps;
3419
3420 if (hw_is_gem(bp->regs, bp->native_io)) {
3421 bp->caps |= MACB_CAPS_MACB_IS_GEM;
3422
3423 dcfg = gem_readl(bp, DCFG1);
3424 if (GEM_BFEXT(IRQCOR, dcfg) == 0)
3425 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;
3426 dcfg = gem_readl(bp, DCFG2);
3427 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0)
3428 bp->caps |= MACB_CAPS_FIFO_MODE;
3429 #ifdef CONFIG_MACB_USE_HWSTAMP
3430 if (gem_has_ptp(bp)) {
3431 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5)))
3432 dev_err(&bp->pdev->dev,
3433 "GEM doesn't support hardware ptp.\n");
3434 else {
3435 bp->hw_dma_cap |= HW_DMA_CAP_PTP;
3436 bp->ptp_info = &gem_ptp_info;
3437 }
3438 }
3439 #endif
3440 }
3441
3442 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);
3443 }
3444
3445 static void macb_probe_queues(void __iomem *mem,
3446 bool native_io,
3447 unsigned int *queue_mask,
3448 unsigned int *num_queues)
3449 {
3450 unsigned int hw_q;
3451
3452 *queue_mask = 0x1;
3453 *num_queues = 1;
3454
3455 /* is it macb or gem ?
3456 *
3457 * We need to read directly from the hardware here because
3458 * we are early in the probe process and don't have the
3459 * MACB_CAPS_MACB_IS_GEM flag positioned
3460 */
3461 if (!hw_is_gem(mem, native_io))
3462 return;
3463
3464 /* bit 0 is never set but queue 0 always exists */
3465 *queue_mask = readl_relaxed(mem + GEM_DCFG6) & 0xff;
3466
3467 *queue_mask |= 0x1;
3468
3469 for (hw_q = 1; hw_q < MACB_MAX_QUEUES; ++hw_q)
3470 if (*queue_mask & (1 << hw_q))
3471 (*num_queues)++;
3472 }
3473
3474 static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,
3475 struct clk **hclk, struct clk **tx_clk,
3476 struct clk **rx_clk, struct clk **tsu_clk)
3477 {
3478 struct macb_platform_data *pdata;
3479 int err;
3480
3481 pdata = dev_get_platdata(&pdev->dev);
3482 if (pdata) {
3483 *pclk = pdata->pclk;
3484 *hclk = pdata->hclk;
3485 } else {
3486 *pclk = devm_clk_get(&pdev->dev, "pclk");
3487 *hclk = devm_clk_get(&pdev->dev, "hclk");
3488 }
3489
3490 if (IS_ERR_OR_NULL(*pclk)) {
3491 err = PTR_ERR(*pclk);
3492 if (!err)
3493 err = -ENODEV;
3494
3495 dev_err(&pdev->dev, "failed to get macb_clk (%d)\n", err);
3496 return err;
3497 }
3498
3499 if (IS_ERR_OR_NULL(*hclk)) {
3500 err = PTR_ERR(*hclk);
3501 if (!err)
3502 err = -ENODEV;
3503
3504 dev_err(&pdev->dev, "failed to get hclk (%d)\n", err);
3505 return err;
3506 }
3507
3508 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");
3509 if (IS_ERR(*tx_clk))
3510 return PTR_ERR(*tx_clk);
3511
3512 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");
3513 if (IS_ERR(*rx_clk))
3514 return PTR_ERR(*rx_clk);
3515
3516 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");
3517 if (IS_ERR(*tsu_clk))
3518 return PTR_ERR(*tsu_clk);
3519
3520 err = clk_prepare_enable(*pclk);
3521 if (err) {
3522 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
3523 return err;
3524 }
3525
3526 err = clk_prepare_enable(*hclk);
3527 if (err) {
3528 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);
3529 goto err_disable_pclk;
3530 }
3531
3532 err = clk_prepare_enable(*tx_clk);
3533 if (err) {
3534 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
3535 goto err_disable_hclk;
3536 }
3537
3538 err = clk_prepare_enable(*rx_clk);
3539 if (err) {
3540 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
3541 goto err_disable_txclk;
3542 }
3543
3544 err = clk_prepare_enable(*tsu_clk);
3545 if (err) {
3546 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);
3547 goto err_disable_rxclk;
3548 }
3549
3550 return 0;
3551
3552 err_disable_rxclk:
3553 clk_disable_unprepare(*rx_clk);
3554
3555 err_disable_txclk:
3556 clk_disable_unprepare(*tx_clk);
3557
3558 err_disable_hclk:
3559 clk_disable_unprepare(*hclk);
3560
3561 err_disable_pclk:
3562 clk_disable_unprepare(*pclk);
3563
3564 return err;
3565 }
3566
3567 static int macb_init(struct platform_device *pdev)
3568 {
3569 struct net_device *dev = platform_get_drvdata(pdev);
3570 unsigned int hw_q, q;
3571 struct macb *bp = netdev_priv(dev);
3572 struct macb_queue *queue;
3573 int err;
3574 u32 val, reg;
3575
3576 bp->tx_ring_size = DEFAULT_TX_RING_SIZE;
3577 bp->rx_ring_size = DEFAULT_RX_RING_SIZE;
3578
3579 /* set the queue register mapping once for all: queue0 has a special
3580 * register mapping but we don't want to test the queue index then
3581 * compute the corresponding register offset at run time.
3582 */
3583 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) {
3584 if (!(bp->queue_mask & (1 << hw_q)))
3585 continue;
3586
3587 queue = &bp->queues[q];
3588 queue->bp = bp;
3589 netif_napi_add(dev, &queue->napi, macb_poll, NAPI_POLL_WEIGHT);
3590 if (hw_q) {
3591 queue->ISR = GEM_ISR(hw_q - 1);
3592 queue->IER = GEM_IER(hw_q - 1);
3593 queue->IDR = GEM_IDR(hw_q - 1);
3594 queue->IMR = GEM_IMR(hw_q - 1);
3595 queue->TBQP = GEM_TBQP(hw_q - 1);
3596 queue->RBQP = GEM_RBQP(hw_q - 1);
3597 queue->RBQS = GEM_RBQS(hw_q - 1);
3598 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
3599 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
3600 queue->TBQPH = GEM_TBQPH(hw_q - 1);
3601 queue->RBQPH = GEM_RBQPH(hw_q - 1);
3602 }
3603 #endif
3604 } else {
3605 /* queue0 uses legacy registers */
3606 queue->ISR = MACB_ISR;
3607 queue->IER = MACB_IER;
3608 queue->IDR = MACB_IDR;
3609 queue->IMR = MACB_IMR;
3610 queue->TBQP = MACB_TBQP;
3611 queue->RBQP = MACB_RBQP;
3612 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
3613 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {
3614 queue->TBQPH = MACB_TBQPH;
3615 queue->RBQPH = MACB_RBQPH;
3616 }
3617 #endif
3618 }
3619
3620 /* get irq: here we use the linux queue index, not the hardware
3621 * queue index. the queue irq definitions in the device tree
3622 * must remove the optional gaps that could exist in the
3623 * hardware queue mask.
3624 */
3625 queue->irq = platform_get_irq(pdev, q);
3626 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,
3627 IRQF_SHARED, dev->name, queue);
3628 if (err) {
3629 dev_err(&pdev->dev,
3630 "Unable to request IRQ %d (error %d)\n",
3631 queue->irq, err);
3632 return err;
3633 }
3634
3635 INIT_WORK(&queue->tx_error_task, macb_tx_error_task);
3636 q++;
3637 }
3638
3639 dev->netdev_ops = &macb_netdev_ops;
3640
3641 /* setup appropriated routines according to adapter type */
3642 if (macb_is_gem(bp)) {
3643 bp->max_tx_length = GEM_MAX_TX_LEN;
3644 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;
3645 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;
3646 bp->macbgem_ops.mog_init_rings = gem_init_rings;
3647 bp->macbgem_ops.mog_rx = gem_rx;
3648 dev->ethtool_ops = &gem_ethtool_ops;
3649 } else {
3650 bp->max_tx_length = MACB_MAX_TX_LEN;
3651 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;
3652 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;
3653 bp->macbgem_ops.mog_init_rings = macb_init_rings;
3654 bp->macbgem_ops.mog_rx = macb_rx;
3655 dev->ethtool_ops = &macb_ethtool_ops;
3656 }
3657
3658 /* Set features */
3659 dev->hw_features = NETIF_F_SG;
3660
3661 /* Check LSO capability */
3662 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6)))
3663 dev->hw_features |= MACB_NETIF_LSO;
3664
3665 /* Checksum offload is only available on gem with packet buffer */
3666 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))
3667 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
3668 if (bp->caps & MACB_CAPS_SG_DISABLED)
3669 dev->hw_features &= ~NETIF_F_SG;
3670 dev->features = dev->hw_features;
3671
3672 /* Check RX Flow Filters support.
3673 * Max Rx flows set by availability of screeners & compare regs:
3674 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs
3675 */
3676 reg = gem_readl(bp, DCFG8);
3677 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),
3678 GEM_BFEXT(T2SCR, reg));
3679 if (bp->max_tuples > 0) {
3680 /* also needs one ethtype match to check IPv4 */
3681 if (GEM_BFEXT(SCR2ETH, reg) > 0) {
3682 /* program this reg now */
3683 reg = 0;
3684 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg);
3685 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg);
3686 /* Filtering is supported in hw but don't enable it in kernel now */
3687 dev->hw_features |= NETIF_F_NTUPLE;
3688 /* init Rx flow definitions */
3689 INIT_LIST_HEAD(&bp->rx_fs_list.list);
3690 bp->rx_fs_list.count = 0;
3691 spin_lock_init(&bp->rx_fs_lock);
3692 } else
3693 bp->max_tuples = 0;
3694 }
3695
3696 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {
3697 val = 0;
3698 if (bp->phy_interface == PHY_INTERFACE_MODE_RGMII)
3699 val = GEM_BIT(RGMII);
3700 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&
3701 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
3702 val = MACB_BIT(RMII);
3703 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))
3704 val = MACB_BIT(MII);
3705
3706 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)
3707 val |= MACB_BIT(CLKEN);
3708
3709 macb_or_gem_writel(bp, USRIO, val);
3710 }
3711
3712 /* Set MII management clock divider */
3713 val = macb_mdc_clk_div(bp);
3714 val |= macb_dbw(bp);
3715 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)
3716 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);
3717 macb_writel(bp, NCFGR, val);
3718
3719 return 0;
3720 }
3721
3722 #if defined(CONFIG_OF)
3723 /* 1518 rounded up */
3724 #define AT91ETHER_MAX_RBUFF_SZ 0x600
3725 /* max number of receive buffers */
3726 #define AT91ETHER_MAX_RX_DESCR 9
3727
3728 static struct sifive_fu540_macb_mgmt *mgmt;
3729
3730 /* Initialize and start the Receiver and Transmit subsystems */
3731 static int at91ether_start(struct net_device *dev)
3732 {
3733 struct macb *lp = netdev_priv(dev);
3734 struct macb_queue *q = &lp->queues[0];
3735 struct macb_dma_desc *desc;
3736 dma_addr_t addr;
3737 u32 ctl;
3738 int i;
3739
3740 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,
3741 (AT91ETHER_MAX_RX_DESCR *
3742 macb_dma_desc_get_size(lp)),
3743 &q->rx_ring_dma, GFP_KERNEL);
3744 if (!q->rx_ring)
3745 return -ENOMEM;
3746
3747 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,
3748 AT91ETHER_MAX_RX_DESCR *
3749 AT91ETHER_MAX_RBUFF_SZ,
3750 &q->rx_buffers_dma, GFP_KERNEL);
3751 if (!q->rx_buffers) {
3752 dma_free_coherent(&lp->pdev->dev,
3753 AT91ETHER_MAX_RX_DESCR *
3754 macb_dma_desc_get_size(lp),
3755 q->rx_ring, q->rx_ring_dma);
3756 q->rx_ring = NULL;
3757 return -ENOMEM;
3758 }
3759
3760 addr = q->rx_buffers_dma;
3761 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) {
3762 desc = macb_rx_desc(q, i);
3763 macb_set_addr(lp, desc, addr);
3764 desc->ctrl = 0;
3765 addr += AT91ETHER_MAX_RBUFF_SZ;
3766 }
3767
3768 /* Set the Wrap bit on the last descriptor */
3769 desc->addr |= MACB_BIT(RX_WRAP);
3770
3771 /* Reset buffer index */
3772 q->rx_tail = 0;
3773
3774 /* Program address of descriptor list in Rx Buffer Queue register */
3775 macb_writel(lp, RBQP, q->rx_ring_dma);
3776
3777 /* Enable Receive and Transmit */
3778 ctl = macb_readl(lp, NCR);
3779 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE));
3780
3781 return 0;
3782 }
3783
3784 /* Open the ethernet interface */
3785 static int at91ether_open(struct net_device *dev)
3786 {
3787 struct macb *lp = netdev_priv(dev);
3788 u32 ctl;
3789 int ret;
3790
3791 /* Clear internal statistics */
3792 ctl = macb_readl(lp, NCR);
3793 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT));
3794
3795 macb_set_hwaddr(lp);
3796
3797 ret = at91ether_start(dev);
3798 if (ret)
3799 return ret;
3800
3801 /* Enable MAC interrupts */
3802 macb_writel(lp, IER, MACB_BIT(RCOMP) |
3803 MACB_BIT(RXUBR) |
3804 MACB_BIT(ISR_TUND) |
3805 MACB_BIT(ISR_RLE) |
3806 MACB_BIT(TCOMP) |
3807 MACB_BIT(ISR_ROVR) |
3808 MACB_BIT(HRESP));
3809
3810 ret = macb_phylink_connect(lp);
3811 if (ret)
3812 return ret;
3813
3814 netif_start_queue(dev);
3815
3816 return 0;
3817 }
3818
3819 /* Close the interface */
3820 static int at91ether_close(struct net_device *dev)
3821 {
3822 struct macb *lp = netdev_priv(dev);
3823 struct macb_queue *q = &lp->queues[0];
3824 u32 ctl;
3825
3826 /* Disable Receiver and Transmitter */
3827 ctl = macb_readl(lp, NCR);
3828 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE)));
3829
3830 /* Disable MAC interrupts */
3831 macb_writel(lp, IDR, MACB_BIT(RCOMP) |
3832 MACB_BIT(RXUBR) |
3833 MACB_BIT(ISR_TUND) |
3834 MACB_BIT(ISR_RLE) |
3835 MACB_BIT(TCOMP) |
3836 MACB_BIT(ISR_ROVR) |
3837 MACB_BIT(HRESP));
3838
3839 netif_stop_queue(dev);
3840
3841 phylink_stop(lp->phylink);
3842 phylink_disconnect_phy(lp->phylink);
3843
3844 dma_free_coherent(&lp->pdev->dev,
3845 AT91ETHER_MAX_RX_DESCR *
3846 macb_dma_desc_get_size(lp),
3847 q->rx_ring, q->rx_ring_dma);
3848 q->rx_ring = NULL;
3849
3850 dma_free_coherent(&lp->pdev->dev,
3851 AT91ETHER_MAX_RX_DESCR * AT91ETHER_MAX_RBUFF_SZ,
3852 q->rx_buffers, q->rx_buffers_dma);
3853 q->rx_buffers = NULL;
3854
3855 return 0;
3856 }
3857
3858 /* Transmit packet */
3859 static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb,
3860 struct net_device *dev)
3861 {
3862 struct macb *lp = netdev_priv(dev);
3863
3864 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) {
3865 netif_stop_queue(dev);
3866
3867 /* Store packet information (to free when Tx completed) */
3868 lp->skb = skb;
3869 lp->skb_length = skb->len;
3870 lp->skb_physaddr = dma_map_single(&lp->pdev->dev, skb->data,
3871 skb->len, DMA_TO_DEVICE);
3872 if (dma_mapping_error(&lp->pdev->dev, lp->skb_physaddr)) {
3873 dev_kfree_skb_any(skb);
3874 dev->stats.tx_dropped++;
3875 netdev_err(dev, "%s: DMA mapping error\n", __func__);
3876 return NETDEV_TX_OK;
3877 }
3878
3879 /* Set address of the data in the Transmit Address register */
3880 macb_writel(lp, TAR, lp->skb_physaddr);
3881 /* Set length of the packet in the Transmit Control register */
3882 macb_writel(lp, TCR, skb->len);
3883
3884 } else {
3885 netdev_err(dev, "%s called, but device is busy!\n", __func__);
3886 return NETDEV_TX_BUSY;
3887 }
3888
3889 return NETDEV_TX_OK;
3890 }
3891
3892 /* Extract received frame from buffer descriptors and sent to upper layers.
3893 * (Called from interrupt context)
3894 */
3895 static void at91ether_rx(struct net_device *dev)
3896 {
3897 struct macb *lp = netdev_priv(dev);
3898 struct macb_queue *q = &lp->queues[0];
3899 struct macb_dma_desc *desc;
3900 unsigned char *p_recv;
3901 struct sk_buff *skb;
3902 unsigned int pktlen;
3903
3904 desc = macb_rx_desc(q, q->rx_tail);
3905 while (desc->addr & MACB_BIT(RX_USED)) {
3906 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;
3907 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);
3908 skb = netdev_alloc_skb(dev, pktlen + 2);
3909 if (skb) {
3910 skb_reserve(skb, 2);
3911 skb_put_data(skb, p_recv, pktlen);
3912
3913 skb->protocol = eth_type_trans(skb, dev);
3914 dev->stats.rx_packets++;
3915 dev->stats.rx_bytes += pktlen;
3916 netif_rx(skb);
3917 } else {
3918 dev->stats.rx_dropped++;
3919 }
3920
3921 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))
3922 dev->stats.multicast++;
3923
3924 /* reset ownership bit */
3925 desc->addr &= ~MACB_BIT(RX_USED);
3926
3927 /* wrap after last buffer */
3928 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)
3929 q->rx_tail = 0;
3930 else
3931 q->rx_tail++;
3932
3933 desc = macb_rx_desc(q, q->rx_tail);
3934 }
3935 }
3936
3937 /* MAC interrupt handler */
3938 static irqreturn_t at91ether_interrupt(int irq, void *dev_id)
3939 {
3940 struct net_device *dev = dev_id;
3941 struct macb *lp = netdev_priv(dev);
3942 u32 intstatus, ctl;
3943
3944 /* MAC Interrupt Status register indicates what interrupts are pending.
3945 * It is automatically cleared once read.
3946 */
3947 intstatus = macb_readl(lp, ISR);
3948
3949 /* Receive complete */
3950 if (intstatus & MACB_BIT(RCOMP))
3951 at91ether_rx(dev);
3952
3953 /* Transmit complete */
3954 if (intstatus & MACB_BIT(TCOMP)) {
3955 /* The TCOM bit is set even if the transmission failed */
3956 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE)))
3957 dev->stats.tx_errors++;
3958
3959 if (lp->skb) {
3960 dev_consume_skb_irq(lp->skb);
3961 lp->skb = NULL;
3962 dma_unmap_single(&lp->pdev->dev, lp->skb_physaddr,
3963 lp->skb_length, DMA_TO_DEVICE);
3964 dev->stats.tx_packets++;
3965 dev->stats.tx_bytes += lp->skb_length;
3966 }
3967 netif_wake_queue(dev);
3968 }
3969
3970 /* Work-around for EMAC Errata section 41.3.1 */
3971 if (intstatus & MACB_BIT(RXUBR)) {
3972 ctl = macb_readl(lp, NCR);
3973 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE));
3974 wmb();
3975 macb_writel(lp, NCR, ctl | MACB_BIT(RE));
3976 }
3977
3978 if (intstatus & MACB_BIT(ISR_ROVR))
3979 netdev_err(dev, "ROVR error\n");
3980
3981 return IRQ_HANDLED;
3982 }
3983
3984 #ifdef CONFIG_NET_POLL_CONTROLLER
3985 static void at91ether_poll_controller(struct net_device *dev)
3986 {
3987 unsigned long flags;
3988
3989 local_irq_save(flags);
3990 at91ether_interrupt(dev->irq, dev);
3991 local_irq_restore(flags);
3992 }
3993 #endif
3994
3995 static const struct net_device_ops at91ether_netdev_ops = {
3996 .ndo_open = at91ether_open,
3997 .ndo_stop = at91ether_close,
3998 .ndo_start_xmit = at91ether_start_xmit,
3999 .ndo_get_stats = macb_get_stats,
4000 .ndo_set_rx_mode = macb_set_rx_mode,
4001 .ndo_set_mac_address = eth_mac_addr,
4002 .ndo_do_ioctl = macb_ioctl,
4003 .ndo_validate_addr = eth_validate_addr,
4004 #ifdef CONFIG_NET_POLL_CONTROLLER
4005 .ndo_poll_controller = at91ether_poll_controller,
4006 #endif
4007 };
4008
4009 static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,
4010 struct clk **hclk, struct clk **tx_clk,
4011 struct clk **rx_clk, struct clk **tsu_clk)
4012 {
4013 int err;
4014
4015 *hclk = NULL;
4016 *tx_clk = NULL;
4017 *rx_clk = NULL;
4018 *tsu_clk = NULL;
4019
4020 *pclk = devm_clk_get(&pdev->dev, "ether_clk");
4021 if (IS_ERR(*pclk))
4022 return PTR_ERR(*pclk);
4023
4024 err = clk_prepare_enable(*pclk);
4025 if (err) {
4026 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);
4027 return err;
4028 }
4029
4030 return 0;
4031 }
4032
4033 static int at91ether_init(struct platform_device *pdev)
4034 {
4035 struct net_device *dev = platform_get_drvdata(pdev);
4036 struct macb *bp = netdev_priv(dev);
4037 int err;
4038 u32 reg;
4039
4040 bp->queues[0].bp = bp;
4041
4042 dev->netdev_ops = &at91ether_netdev_ops;
4043 dev->ethtool_ops = &macb_ethtool_ops;
4044
4045 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,
4046 0, dev->name, dev);
4047 if (err)
4048 return err;
4049
4050 macb_writel(bp, NCR, 0);
4051
4052 reg = MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG);
4053 if (bp->phy_interface == PHY_INTERFACE_MODE_RMII)
4054 reg |= MACB_BIT(RM9200_RMII);
4055
4056 macb_writel(bp, NCFGR, reg);
4057
4058 return 0;
4059 }
4060
4061 static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw,
4062 unsigned long parent_rate)
4063 {
4064 return mgmt->rate;
4065 }
4066
4067 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate,
4068 unsigned long *parent_rate)
4069 {
4070 if (WARN_ON(rate < 2500000))
4071 return 2500000;
4072 else if (rate == 2500000)
4073 return 2500000;
4074 else if (WARN_ON(rate < 13750000))
4075 return 2500000;
4076 else if (WARN_ON(rate < 25000000))
4077 return 25000000;
4078 else if (rate == 25000000)
4079 return 25000000;
4080 else if (WARN_ON(rate < 75000000))
4081 return 25000000;
4082 else if (WARN_ON(rate < 125000000))
4083 return 125000000;
4084 else if (rate == 125000000)
4085 return 125000000;
4086
4087 WARN_ON(rate > 125000000);
4088
4089 return 125000000;
4090 }
4091
4092 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate,
4093 unsigned long parent_rate)
4094 {
4095 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate);
4096 if (rate != 125000000)
4097 iowrite32(1, mgmt->reg);
4098 else
4099 iowrite32(0, mgmt->reg);
4100 mgmt->rate = rate;
4101
4102 return 0;
4103 }
4104
4105 static const struct clk_ops fu540_c000_ops = {
4106 .recalc_rate = fu540_macb_tx_recalc_rate,
4107 .round_rate = fu540_macb_tx_round_rate,
4108 .set_rate = fu540_macb_tx_set_rate,
4109 };
4110
4111 static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,
4112 struct clk **hclk, struct clk **tx_clk,
4113 struct clk **rx_clk, struct clk **tsu_clk)
4114 {
4115 struct clk_init_data init;
4116 int err = 0;
4117
4118 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);
4119 if (err)
4120 return err;
4121
4122 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);
4123 if (!mgmt)
4124 return -ENOMEM;
4125
4126 init.name = "sifive-gemgxl-mgmt";
4127 init.ops = &fu540_c000_ops;
4128 init.flags = 0;
4129 init.num_parents = 0;
4130
4131 mgmt->rate = 0;
4132 mgmt->hw.init = &init;
4133
4134 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);
4135 if (IS_ERR(*tx_clk))
4136 return PTR_ERR(*tx_clk);
4137
4138 err = clk_prepare_enable(*tx_clk);
4139 if (err)
4140 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);
4141 else
4142 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);
4143
4144 return 0;
4145 }
4146
4147 static int fu540_c000_init(struct platform_device *pdev)
4148 {
4149 struct resource *res;
4150
4151 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
4152 if (!res)
4153 return -ENODEV;
4154
4155 mgmt->reg = ioremap(res->start, resource_size(res));
4156 if (!mgmt->reg)
4157 return -ENOMEM;
4158
4159 return macb_init(pdev);
4160 }
4161
4162 static const struct macb_config fu540_c000_config = {
4163 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |
4164 MACB_CAPS_GEM_HAS_PTP,
4165 .dma_burst_length = 16,
4166 .clk_init = fu540_c000_clk_init,
4167 .init = fu540_c000_init,
4168 .jumbo_max_len = 10240,
4169 };
4170
4171 static const struct macb_config at91sam9260_config = {
4172 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4173 .clk_init = macb_clk_init,
4174 .init = macb_init,
4175 };
4176
4177 static const struct macb_config sama5d3macb_config = {
4178 .caps = MACB_CAPS_SG_DISABLED
4179 | MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4180 .clk_init = macb_clk_init,
4181 .init = macb_init,
4182 };
4183
4184 static const struct macb_config pc302gem_config = {
4185 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE,
4186 .dma_burst_length = 16,
4187 .clk_init = macb_clk_init,
4188 .init = macb_init,
4189 };
4190
4191 static const struct macb_config sama5d2_config = {
4192 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4193 .dma_burst_length = 16,
4194 .clk_init = macb_clk_init,
4195 .init = macb_init,
4196 };
4197
4198 static const struct macb_config sama5d3_config = {
4199 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE
4200 | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,
4201 .dma_burst_length = 16,
4202 .clk_init = macb_clk_init,
4203 .init = macb_init,
4204 .jumbo_max_len = 10240,
4205 };
4206
4207 static const struct macb_config sama5d4_config = {
4208 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,
4209 .dma_burst_length = 4,
4210 .clk_init = macb_clk_init,
4211 .init = macb_init,
4212 };
4213
4214 static const struct macb_config emac_config = {
4215 .caps = MACB_CAPS_NEEDS_RSTONUBR,
4216 .clk_init = at91ether_clk_init,
4217 .init = at91ether_init,
4218 };
4219
4220 static const struct macb_config np4_config = {
4221 .caps = MACB_CAPS_USRIO_DISABLED,
4222 .clk_init = macb_clk_init,
4223 .init = macb_init,
4224 };
4225
4226 static const struct macb_config zynqmp_config = {
4227 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4228 MACB_CAPS_JUMBO |
4229 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH,
4230 .dma_burst_length = 16,
4231 .clk_init = macb_clk_init,
4232 .init = macb_init,
4233 .jumbo_max_len = 10240,
4234 };
4235
4236 static const struct macb_config zynq_config = {
4237 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF |
4238 MACB_CAPS_NEEDS_RSTONUBR,
4239 .dma_burst_length = 16,
4240 .clk_init = macb_clk_init,
4241 .init = macb_init,
4242 };
4243
4244 static const struct of_device_id macb_dt_ids[] = {
4245 { .compatible = "cdns,at32ap7000-macb" },
4246 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4247 { .compatible = "cdns,macb" },
4248 { .compatible = "cdns,np4-macb", .data = &np4_config },
4249 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4250 { .compatible = "cdns,gem", .data = &pc302gem_config },
4251 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4252 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4253 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4254 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4255 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4256 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4257 { .compatible = "cdns,emac", .data = &emac_config },
4258 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config},
4259 { .compatible = "cdns,zynq-gem", .data = &zynq_config },
4260 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4261 { /* sentinel */ }
4262 };
4263 MODULE_DEVICE_TABLE(of, macb_dt_ids);
4264 #endif /* CONFIG_OF */
4265
4266 static const struct macb_config default_gem_config = {
4267 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |
4268 MACB_CAPS_JUMBO |
4269 MACB_CAPS_GEM_HAS_PTP,
4270 .dma_burst_length = 16,
4271 .clk_init = macb_clk_init,
4272 .init = macb_init,
4273 .jumbo_max_len = 10240,
4274 };
4275
4276 static int macb_probe(struct platform_device *pdev)
4277 {
4278 const struct macb_config *macb_config = &default_gem_config;
4279 int (*clk_init)(struct platform_device *, struct clk **,
4280 struct clk **, struct clk **, struct clk **,
4281 struct clk **) = macb_config->clk_init;
4282 int (*init)(struct platform_device *) = macb_config->init;
4283 struct device_node *np = pdev->dev.of_node;
4284 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;
4285 struct clk *tsu_clk = NULL;
4286 unsigned int queue_mask, num_queues;
4287 bool native_io;
4288 phy_interface_t interface;
4289 struct net_device *dev;
4290 struct resource *regs;
4291 void __iomem *mem;
4292 const char *mac;
4293 struct macb *bp;
4294 int err, val;
4295
4296 regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4297 mem = devm_ioremap_resource(&pdev->dev, regs);
4298 if (IS_ERR(mem))
4299 return PTR_ERR(mem);
4300
4301 if (np) {
4302 const struct of_device_id *match;
4303
4304 match = of_match_node(macb_dt_ids, np);
4305 if (match && match->data) {
4306 macb_config = match->data;
4307 clk_init = macb_config->clk_init;
4308 init = macb_config->init;
4309 }
4310 }
4311
4312 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk);
4313 if (err)
4314 return err;
4315
4316 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);
4317 pm_runtime_use_autosuspend(&pdev->dev);
4318 pm_runtime_get_noresume(&pdev->dev);
4319 pm_runtime_set_active(&pdev->dev);
4320 pm_runtime_enable(&pdev->dev);
4321 native_io = hw_is_native_io(mem);
4322
4323 macb_probe_queues(mem, native_io, &queue_mask, &num_queues);
4324 dev = alloc_etherdev_mq(sizeof(*bp), num_queues);
4325 if (!dev) {
4326 err = -ENOMEM;
4327 goto err_disable_clocks;
4328 }
4329
4330 dev->base_addr = regs->start;
4331
4332 SET_NETDEV_DEV(dev, &pdev->dev);
4333
4334 bp = netdev_priv(dev);
4335 bp->pdev = pdev;
4336 bp->dev = dev;
4337 bp->regs = mem;
4338 bp->native_io = native_io;
4339 if (native_io) {
4340 bp->macb_reg_readl = hw_readl_native;
4341 bp->macb_reg_writel = hw_writel_native;
4342 } else {
4343 bp->macb_reg_readl = hw_readl;
4344 bp->macb_reg_writel = hw_writel;
4345 }
4346 bp->num_queues = num_queues;
4347 bp->queue_mask = queue_mask;
4348 if (macb_config)
4349 bp->dma_burst_length = macb_config->dma_burst_length;
4350 bp->pclk = pclk;
4351 bp->hclk = hclk;
4352 bp->tx_clk = tx_clk;
4353 bp->rx_clk = rx_clk;
4354 bp->tsu_clk = tsu_clk;
4355 if (macb_config)
4356 bp->jumbo_max_len = macb_config->jumbo_max_len;
4357
4358 bp->wol = 0;
4359 if (of_get_property(np, "magic-packet", NULL))
4360 bp->wol |= MACB_WOL_HAS_MAGIC_PACKET;
4361 device_init_wakeup(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET);
4362
4363 spin_lock_init(&bp->lock);
4364
4365 /* setup capabilities */
4366 macb_configure_caps(bp, macb_config);
4367
4368 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
4369 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) {
4370 dma_set_mask(&pdev->dev, DMA_BIT_MASK(44));
4371 bp->hw_dma_cap |= HW_DMA_CAP_64B;
4372 }
4373 #endif
4374 platform_set_drvdata(pdev, dev);
4375
4376 dev->irq = platform_get_irq(pdev, 0);
4377 if (dev->irq < 0) {
4378 err = dev->irq;
4379 goto err_out_free_netdev;
4380 }
4381
4382 /* MTU range: 68 - 1500 or 10240 */
4383 dev->min_mtu = GEM_MTU_MIN_SIZE;
4384 if (bp->caps & MACB_CAPS_JUMBO)
4385 dev->max_mtu = gem_readl(bp, JML) - ETH_HLEN - ETH_FCS_LEN;
4386 else
4387 dev->max_mtu = ETH_DATA_LEN;
4388
4389 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {
4390 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10));
4391 if (val)
4392 bp->rx_bd_rd_prefetch = (2 << (val - 1)) *
4393 macb_dma_desc_get_size(bp);
4394
4395 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10));
4396 if (val)
4397 bp->tx_bd_rd_prefetch = (2 << (val - 1)) *
4398 macb_dma_desc_get_size(bp);
4399 }
4400
4401 bp->rx_intr_mask = MACB_RX_INT_FLAGS;
4402 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)
4403 bp->rx_intr_mask |= MACB_BIT(RXUBR);
4404
4405 mac = of_get_mac_address(np);
4406 if (PTR_ERR(mac) == -EPROBE_DEFER) {
4407 err = -EPROBE_DEFER;
4408 goto err_out_free_netdev;
4409 } else if (!IS_ERR_OR_NULL(mac)) {
4410 ether_addr_copy(bp->dev->dev_addr, mac);
4411 } else {
4412 macb_get_hwaddr(bp);
4413 }
4414
4415 err = of_get_phy_mode(np, &interface);
4416 if (err)
4417 /* not found in DT, MII by default */
4418 bp->phy_interface = PHY_INTERFACE_MODE_MII;
4419 else
4420 bp->phy_interface = interface;
4421
4422 bp->speed = SPEED_UNKNOWN;
4423
4424 /* IP specific init */
4425 err = init(pdev);
4426 if (err)
4427 goto err_out_free_netdev;
4428
4429 err = macb_mii_init(bp);
4430 if (err)
4431 goto err_out_free_netdev;
4432
4433 netif_carrier_off(dev);
4434
4435 err = register_netdev(dev);
4436 if (err) {
4437 dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
4438 goto err_out_unregister_mdio;
4439 }
4440
4441 tasklet_init(&bp->hresp_err_tasklet, macb_hresp_error_task,
4442 (unsigned long)bp);
4443
4444 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",
4445 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID),
4446 dev->base_addr, dev->irq, dev->dev_addr);
4447
4448 pm_runtime_mark_last_busy(&bp->pdev->dev);
4449 pm_runtime_put_autosuspend(&bp->pdev->dev);
4450
4451 return 0;
4452
4453 err_out_unregister_mdio:
4454 mdiobus_unregister(bp->mii_bus);
4455 mdiobus_free(bp->mii_bus);
4456
4457 err_out_free_netdev:
4458 free_netdev(dev);
4459
4460 err_disable_clocks:
4461 clk_disable_unprepare(tx_clk);
4462 clk_disable_unprepare(hclk);
4463 clk_disable_unprepare(pclk);
4464 clk_disable_unprepare(rx_clk);
4465 clk_disable_unprepare(tsu_clk);
4466 pm_runtime_disable(&pdev->dev);
4467 pm_runtime_set_suspended(&pdev->dev);
4468 pm_runtime_dont_use_autosuspend(&pdev->dev);
4469
4470 return err;
4471 }
4472
4473 static int macb_remove(struct platform_device *pdev)
4474 {
4475 struct net_device *dev;
4476 struct macb *bp;
4477
4478 dev = platform_get_drvdata(pdev);
4479
4480 if (dev) {
4481 bp = netdev_priv(dev);
4482 mdiobus_unregister(bp->mii_bus);
4483 mdiobus_free(bp->mii_bus);
4484
4485 unregister_netdev(dev);
4486 tasklet_kill(&bp->hresp_err_tasklet);
4487 pm_runtime_disable(&pdev->dev);
4488 pm_runtime_dont_use_autosuspend(&pdev->dev);
4489 if (!pm_runtime_suspended(&pdev->dev)) {
4490 clk_disable_unprepare(bp->tx_clk);
4491 clk_disable_unprepare(bp->hclk);
4492 clk_disable_unprepare(bp->pclk);
4493 clk_disable_unprepare(bp->rx_clk);
4494 clk_disable_unprepare(bp->tsu_clk);
4495 pm_runtime_set_suspended(&pdev->dev);
4496 }
4497 phylink_destroy(bp->phylink);
4498 free_netdev(dev);
4499 }
4500
4501 return 0;
4502 }
4503
4504 static int __maybe_unused macb_suspend(struct device *dev)
4505 {
4506 struct net_device *netdev = dev_get_drvdata(dev);
4507 struct macb *bp = netdev_priv(netdev);
4508 struct macb_queue *queue = bp->queues;
4509 unsigned long flags;
4510 unsigned int q;
4511
4512 if (!netif_running(netdev))
4513 return 0;
4514
4515 if (bp->wol & MACB_WOL_ENABLED) {
4516 macb_writel(bp, IER, MACB_BIT(WOL));
4517 macb_writel(bp, WOL, MACB_BIT(MAG));
4518 enable_irq_wake(bp->queues[0].irq);
4519 netif_device_detach(netdev);
4520 } else {
4521 netif_device_detach(netdev);
4522 for (q = 0, queue = bp->queues; q < bp->num_queues;
4523 ++q, ++queue)
4524 napi_disable(&queue->napi);
4525 rtnl_lock();
4526 phylink_stop(bp->phylink);
4527 rtnl_unlock();
4528 spin_lock_irqsave(&bp->lock, flags);
4529 macb_reset_hw(bp);
4530 spin_unlock_irqrestore(&bp->lock, flags);
4531
4532 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
4533 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);
4534
4535 if (netdev->hw_features & NETIF_F_NTUPLE)
4536 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);
4537 }
4538
4539 netif_carrier_off(netdev);
4540 if (bp->ptp_info)
4541 bp->ptp_info->ptp_remove(netdev);
4542 pm_runtime_force_suspend(dev);
4543
4544 return 0;
4545 }
4546
4547 static int __maybe_unused macb_resume(struct device *dev)
4548 {
4549 struct net_device *netdev = dev_get_drvdata(dev);
4550 struct macb *bp = netdev_priv(netdev);
4551 struct macb_queue *queue = bp->queues;
4552 unsigned int q;
4553
4554 if (!netif_running(netdev))
4555 return 0;
4556
4557 pm_runtime_force_resume(dev);
4558
4559 if (bp->wol & MACB_WOL_ENABLED) {
4560 macb_writel(bp, IDR, MACB_BIT(WOL));
4561 macb_writel(bp, WOL, 0);
4562 disable_irq_wake(bp->queues[0].irq);
4563 } else {
4564 macb_writel(bp, NCR, MACB_BIT(MPE));
4565
4566 if (netdev->hw_features & NETIF_F_NTUPLE)
4567 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);
4568
4569 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))
4570 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);
4571
4572 for (q = 0, queue = bp->queues; q < bp->num_queues;
4573 ++q, ++queue)
4574 napi_enable(&queue->napi);
4575 rtnl_lock();
4576 phylink_start(bp->phylink);
4577 rtnl_unlock();
4578 }
4579
4580 macb_init_hw(bp);
4581 macb_set_rx_mode(netdev);
4582 macb_restore_features(bp);
4583 netif_device_attach(netdev);
4584 if (bp->ptp_info)
4585 bp->ptp_info->ptp_init(netdev);
4586
4587 return 0;
4588 }
4589
4590 static int __maybe_unused macb_runtime_suspend(struct device *dev)
4591 {
4592 struct net_device *netdev = dev_get_drvdata(dev);
4593 struct macb *bp = netdev_priv(netdev);
4594
4595 if (!(device_may_wakeup(&bp->dev->dev))) {
4596 clk_disable_unprepare(bp->tx_clk);
4597 clk_disable_unprepare(bp->hclk);
4598 clk_disable_unprepare(bp->pclk);
4599 clk_disable_unprepare(bp->rx_clk);
4600 }
4601 clk_disable_unprepare(bp->tsu_clk);
4602
4603 return 0;
4604 }
4605
4606 static int __maybe_unused macb_runtime_resume(struct device *dev)
4607 {
4608 struct net_device *netdev = dev_get_drvdata(dev);
4609 struct macb *bp = netdev_priv(netdev);
4610
4611 if (!(device_may_wakeup(&bp->dev->dev))) {
4612 clk_prepare_enable(bp->pclk);
4613 clk_prepare_enable(bp->hclk);
4614 clk_prepare_enable(bp->tx_clk);
4615 clk_prepare_enable(bp->rx_clk);
4616 }
4617 clk_prepare_enable(bp->tsu_clk);
4618
4619 return 0;
4620 }
4621
4622 static const struct dev_pm_ops macb_pm_ops = {
4623 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume)
4624 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL)
4625 };
4626
4627 static struct platform_driver macb_driver = {
4628 .probe = macb_probe,
4629 .remove = macb_remove,
4630 .driver = {
4631 .name = "macb",
4632 .of_match_table = of_match_ptr(macb_dt_ids),
4633 .pm = &macb_pm_ops,
4634 },
4635 };
4636
4637 module_platform_driver(macb_driver);
4638
4639 MODULE_LICENSE("GPL");
4640 MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver");
4641 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
4642 MODULE_ALIAS("platform:macb");
Linux with added FPC-III support