?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / drivers / net / ethernet / ti / cpsw_priv.c
blob97a058ca60ac9b177f7945e3b9be9ccfe9eb070e
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Texas Instruments Ethernet Switch Driver
4 *
5 * Copyright (C) 2019 Texas Instruments
6 */
7
8 #include <linux/bpf.h>
9 #include <linux/bpf_trace.h>
10 #include <linux/if_ether.h>
11 #include <linux/if_vlan.h>
12 #include <linux/kmemleak.h>
13 #include <linux/module.h>
14 #include <linux/netdevice.h>
15 #include <linux/net_tstamp.h>
16 #include <linux/of.h>
17 #include <linux/phy.h>
18 #include <linux/platform_device.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/skbuff.h>
21 #include <net/page_pool.h>
22 #include <net/pkt_cls.h>
23
24 #include "cpsw.h"
25 #include "cpts.h"
26 #include "cpsw_ale.h"
27 #include "cpsw_priv.h"
28 #include "cpsw_sl.h"
29 #include "davinci_cpdma.h"
30
31 int (*cpsw_slave_index)(struct cpsw_common *cpsw, struct cpsw_priv *priv);
32
33 void cpsw_intr_enable(struct cpsw_common *cpsw)
34 {
35 writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);
36 writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);
37
38 cpdma_ctlr_int_ctrl(cpsw->dma, true);
39 }
40
41 void cpsw_intr_disable(struct cpsw_common *cpsw)
42 {
43 writel_relaxed(0, &cpsw->wr_regs->tx_en);
44 writel_relaxed(0, &cpsw->wr_regs->rx_en);
45
46 cpdma_ctlr_int_ctrl(cpsw->dma, false);
47 }
48
49 void cpsw_tx_handler(void *token, int len, int status)
50 {
51 struct cpsw_meta_xdp *xmeta;
52 struct xdp_frame *xdpf;
53 struct net_device *ndev;
54 struct netdev_queue *txq;
55 struct sk_buff *skb;
56 int ch;
57
58 if (cpsw_is_xdpf_handle(token)) {
59 xdpf = cpsw_handle_to_xdpf(token);
60 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET;
61 ndev = xmeta->ndev;
62 ch = xmeta->ch;
63 xdp_return_frame(xdpf);
64 } else {
65 skb = token;
66 ndev = skb->dev;
67 ch = skb_get_queue_mapping(skb);
68 cpts_tx_timestamp(ndev_to_cpsw(ndev)->cpts, skb);
69 dev_kfree_skb_any(skb);
70 }
71
72 /* Check whether the queue is stopped due to stalled tx dma, if the
73 * queue is stopped then start the queue as we have free desc for tx
74 */
75 txq = netdev_get_tx_queue(ndev, ch);
76 if (unlikely(netif_tx_queue_stopped(txq)))
77 netif_tx_wake_queue(txq);
78
79 ndev->stats.tx_packets++;
80 ndev->stats.tx_bytes += len;
81 }
82
83 irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
84 {
85 struct cpsw_common *cpsw = dev_id;
86
87 writel(0, &cpsw->wr_regs->tx_en);
88 cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
89
90 if (cpsw->quirk_irq) {
91 disable_irq_nosync(cpsw->irqs_table[1]);
92 cpsw->tx_irq_disabled = true;
93 }
94
95 napi_schedule(&cpsw->napi_tx);
96 return IRQ_HANDLED;
97 }
98
99 irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
100 {
101 struct cpsw_common *cpsw = dev_id;
102
103 writel(0, &cpsw->wr_regs->rx_en);
104 cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
105
106 if (cpsw->quirk_irq) {
107 disable_irq_nosync(cpsw->irqs_table[0]);
108 cpsw->rx_irq_disabled = true;
109 }
110
111 napi_schedule(&cpsw->napi_rx);
112 return IRQ_HANDLED;
113 }
114
115 int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
116 {
117 struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
118 int num_tx, cur_budget, ch;
119 u32 ch_map;
120 struct cpsw_vector *txv;
121
122 /* process every unprocessed channel */
123 ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
124 for (ch = 0, num_tx = 0; ch_map & 0xff; ch_map <<= 1, ch++) {
125 if (!(ch_map & 0x80))
126 continue;
127
128 txv = &cpsw->txv[ch];
129 if (unlikely(txv->budget > budget - num_tx))
130 cur_budget = budget - num_tx;
131 else
132 cur_budget = txv->budget;
133
134 num_tx += cpdma_chan_process(txv->ch, cur_budget);
135 if (num_tx >= budget)
136 break;
137 }
138
139 if (num_tx < budget) {
140 napi_complete(napi_tx);
141 writel(0xff, &cpsw->wr_regs->tx_en);
142 }
143
144 return num_tx;
145 }
146
147 int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
148 {
149 struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
150 int num_tx;
151
152 num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);
153 if (num_tx < budget) {
154 napi_complete(napi_tx);
155 writel(0xff, &cpsw->wr_regs->tx_en);
156 if (cpsw->tx_irq_disabled) {
157 cpsw->tx_irq_disabled = false;
158 enable_irq(cpsw->irqs_table[1]);
159 }
160 }
161
162 return num_tx;
163 }
164
165 int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
166 {
167 struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
168 int num_rx, cur_budget, ch;
169 u32 ch_map;
170 struct cpsw_vector *rxv;
171
172 /* process every unprocessed channel */
173 ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
174 for (ch = 0, num_rx = 0; ch_map; ch_map >>= 1, ch++) {
175 if (!(ch_map & 0x01))
176 continue;
177
178 rxv = &cpsw->rxv[ch];
179 if (unlikely(rxv->budget > budget - num_rx))
180 cur_budget = budget - num_rx;
181 else
182 cur_budget = rxv->budget;
183
184 num_rx += cpdma_chan_process(rxv->ch, cur_budget);
185 if (num_rx >= budget)
186 break;
187 }
188
189 if (num_rx < budget) {
190 napi_complete_done(napi_rx, num_rx);
191 writel(0xff, &cpsw->wr_regs->rx_en);
192 }
193
194 return num_rx;
195 }
196
197 int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
198 {
199 struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
200 int num_rx;
201
202 num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);
203 if (num_rx < budget) {
204 napi_complete_done(napi_rx, num_rx);
205 writel(0xff, &cpsw->wr_regs->rx_en);
206 if (cpsw->rx_irq_disabled) {
207 cpsw->rx_irq_disabled = false;
208 enable_irq(cpsw->irqs_table[0]);
209 }
210 }
211
212 return num_rx;
213 }
214
215 void cpsw_rx_vlan_encap(struct sk_buff *skb)
216 {
217 struct cpsw_priv *priv = netdev_priv(skb->dev);
218 u32 rx_vlan_encap_hdr = *((u32 *)skb->data);
219 struct cpsw_common *cpsw = priv->cpsw;
220 u16 vtag, vid, prio, pkt_type;
221
222 /* Remove VLAN header encapsulation word */
223 skb_pull(skb, CPSW_RX_VLAN_ENCAP_HDR_SIZE);
224
225 pkt_type = (rx_vlan_encap_hdr >>
226 CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT) &
227 CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK;
228 /* Ignore unknown & Priority-tagged packets*/
229 if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV ||
230 pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG)
231 return;
232
233 vid = (rx_vlan_encap_hdr >>
234 CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT) &
235 VLAN_VID_MASK;
236 /* Ignore vid 0 and pass packet as is */
237 if (!vid)
238 return;
239
240 /* Untag P0 packets if set for vlan */
241 if (!cpsw_ale_get_vlan_p0_untag(cpsw->ale, vid)) {
242 prio = (rx_vlan_encap_hdr >>
243 CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT) &
244 CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK;
245
246 vtag = (prio << VLAN_PRIO_SHIFT) | vid;
247 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
248 }
249
250 /* strip vlan tag for VLAN-tagged packet */
251 if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG) {
252 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
253 skb_pull(skb, VLAN_HLEN);
254 }
255 }
256
257 void cpsw_set_slave_mac(struct cpsw_slave *slave, struct cpsw_priv *priv)
258 {
259 slave_write(slave, mac_hi(priv->mac_addr), SA_HI);
260 slave_write(slave, mac_lo(priv->mac_addr), SA_LO);
261 }
262
263 void soft_reset(const char *module, void __iomem *reg)
264 {
265 unsigned long timeout = jiffies + HZ;
266
267 writel_relaxed(1, reg);
268 do {
269 cpu_relax();
270 } while ((readl_relaxed(reg) & 1) && time_after(timeout, jiffies));
271
272 WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module);
273 }
274
275 void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue)
276 {
277 struct cpsw_priv *priv = netdev_priv(ndev);
278 struct cpsw_common *cpsw = priv->cpsw;
279 int ch;
280
281 cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
282 ndev->stats.tx_errors++;
283 cpsw_intr_disable(cpsw);
284 for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
285 cpdma_chan_stop(cpsw->txv[ch].ch);
286 cpdma_chan_start(cpsw->txv[ch].ch);
287 }
288
289 cpsw_intr_enable(cpsw);
290 netif_trans_update(ndev);
291 netif_tx_wake_all_queues(ndev);
292 }
293
294 static int cpsw_get_common_speed(struct cpsw_common *cpsw)
295 {
296 int i, speed;
297
298 for (i = 0, speed = 0; i < cpsw->data.slaves; i++)
299 if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)
300 speed += cpsw->slaves[i].phy->speed;
301
302 return speed;
303 }
304
305 int cpsw_need_resplit(struct cpsw_common *cpsw)
306 {
307 int i, rlim_ch_num;
308 int speed, ch_rate;
309
310 /* re-split resources only in case speed was changed */
311 speed = cpsw_get_common_speed(cpsw);
312 if (speed == cpsw->speed || !speed)
313 return 0;
314
315 cpsw->speed = speed;
316
317 for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {
318 ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);
319 if (!ch_rate)
320 break;
321
322 rlim_ch_num++;
323 }
324
325 /* cases not dependent on speed */
326 if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)
327 return 0;
328
329 return 1;
330 }
331
332 void cpsw_split_res(struct cpsw_common *cpsw)
333 {
334 u32 consumed_rate = 0, bigest_rate = 0;
335 struct cpsw_vector *txv = cpsw->txv;
336 int i, ch_weight, rlim_ch_num = 0;
337 int budget, bigest_rate_ch = 0;
338 u32 ch_rate, max_rate;
339 int ch_budget = 0;
340
341 for (i = 0; i < cpsw->tx_ch_num; i++) {
342 ch_rate = cpdma_chan_get_rate(txv[i].ch);
343 if (!ch_rate)
344 continue;
345
346 rlim_ch_num++;
347 consumed_rate += ch_rate;
348 }
349
350 if (cpsw->tx_ch_num == rlim_ch_num) {
351 max_rate = consumed_rate;
352 } else if (!rlim_ch_num) {
353 ch_budget = CPSW_POLL_WEIGHT / cpsw->tx_ch_num;
354 bigest_rate = 0;
355 max_rate = consumed_rate;
356 } else {
357 max_rate = cpsw->speed * 1000;
358
359 /* if max_rate is less then expected due to reduced link speed,
360 * split proportionally according next potential max speed
361 */
362 if (max_rate < consumed_rate)
363 max_rate *= 10;
364
365 if (max_rate < consumed_rate)
366 max_rate *= 10;
367
368 ch_budget = (consumed_rate * CPSW_POLL_WEIGHT) / max_rate;
369 ch_budget = (CPSW_POLL_WEIGHT - ch_budget) /
370 (cpsw->tx_ch_num - rlim_ch_num);
371 bigest_rate = (max_rate - consumed_rate) /
372 (cpsw->tx_ch_num - rlim_ch_num);
373 }
374
375 /* split tx weight/budget */
376 budget = CPSW_POLL_WEIGHT;
377 for (i = 0; i < cpsw->tx_ch_num; i++) {
378 ch_rate = cpdma_chan_get_rate(txv[i].ch);
379 if (ch_rate) {
380 txv[i].budget = (ch_rate * CPSW_POLL_WEIGHT) / max_rate;
381 if (!txv[i].budget)
382 txv[i].budget++;
383 if (ch_rate > bigest_rate) {
384 bigest_rate_ch = i;
385 bigest_rate = ch_rate;
386 }
387
388 ch_weight = (ch_rate * 100) / max_rate;
389 if (!ch_weight)
390 ch_weight++;
391 cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);
392 } else {
393 txv[i].budget = ch_budget;
394 if (!bigest_rate_ch)
395 bigest_rate_ch = i;
396 cpdma_chan_set_weight(cpsw->txv[i].ch, 0);
397 }
398
399 budget -= txv[i].budget;
400 }
401
402 if (budget)
403 txv[bigest_rate_ch].budget += budget;
404
405 /* split rx budget */
406 budget = CPSW_POLL_WEIGHT;
407 ch_budget = budget / cpsw->rx_ch_num;
408 for (i = 0; i < cpsw->rx_ch_num; i++) {
409 cpsw->rxv[i].budget = ch_budget;
410 budget -= ch_budget;
411 }
412
413 if (budget)
414 cpsw->rxv[0].budget += budget;
415 }
416
417 int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs,
418 int ale_ageout, phys_addr_t desc_mem_phys,
419 int descs_pool_size)
420 {
421 u32 slave_offset, sliver_offset, slave_size;
422 struct cpsw_ale_params ale_params;
423 struct cpsw_platform_data *data;
424 struct cpdma_params dma_params;
425 struct device *dev = cpsw->dev;
426 struct device_node *cpts_node;
427 void __iomem *cpts_regs;
428 int ret = 0, i;
429
430 data = &cpsw->data;
431 cpsw->rx_ch_num = 1;
432 cpsw->tx_ch_num = 1;
433
434 cpsw->version = readl(&cpsw->regs->id_ver);
435
436 memset(&dma_params, 0, sizeof(dma_params));
437 memset(&ale_params, 0, sizeof(ale_params));
438
439 switch (cpsw->version) {
440 case CPSW_VERSION_1:
441 cpsw->host_port_regs = ss_regs + CPSW1_HOST_PORT_OFFSET;
442 cpts_regs = ss_regs + CPSW1_CPTS_OFFSET;
443 cpsw->hw_stats = ss_regs + CPSW1_HW_STATS;
444 dma_params.dmaregs = ss_regs + CPSW1_CPDMA_OFFSET;
445 dma_params.txhdp = ss_regs + CPSW1_STATERAM_OFFSET;
446 ale_params.ale_regs = ss_regs + CPSW1_ALE_OFFSET;
447 slave_offset = CPSW1_SLAVE_OFFSET;
448 slave_size = CPSW1_SLAVE_SIZE;
449 sliver_offset = CPSW1_SLIVER_OFFSET;
450 dma_params.desc_mem_phys = 0;
451 break;
452 case CPSW_VERSION_2:
453 case CPSW_VERSION_3:
454 case CPSW_VERSION_4:
455 cpsw->host_port_regs = ss_regs + CPSW2_HOST_PORT_OFFSET;
456 cpts_regs = ss_regs + CPSW2_CPTS_OFFSET;
457 cpsw->hw_stats = ss_regs + CPSW2_HW_STATS;
458 dma_params.dmaregs = ss_regs + CPSW2_CPDMA_OFFSET;
459 dma_params.txhdp = ss_regs + CPSW2_STATERAM_OFFSET;
460 ale_params.ale_regs = ss_regs + CPSW2_ALE_OFFSET;
461 slave_offset = CPSW2_SLAVE_OFFSET;
462 slave_size = CPSW2_SLAVE_SIZE;
463 sliver_offset = CPSW2_SLIVER_OFFSET;
464 dma_params.desc_mem_phys = desc_mem_phys;
465 break;
466 default:
467 dev_err(dev, "unknown version 0x%08x\n", cpsw->version);
468 return -ENODEV;
469 }
470
471 for (i = 0; i < cpsw->data.slaves; i++) {
472 struct cpsw_slave *slave = &cpsw->slaves[i];
473 void __iomem *regs = cpsw->regs;
474
475 slave->slave_num = i;
476 slave->data = &cpsw->data.slave_data[i];
477 slave->regs = regs + slave_offset;
478 slave->port_vlan = slave->data->dual_emac_res_vlan;
479 slave->mac_sl = cpsw_sl_get("cpsw", dev, regs + sliver_offset);
480 if (IS_ERR(slave->mac_sl))
481 return PTR_ERR(slave->mac_sl);
482
483 slave_offset += slave_size;
484 sliver_offset += SLIVER_SIZE;
485 }
486
487 ale_params.dev = dev;
488 ale_params.ale_ageout = ale_ageout;
489 ale_params.ale_entries = data->ale_entries;
490 ale_params.ale_ports = CPSW_ALE_PORTS_NUM;
491
492 cpsw->ale = cpsw_ale_create(&ale_params);
493 if (!cpsw->ale) {
494 dev_err(dev, "error initializing ale engine\n");
495 return -ENODEV;
496 }
497
498 dma_params.dev = dev;
499 dma_params.rxthresh = dma_params.dmaregs + CPDMA_RXTHRESH;
500 dma_params.rxfree = dma_params.dmaregs + CPDMA_RXFREE;
501 dma_params.rxhdp = dma_params.txhdp + CPDMA_RXHDP;
502 dma_params.txcp = dma_params.txhdp + CPDMA_TXCP;
503 dma_params.rxcp = dma_params.txhdp + CPDMA_RXCP;
504
505 dma_params.num_chan = data->channels;
506 dma_params.has_soft_reset = true;
507 dma_params.min_packet_size = CPSW_MIN_PACKET_SIZE;
508 dma_params.desc_mem_size = data->bd_ram_size;
509 dma_params.desc_align = 16;
510 dma_params.has_ext_regs = true;
511 dma_params.desc_hw_addr = dma_params.desc_mem_phys;
512 dma_params.bus_freq_mhz = cpsw->bus_freq_mhz;
513 dma_params.descs_pool_size = descs_pool_size;
514
515 cpsw->dma = cpdma_ctlr_create(&dma_params);
516 if (!cpsw->dma) {
517 dev_err(dev, "error initializing dma\n");
518 return -ENOMEM;
519 }
520
521 cpts_node = of_get_child_by_name(cpsw->dev->of_node, "cpts");
522 if (!cpts_node)
523 cpts_node = cpsw->dev->of_node;
524
525 cpsw->cpts = cpts_create(cpsw->dev, cpts_regs, cpts_node);
526 if (IS_ERR(cpsw->cpts)) {
527 ret = PTR_ERR(cpsw->cpts);
528 cpdma_ctlr_destroy(cpsw->dma);
529 }
530 of_node_put(cpts_node);
531
532 return ret;
533 }
534
535 #if IS_ENABLED(CONFIG_TI_CPTS)
536
537 static void cpsw_hwtstamp_v1(struct cpsw_priv *priv)
538 {
539 struct cpsw_common *cpsw = priv->cpsw;
540 struct cpsw_slave *slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
541 u32 ts_en, seq_id;
542
543 if (!priv->tx_ts_enabled && !priv->rx_ts_enabled) {
544 slave_write(slave, 0, CPSW1_TS_CTL);
545 return;
546 }
547
548 seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
549 ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
550
551 if (priv->tx_ts_enabled)
552 ts_en |= CPSW_V1_TS_TX_EN;
553
554 if (priv->rx_ts_enabled)
555 ts_en |= CPSW_V1_TS_RX_EN;
556
557 slave_write(slave, ts_en, CPSW1_TS_CTL);
558 slave_write(slave, seq_id, CPSW1_TS_SEQ_LTYPE);
559 }
560
561 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
562 {
563 struct cpsw_common *cpsw = priv->cpsw;
564 struct cpsw_slave *slave;
565 u32 ctrl, mtype;
566
567 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
568
569 ctrl = slave_read(slave, CPSW2_CONTROL);
570 switch (cpsw->version) {
571 case CPSW_VERSION_2:
572 ctrl &= ~CTRL_V2_ALL_TS_MASK;
573
574 if (priv->tx_ts_enabled)
575 ctrl |= CTRL_V2_TX_TS_BITS;
576
577 if (priv->rx_ts_enabled)
578 ctrl |= CTRL_V2_RX_TS_BITS;
579 break;
580 case CPSW_VERSION_3:
581 default:
582 ctrl &= ~CTRL_V3_ALL_TS_MASK;
583
584 if (priv->tx_ts_enabled)
585 ctrl |= CTRL_V3_TX_TS_BITS;
586
587 if (priv->rx_ts_enabled)
588 ctrl |= CTRL_V3_RX_TS_BITS;
589 break;
590 }
591
592 mtype = (30 << TS_SEQ_ID_OFFSET_SHIFT) | EVENT_MSG_BITS;
593
594 slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
595 slave_write(slave, ctrl, CPSW2_CONTROL);
596 writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);
597 writel_relaxed(ETH_P_8021Q, &cpsw->regs->vlan_ltype);
598 }
599
600 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
601 {
602 struct cpsw_priv *priv = netdev_priv(dev);
603 struct cpsw_common *cpsw = priv->cpsw;
604 struct hwtstamp_config cfg;
605
606 if (cpsw->version != CPSW_VERSION_1 &&
607 cpsw->version != CPSW_VERSION_2 &&
608 cpsw->version != CPSW_VERSION_3)
609 return -EOPNOTSUPP;
610
611 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
612 return -EFAULT;
613
614 /* reserved for future extensions */
615 if (cfg.flags)
616 return -EINVAL;
617
618 if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
619 return -ERANGE;
620
621 switch (cfg.rx_filter) {
622 case HWTSTAMP_FILTER_NONE:
623 priv->rx_ts_enabled = 0;
624 break;
625 case HWTSTAMP_FILTER_ALL:
626 case HWTSTAMP_FILTER_NTP_ALL:
627 return -ERANGE;
628 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
629 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
630 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
631 priv->rx_ts_enabled = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
632 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
633 break;
634 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
635 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
636 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
637 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
638 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
639 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
640 case HWTSTAMP_FILTER_PTP_V2_EVENT:
641 case HWTSTAMP_FILTER_PTP_V2_SYNC:
642 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
643 priv->rx_ts_enabled = HWTSTAMP_FILTER_PTP_V2_EVENT;
644 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
645 break;
646 default:
647 return -ERANGE;
648 }
649
650 priv->tx_ts_enabled = cfg.tx_type == HWTSTAMP_TX_ON;
651
652 switch (cpsw->version) {
653 case CPSW_VERSION_1:
654 cpsw_hwtstamp_v1(priv);
655 break;
656 case CPSW_VERSION_2:
657 case CPSW_VERSION_3:
658 cpsw_hwtstamp_v2(priv);
659 break;
660 default:
661 WARN_ON(1);
662 }
663
664 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
665 }
666
667 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
668 {
669 struct cpsw_common *cpsw = ndev_to_cpsw(dev);
670 struct cpsw_priv *priv = netdev_priv(dev);
671 struct hwtstamp_config cfg;
672
673 if (cpsw->version != CPSW_VERSION_1 &&
674 cpsw->version != CPSW_VERSION_2 &&
675 cpsw->version != CPSW_VERSION_3)
676 return -EOPNOTSUPP;
677
678 cfg.flags = 0;
679 cfg.tx_type = priv->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
680 cfg.rx_filter = priv->rx_ts_enabled;
681
682 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
683 }
684 #else
685 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
686 {
687 return -EOPNOTSUPP;
688 }
689
690 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
691 {
692 return -EOPNOTSUPP;
693 }
694 #endif /*CONFIG_TI_CPTS*/
695
696 int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
697 {
698 struct cpsw_priv *priv = netdev_priv(dev);
699 struct cpsw_common *cpsw = priv->cpsw;
700 int slave_no = cpsw_slave_index(cpsw, priv);
701
702 if (!netif_running(dev))
703 return -EINVAL;
704
705 switch (cmd) {
706 case SIOCSHWTSTAMP:
707 return cpsw_hwtstamp_set(dev, req);
708 case SIOCGHWTSTAMP:
709 return cpsw_hwtstamp_get(dev, req);
710 }
711
712 if (!cpsw->slaves[slave_no].phy)
713 return -EOPNOTSUPP;
714 return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
715 }
716
717 int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate)
718 {
719 struct cpsw_priv *priv = netdev_priv(ndev);
720 struct cpsw_common *cpsw = priv->cpsw;
721 struct cpsw_slave *slave;
722 u32 min_rate;
723 u32 ch_rate;
724 int i, ret;
725
726 ch_rate = netdev_get_tx_queue(ndev, queue)->tx_maxrate;
727 if (ch_rate == rate)
728 return 0;
729
730 ch_rate = rate * 1000;
731 min_rate = cpdma_chan_get_min_rate(cpsw->dma);
732 if ((ch_rate < min_rate && ch_rate)) {
733 dev_err(priv->dev, "The channel rate cannot be less than %dMbps",
734 min_rate);
735 return -EINVAL;
736 }
737
738 if (rate > cpsw->speed) {
739 dev_err(priv->dev, "The channel rate cannot be more than 2Gbps");
740 return -EINVAL;
741 }
742
743 ret = pm_runtime_get_sync(cpsw->dev);
744 if (ret < 0) {
745 pm_runtime_put_noidle(cpsw->dev);
746 return ret;
747 }
748
749 ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);
750 pm_runtime_put(cpsw->dev);
751
752 if (ret)
753 return ret;
754
755 /* update rates for slaves tx queues */
756 for (i = 0; i < cpsw->data.slaves; i++) {
757 slave = &cpsw->slaves[i];
758 if (!slave->ndev)
759 continue;
760
761 netdev_get_tx_queue(slave->ndev, queue)->tx_maxrate = rate;
762 }
763
764 cpsw_split_res(cpsw);
765 return ret;
766 }
767
768 static int cpsw_tc_to_fifo(int tc, int num_tc)
769 {
770 if (tc == num_tc - 1)
771 return 0;
772
773 return CPSW_FIFO_SHAPERS_NUM - tc;
774 }
775
776 bool cpsw_shp_is_off(struct cpsw_priv *priv)
777 {
778 struct cpsw_common *cpsw = priv->cpsw;
779 struct cpsw_slave *slave;
780 u32 shift, mask, val;
781
782 val = readl_relaxed(&cpsw->regs->ptype);
783
784 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
785 shift = CPSW_FIFO_SHAPE_EN_SHIFT + 3 * slave->slave_num;
786 mask = 7 << shift;
787 val = val & mask;
788
789 return !val;
790 }
791
792 static void cpsw_fifo_shp_on(struct cpsw_priv *priv, int fifo, int on)
793 {
794 struct cpsw_common *cpsw = priv->cpsw;
795 struct cpsw_slave *slave;
796 u32 shift, mask, val;
797
798 val = readl_relaxed(&cpsw->regs->ptype);
799
800 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
801 shift = CPSW_FIFO_SHAPE_EN_SHIFT + 3 * slave->slave_num;
802 mask = (1 << --fifo) << shift;
803 val = on ? val | mask : val & ~mask;
804
805 writel_relaxed(val, &cpsw->regs->ptype);
806 }
807
808 static int cpsw_set_fifo_bw(struct cpsw_priv *priv, int fifo, int bw)
809 {
810 struct cpsw_common *cpsw = priv->cpsw;
811 u32 val = 0, send_pct, shift;
812 struct cpsw_slave *slave;
813 int pct = 0, i;
814
815 if (bw > priv->shp_cfg_speed * 1000)
816 goto err;
817
818 /* shaping has to stay enabled for highest fifos linearly
819 * and fifo bw no more then interface can allow
820 */
821 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
822 send_pct = slave_read(slave, SEND_PERCENT);
823 for (i = CPSW_FIFO_SHAPERS_NUM; i > 0; i--) {
824 if (!bw) {
825 if (i >= fifo || !priv->fifo_bw[i])
826 continue;
827
828 dev_warn(priv->dev, "Prev FIFO%d is shaped", i);
829 continue;
830 }
831
832 if (!priv->fifo_bw[i] && i > fifo) {
833 dev_err(priv->dev, "Upper FIFO%d is not shaped", i);
834 return -EINVAL;
835 }
836
837 shift = (i - 1) * 8;
838 if (i == fifo) {
839 send_pct &= ~(CPSW_PCT_MASK << shift);
840 val = DIV_ROUND_UP(bw, priv->shp_cfg_speed * 10);
841 if (!val)
842 val = 1;
843
844 send_pct |= val << shift;
845 pct += val;
846 continue;
847 }
848
849 if (priv->fifo_bw[i])
850 pct += (send_pct >> shift) & CPSW_PCT_MASK;
851 }
852
853 if (pct >= 100)
854 goto err;
855
856 slave_write(slave, send_pct, SEND_PERCENT);
857 priv->fifo_bw[fifo] = bw;
858
859 dev_warn(priv->dev, "set FIFO%d bw = %d\n", fifo,
860 DIV_ROUND_CLOSEST(val * priv->shp_cfg_speed, 100));
861
862 return 0;
863 err:
864 dev_err(priv->dev, "Bandwidth doesn't fit in tc configuration");
865 return -EINVAL;
866 }
867
868 static int cpsw_set_fifo_rlimit(struct cpsw_priv *priv, int fifo, int bw)
869 {
870 struct cpsw_common *cpsw = priv->cpsw;
871 struct cpsw_slave *slave;
872 u32 tx_in_ctl_rg, val;
873 int ret;
874
875 ret = cpsw_set_fifo_bw(priv, fifo, bw);
876 if (ret)
877 return ret;
878
879 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
880 tx_in_ctl_rg = cpsw->version == CPSW_VERSION_1 ?
881 CPSW1_TX_IN_CTL : CPSW2_TX_IN_CTL;
882
883 if (!bw)
884 cpsw_fifo_shp_on(priv, fifo, bw);
885
886 val = slave_read(slave, tx_in_ctl_rg);
887 if (cpsw_shp_is_off(priv)) {
888 /* disable FIFOs rate limited queues */
889 val &= ~(0xf << CPSW_FIFO_RATE_EN_SHIFT);
890
891 /* set type of FIFO queues to normal priority mode */
892 val &= ~(3 << CPSW_FIFO_QUEUE_TYPE_SHIFT);
893
894 /* set type of FIFO queues to be rate limited */
895 if (bw)
896 val |= 2 << CPSW_FIFO_QUEUE_TYPE_SHIFT;
897 else
898 priv->shp_cfg_speed = 0;
899 }
900
901 /* toggle a FIFO rate limited queue */
902 if (bw)
903 val |= BIT(fifo + CPSW_FIFO_RATE_EN_SHIFT);
904 else
905 val &= ~BIT(fifo + CPSW_FIFO_RATE_EN_SHIFT);
906 slave_write(slave, val, tx_in_ctl_rg);
907
908 /* FIFO transmit shape enable */
909 cpsw_fifo_shp_on(priv, fifo, bw);
910 return 0;
911 }
912
913 /* Defaults:
914 * class A - prio 3
915 * class B - prio 2
916 * shaping for class A should be set first
917 */
918 static int cpsw_set_cbs(struct net_device *ndev,
919 struct tc_cbs_qopt_offload *qopt)
920 {
921 struct cpsw_priv *priv = netdev_priv(ndev);
922 struct cpsw_common *cpsw = priv->cpsw;
923 struct cpsw_slave *slave;
924 int prev_speed = 0;
925 int tc, ret, fifo;
926 u32 bw = 0;
927
928 tc = netdev_txq_to_tc(priv->ndev, qopt->queue);
929
930 /* enable channels in backward order, as highest FIFOs must be rate
931 * limited first and for compliance with CPDMA rate limited channels
932 * that also used in bacward order. FIFO0 cannot be rate limited.
933 */
934 fifo = cpsw_tc_to_fifo(tc, ndev->num_tc);
935 if (!fifo) {
936 dev_err(priv->dev, "Last tc%d can't be rate limited", tc);
937 return -EINVAL;
938 }
939
940 /* do nothing, it's disabled anyway */
941 if (!qopt->enable && !priv->fifo_bw[fifo])
942 return 0;
943
944 /* shapers can be set if link speed is known */
945 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
946 if (slave->phy && slave->phy->link) {
947 if (priv->shp_cfg_speed &&
948 priv->shp_cfg_speed != slave->phy->speed)
949 prev_speed = priv->shp_cfg_speed;
950
951 priv->shp_cfg_speed = slave->phy->speed;
952 }
953
954 if (!priv->shp_cfg_speed) {
955 dev_err(priv->dev, "Link speed is not known");
956 return -1;
957 }
958
959 ret = pm_runtime_get_sync(cpsw->dev);
960 if (ret < 0) {
961 pm_runtime_put_noidle(cpsw->dev);
962 return ret;
963 }
964
965 bw = qopt->enable ? qopt->idleslope : 0;
966 ret = cpsw_set_fifo_rlimit(priv, fifo, bw);
967 if (ret) {
968 priv->shp_cfg_speed = prev_speed;
969 prev_speed = 0;
970 }
971
972 if (bw && prev_speed)
973 dev_warn(priv->dev,
974 "Speed was changed, CBS shaper speeds are changed!");
975
976 pm_runtime_put_sync(cpsw->dev);
977 return ret;
978 }
979
980 static int cpsw_set_mqprio(struct net_device *ndev, void *type_data)
981 {
982 struct tc_mqprio_qopt_offload *mqprio = type_data;
983 struct cpsw_priv *priv = netdev_priv(ndev);
984 struct cpsw_common *cpsw = priv->cpsw;
985 int fifo, num_tc, count, offset;
986 struct cpsw_slave *slave;
987 u32 tx_prio_map = 0;
988 int i, tc, ret;
989
990 num_tc = mqprio->qopt.num_tc;
991 if (num_tc > CPSW_TC_NUM)
992 return -EINVAL;
993
994 if (mqprio->mode != TC_MQPRIO_MODE_DCB)
995 return -EINVAL;
996
997 ret = pm_runtime_get_sync(cpsw->dev);
998 if (ret < 0) {
999 pm_runtime_put_noidle(cpsw->dev);
1000 return ret;
1001 }
1002
1003 if (num_tc) {
1004 for (i = 0; i < 8; i++) {
1005 tc = mqprio->qopt.prio_tc_map[i];
1006 fifo = cpsw_tc_to_fifo(tc, num_tc);
1007 tx_prio_map |= fifo << (4 * i);
1008 }
1009
1010 netdev_set_num_tc(ndev, num_tc);
1011 for (i = 0; i < num_tc; i++) {
1012 count = mqprio->qopt.count[i];
1013 offset = mqprio->qopt.offset[i];
1014 netdev_set_tc_queue(ndev, i, count, offset);
1015 }
1016 }
1017
1018 if (!mqprio->qopt.hw) {
1019 /* restore default configuration */
1020 netdev_reset_tc(ndev);
1021 tx_prio_map = TX_PRIORITY_MAPPING;
1022 }
1023
1024 priv->mqprio_hw = mqprio->qopt.hw;
1025
1026 offset = cpsw->version == CPSW_VERSION_1 ?
1027 CPSW1_TX_PRI_MAP : CPSW2_TX_PRI_MAP;
1028
1029 slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1030 slave_write(slave, tx_prio_map, offset);
1031
1032 pm_runtime_put_sync(cpsw->dev);
1033
1034 return 0;
1035 }
1036
1037 int cpsw_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1038 void *type_data)
1039 {
1040 switch (type) {
1041 case TC_SETUP_QDISC_CBS:
1042 return cpsw_set_cbs(ndev, type_data);
1043
1044 case TC_SETUP_QDISC_MQPRIO:
1045 return cpsw_set_mqprio(ndev, type_data);
1046
1047 default:
1048 return -EOPNOTSUPP;
1049 }
1050 }
1051
1052 void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv)
1053 {
1054 int fifo, bw;
1055
1056 for (fifo = CPSW_FIFO_SHAPERS_NUM; fifo > 0; fifo--) {
1057 bw = priv->fifo_bw[fifo];
1058 if (!bw)
1059 continue;
1060
1061 cpsw_set_fifo_rlimit(priv, fifo, bw);
1062 }
1063 }
1064
1065 void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv)
1066 {
1067 struct cpsw_common *cpsw = priv->cpsw;
1068 u32 tx_prio_map = 0;
1069 int i, tc, fifo;
1070 u32 tx_prio_rg;
1071
1072 if (!priv->mqprio_hw)
1073 return;
1074
1075 for (i = 0; i < 8; i++) {
1076 tc = netdev_get_prio_tc_map(priv->ndev, i);
1077 fifo = CPSW_FIFO_SHAPERS_NUM - tc;
1078 tx_prio_map |= fifo << (4 * i);
1079 }
1080
1081 tx_prio_rg = cpsw->version == CPSW_VERSION_1 ?
1082 CPSW1_TX_PRI_MAP : CPSW2_TX_PRI_MAP;
1083
1084 slave_write(slave, tx_prio_map, tx_prio_rg);
1085 }
1086
1087 int cpsw_fill_rx_channels(struct cpsw_priv *priv)
1088 {
1089 struct cpsw_common *cpsw = priv->cpsw;
1090 struct cpsw_meta_xdp *xmeta;
1091 struct page_pool *pool;
1092 struct page *page;
1093 int ch_buf_num;
1094 int ch, i, ret;
1095 dma_addr_t dma;
1096
1097 for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1098 pool = cpsw->page_pool[ch];
1099 ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
1100 for (i = 0; i < ch_buf_num; i++) {
1101 page = page_pool_dev_alloc_pages(pool);
1102 if (!page) {
1103 cpsw_err(priv, ifup, "allocate rx page err\n");
1104 return -ENOMEM;
1105 }
1106
1107 xmeta = page_address(page) + CPSW_XMETA_OFFSET;
1108 xmeta->ndev = priv->ndev;
1109 xmeta->ch = ch;
1110
1111 dma = page_pool_get_dma_addr(page) + CPSW_HEADROOM;
1112 ret = cpdma_chan_idle_submit_mapped(cpsw->rxv[ch].ch,
1113 page, dma,
1114 cpsw->rx_packet_max,
1115 0);
1116 if (ret < 0) {
1117 cpsw_err(priv, ifup,
1118 "cannot submit page to channel %d rx, error %d\n",
1119 ch, ret);
1120 page_pool_recycle_direct(pool, page);
1121 return ret;
1122 }
1123 }
1124
1125 cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
1126 ch, ch_buf_num);
1127 }
1128
1129 return 0;
1130 }
1131
1132 static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw,
1133 int size)
1134 {
1135 struct page_pool_params pp_params;
1136 struct page_pool *pool;
1137
1138 pp_params.order = 0;
1139 pp_params.flags = PP_FLAG_DMA_MAP;
1140 pp_params.pool_size = size;
1141 pp_params.nid = NUMA_NO_NODE;
1142 pp_params.dma_dir = DMA_BIDIRECTIONAL;
1143 pp_params.dev = cpsw->dev;
1144
1145 pool = page_pool_create(&pp_params);
1146 if (IS_ERR(pool))
1147 dev_err(cpsw->dev, "cannot create rx page pool\n");
1148
1149 return pool;
1150 }
1151
1152 static int cpsw_create_rx_pool(struct cpsw_common *cpsw, int ch)
1153 {
1154 struct page_pool *pool;
1155 int ret = 0, pool_size;
1156
1157 pool_size = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
1158 pool = cpsw_create_page_pool(cpsw, pool_size);
1159 if (IS_ERR(pool))
1160 ret = PTR_ERR(pool);
1161 else
1162 cpsw->page_pool[ch] = pool;
1163
1164 return ret;
1165 }
1166
1167 static int cpsw_ndev_create_xdp_rxq(struct cpsw_priv *priv, int ch)
1168 {
1169 struct cpsw_common *cpsw = priv->cpsw;
1170 struct xdp_rxq_info *rxq;
1171 struct page_pool *pool;
1172 int ret;
1173
1174 pool = cpsw->page_pool[ch];
1175 rxq = &priv->xdp_rxq[ch];
1176
1177 ret = xdp_rxq_info_reg(rxq, priv->ndev, ch);
1178 if (ret)
1179 return ret;
1180
1181 ret = xdp_rxq_info_reg_mem_model(rxq, MEM_TYPE_PAGE_POOL, pool);
1182 if (ret)
1183 xdp_rxq_info_unreg(rxq);
1184
1185 return ret;
1186 }
1187
1188 static void cpsw_ndev_destroy_xdp_rxq(struct cpsw_priv *priv, int ch)
1189 {
1190 struct xdp_rxq_info *rxq = &priv->xdp_rxq[ch];
1191
1192 if (!xdp_rxq_info_is_reg(rxq))
1193 return;
1194
1195 xdp_rxq_info_unreg(rxq);
1196 }
1197
1198 void cpsw_destroy_xdp_rxqs(struct cpsw_common *cpsw)
1199 {
1200 struct net_device *ndev;
1201 int i, ch;
1202
1203 for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1204 for (i = 0; i < cpsw->data.slaves; i++) {
1205 ndev = cpsw->slaves[i].ndev;
1206 if (!ndev)
1207 continue;
1208
1209 cpsw_ndev_destroy_xdp_rxq(netdev_priv(ndev), ch);
1210 }
1211
1212 page_pool_destroy(cpsw->page_pool[ch]);
1213 cpsw->page_pool[ch] = NULL;
1214 }
1215 }
1216
1217 int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw)
1218 {
1219 struct net_device *ndev;
1220 int i, ch, ret;
1221
1222 for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1223 ret = cpsw_create_rx_pool(cpsw, ch);
1224 if (ret)
1225 goto err_cleanup;
1226
1227 /* using same page pool is allowed as no running rx handlers
1228 * simultaneously for both ndevs
1229 */
1230 for (i = 0; i < cpsw->data.slaves; i++) {
1231 ndev = cpsw->slaves[i].ndev;
1232 if (!ndev)
1233 continue;
1234
1235 ret = cpsw_ndev_create_xdp_rxq(netdev_priv(ndev), ch);
1236 if (ret)
1237 goto err_cleanup;
1238 }
1239 }
1240
1241 return 0;
1242
1243 err_cleanup:
1244 cpsw_destroy_xdp_rxqs(cpsw);
1245
1246 return ret;
1247 }
1248
1249 static int cpsw_xdp_prog_setup(struct cpsw_priv *priv, struct netdev_bpf *bpf)
1250 {
1251 struct bpf_prog *prog = bpf->prog;
1252
1253 if (!priv->xdpi.prog && !prog)
1254 return 0;
1255
1256 if (!xdp_attachment_flags_ok(&priv->xdpi, bpf))
1257 return -EBUSY;
1258
1259 WRITE_ONCE(priv->xdp_prog, prog);
1260
1261 xdp_attachment_setup(&priv->xdpi, bpf);
1262
1263 return 0;
1264 }
1265
1266 int cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
1267 {
1268 struct cpsw_priv *priv = netdev_priv(ndev);
1269
1270 switch (bpf->command) {
1271 case XDP_SETUP_PROG:
1272 return cpsw_xdp_prog_setup(priv, bpf);
1273
1274 case XDP_QUERY_PROG:
1275 return xdp_attachment_query(&priv->xdpi, bpf);
1276
1277 default:
1278 return -EINVAL;
1279 }
1280 }
1281
1282 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf,
1283 struct page *page, int port)
1284 {
1285 struct cpsw_common *cpsw = priv->cpsw;
1286 struct cpsw_meta_xdp *xmeta;
1287 struct cpdma_chan *txch;
1288 dma_addr_t dma;
1289 int ret;
1290
1291 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET;
1292 xmeta->ndev = priv->ndev;
1293 xmeta->ch = 0;
1294 txch = cpsw->txv[0].ch;
1295
1296 if (page) {
1297 dma = page_pool_get_dma_addr(page);
1298 dma += xdpf->headroom + sizeof(struct xdp_frame);
1299 ret = cpdma_chan_submit_mapped(txch, cpsw_xdpf_to_handle(xdpf),
1300 dma, xdpf->len, port);
1301 } else {
1302 if (sizeof(*xmeta) > xdpf->headroom) {
1303 xdp_return_frame_rx_napi(xdpf);
1304 return -EINVAL;
1305 }
1306
1307 ret = cpdma_chan_submit(txch, cpsw_xdpf_to_handle(xdpf),
1308 xdpf->data, xdpf->len, port);
1309 }
1310
1311 if (ret) {
1312 priv->ndev->stats.tx_dropped++;
1313 xdp_return_frame_rx_napi(xdpf);
1314 }
1315
1316 return ret;
1317 }
1318
1319 int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp,
1320 struct page *page, int port)
1321 {
1322 struct cpsw_common *cpsw = priv->cpsw;
1323 struct net_device *ndev = priv->ndev;
1324 int ret = CPSW_XDP_CONSUMED;
1325 struct xdp_frame *xdpf;
1326 struct bpf_prog *prog;
1327 u32 act;
1328
1329 rcu_read_lock();
1330
1331 prog = READ_ONCE(priv->xdp_prog);
1332 if (!prog) {
1333 ret = CPSW_XDP_PASS;
1334 goto out;
1335 }
1336
1337 act = bpf_prog_run_xdp(prog, xdp);
1338 switch (act) {
1339 case XDP_PASS:
1340 ret = CPSW_XDP_PASS;
1341 break;
1342 case XDP_TX:
1343 xdpf = convert_to_xdp_frame(xdp);
1344 if (unlikely(!xdpf))
1345 goto drop;
1346
1347 cpsw_xdp_tx_frame(priv, xdpf, page, port);
1348 break;
1349 case XDP_REDIRECT:
1350 if (xdp_do_redirect(ndev, xdp, prog))
1351 goto drop;
1352
1353 /* Have to flush here, per packet, instead of doing it in bulk
1354 * at the end of the napi handler. The RX devices on this
1355 * particular hardware is sharing a common queue, so the
1356 * incoming device might change per packet.
1357 */
1358 xdp_do_flush_map();
1359 break;
1360 default:
1361 bpf_warn_invalid_xdp_action(act);
1362 /* fall through */
1363 case XDP_ABORTED:
1364 trace_xdp_exception(ndev, prog, act);
1365 /* fall through -- handle aborts by dropping packet */
1366 case XDP_DROP:
1367 goto drop;
1368 }
1369 out:
1370 rcu_read_unlock();
1371 return ret;
1372 drop:
1373 rcu_read_unlock();
1374 page_pool_recycle_direct(cpsw->page_pool[ch], page);
1375 return ret;
1376 }
Linux with added FPC-III support