drivers/net/ethernet/microchip/sparx5/sparx5_fdma.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /* Microchip Sparx5 Switch driver
   3  *
   4  * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries.
   5  *
   6  * The Sparx5 Chip Register Model can be browsed at this location:
   7  * https://github.com/microchip-ung/sparx-5_reginfo
   8  */
   9
  10 #include <linux/types.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/netdevice.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/ip.h>
  15 #include <linux/dma-mapping.h>
  16
  17 #include "sparx5_main_regs.h"
  18 #include "sparx5_main.h"
  19 #include "sparx5_port.h"
  20
  21 #define FDMA_XTR_CHANNEL                6
  22 #define FDMA_INJ_CHANNEL                0
  23
  24 #define FDMA_XTR_BUFFER_SIZE            2048
  25 #define FDMA_WEIGHT                     4
  26
  27 static int sparx5_fdma_tx_dataptr_cb(struct fdma *fdma, int dcb, int db,
  28                                      u64 *dataptr)
  29 {
  30         *dataptr = fdma->dma + (sizeof(struct fdma_dcb) * fdma->n_dcbs) +
  31                    ((dcb * fdma->n_dbs + db) * fdma->db_size);
  32
  33         return 0;
  34 }
  35
  36 static int sparx5_fdma_rx_dataptr_cb(struct fdma *fdma, int dcb, int db,
  37                                      u64 *dataptr)
  38 {
  39         struct sparx5 *sparx5 = fdma->priv;
  40         struct sparx5_rx *rx = &sparx5->rx;
  41         struct sk_buff *skb;
  42
  43         skb = __netdev_alloc_skb(rx->ndev, fdma->db_size, GFP_ATOMIC);
  44         if (unlikely(!skb))
  45                 return -ENOMEM;
  46
  47         *dataptr = virt_to_phys(skb->data);
  48
  49         rx->skb[dcb][db] = skb;
  50
  51         return 0;
  52 }
  53
  54 static void sparx5_fdma_rx_activate(struct sparx5 *sparx5, struct sparx5_rx *rx)
  55 {
  56         struct fdma *fdma = &rx->fdma;
  57
  58         /* Write the buffer address in the LLP and LLP1 regs */
  59         spx5_wr(((u64)fdma->dma) & GENMASK(31, 0), sparx5,
  60                 FDMA_DCB_LLP(fdma->channel_id));
  61         spx5_wr(((u64)fdma->dma) >> 32, sparx5,
  62                 FDMA_DCB_LLP1(fdma->channel_id));
  63
  64         /* Set the number of RX DBs to be used, and DB end-of-frame interrupt */
  65         spx5_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) |
  66                 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) |
  67                 FDMA_CH_CFG_CH_INJ_PORT_SET(XTR_QUEUE),
  68                 sparx5, FDMA_CH_CFG(fdma->channel_id));
  69
  70         /* Set the RX Watermark to max */
  71         spx5_rmw(FDMA_XTR_CFG_XTR_FIFO_WM_SET(31), FDMA_XTR_CFG_XTR_FIFO_WM,
  72                  sparx5,
  73                  FDMA_XTR_CFG);
  74
  75         /* Start RX fdma */
  76         spx5_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(0), FDMA_PORT_CTRL_XTR_STOP,
  77                  sparx5, FDMA_PORT_CTRL(0));
  78
  79         /* Enable RX channel DB interrupt */
  80         spx5_rmw(BIT(fdma->channel_id),
  81                  BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,
  82                  sparx5, FDMA_INTR_DB_ENA);
  83
  84         /* Activate the RX channel */
  85         spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_ACTIVATE);
  86 }
  87
  88 static void sparx5_fdma_rx_deactivate(struct sparx5 *sparx5, struct sparx5_rx *rx)
  89 {
  90         struct fdma *fdma = &rx->fdma;
  91
  92         /* Deactivate the RX channel */
  93         spx5_rmw(0, BIT(fdma->channel_id) & FDMA_CH_ACTIVATE_CH_ACTIVATE,
  94                  sparx5, FDMA_CH_ACTIVATE);
  95
  96         /* Disable RX channel DB interrupt */
  97         spx5_rmw(0, BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,
  98                  sparx5, FDMA_INTR_DB_ENA);
  99
 100         /* Stop RX fdma */
 101         spx5_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(1), FDMA_PORT_CTRL_XTR_STOP,
 102                  sparx5, FDMA_PORT_CTRL(0));
 103 }
 104
 105 static void sparx5_fdma_tx_activate(struct sparx5 *sparx5, struct sparx5_tx *tx)
 106 {
 107         struct fdma *fdma = &tx->fdma;
 108
 109         /* Write the buffer address in the LLP and LLP1 regs */
 110         spx5_wr(((u64)fdma->dma) & GENMASK(31, 0), sparx5,
 111                 FDMA_DCB_LLP(fdma->channel_id));
 112         spx5_wr(((u64)fdma->dma) >> 32, sparx5,
 113                 FDMA_DCB_LLP1(fdma->channel_id));
 114
 115         /* Set the number of TX DBs to be used, and DB end-of-frame interrupt */
 116         spx5_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) |
 117                 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) |
 118                 FDMA_CH_CFG_CH_INJ_PORT_SET(INJ_QUEUE),
 119                 sparx5, FDMA_CH_CFG(fdma->channel_id));
 120
 121         /* Start TX fdma */
 122         spx5_rmw(FDMA_PORT_CTRL_INJ_STOP_SET(0), FDMA_PORT_CTRL_INJ_STOP,
 123                  sparx5, FDMA_PORT_CTRL(0));
 124
 125         /* Activate the channel */
 126         spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_ACTIVATE);
 127 }
 128
 129 static void sparx5_fdma_tx_deactivate(struct sparx5 *sparx5, struct sparx5_tx *tx)
 130 {
 131         /* Disable the channel */
 132         spx5_rmw(0, BIT(tx->fdma.channel_id) & FDMA_CH_ACTIVATE_CH_ACTIVATE,
 133                  sparx5, FDMA_CH_ACTIVATE);
 134 }
 135
 136 static void sparx5_fdma_reload(struct sparx5 *sparx5, struct fdma *fdma)
 137 {
 138         /* Reload the RX channel */
 139         spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_RELOAD);
 140 }
 141
 142 static bool sparx5_fdma_rx_get_frame(struct sparx5 *sparx5, struct sparx5_rx *rx)
 143 {
 144         struct fdma *fdma = &rx->fdma;
 145         struct sparx5_port *port;
 146         struct fdma_db *db_hw;
 147         struct frame_info fi;
 148         struct sk_buff *skb;
 149
 150         /* Check if the DCB is done */
 151         db_hw = fdma_db_next_get(fdma);
 152         if (unlikely(!fdma_db_is_done(db_hw)))
 153                 return false;
 154         skb = rx->skb[fdma->dcb_index][fdma->db_index];
 155         skb_put(skb, fdma_db_len_get(db_hw));
 156         /* Now do the normal processing of the skb */
 157         sparx5_ifh_parse(sparx5, (u32 *)skb->data, &fi);
 158         /* Map to port netdev */
 159         port = fi.src_port < sparx5->data->consts->n_ports ?
 160                        sparx5->ports[fi.src_port] :
 161                        NULL;
 162         if (!port || !port->ndev) {
 163                 dev_err(sparx5->dev, "Data on inactive port %d\n", fi.src_port);
 164                 sparx5_xtr_flush(sparx5, XTR_QUEUE);
 165                 return false;
 166         }
 167         skb->dev = port->ndev;
 168         skb_pull(skb, IFH_LEN * sizeof(u32));
 169         if (likely(!(skb->dev->features & NETIF_F_RXFCS)))
 170                 skb_trim(skb, skb->len - ETH_FCS_LEN);
 171
 172         sparx5_ptp_rxtstamp(sparx5, skb, fi.timestamp);
 173         skb->protocol = eth_type_trans(skb, skb->dev);
 174         /* Everything we see on an interface that is in the HW bridge
 175          * has already been forwarded
 176          */
 177         if (test_bit(port->portno, sparx5->bridge_mask))
 178                 skb->offload_fwd_mark = 1;
 179         skb->dev->stats.rx_bytes += skb->len;
 180         skb->dev->stats.rx_packets++;
 181         rx->packets++;
 182         netif_receive_skb(skb);
 183         return true;
 184 }
 185
 186 static int sparx5_fdma_napi_callback(struct napi_struct *napi, int weight)
 187 {
 188         struct sparx5_rx *rx = container_of(napi, struct sparx5_rx, napi);
 189         struct sparx5 *sparx5 = container_of(rx, struct sparx5, rx);
 190         struct fdma *fdma = &rx->fdma;
 191         int counter = 0;
 192
 193         while (counter < weight && sparx5_fdma_rx_get_frame(sparx5, rx)) {
 194                 fdma_db_advance(fdma);
 195                 counter++;
 196                 /* Check if the DCB can be reused */
 197                 if (fdma_dcb_is_reusable(fdma))
 198                         continue;
 199                 fdma_dcb_add(fdma, fdma->dcb_index,
 200                              FDMA_DCB_INFO_DATAL(fdma->db_size),
 201                              FDMA_DCB_STATUS_INTR);
 202                 fdma_db_reset(fdma);
 203                 fdma_dcb_advance(fdma);
 204         }
 205         if (counter < weight) {
 206                 napi_complete_done(&rx->napi, counter);
 207                 spx5_rmw(BIT(fdma->channel_id),
 208                          BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,
 209                          sparx5, FDMA_INTR_DB_ENA);
 210         }
 211         if (counter)
 212                 sparx5_fdma_reload(sparx5, fdma);
 213         return counter;
 214 }
 215
 216 int sparx5_fdma_xmit(struct sparx5 *sparx5, u32 *ifh, struct sk_buff *skb)
 217 {
 218         struct sparx5_tx *tx = &sparx5->tx;
 219         struct fdma *fdma = &tx->fdma;
 220         static bool first_time = true;
 221         void *virt_addr;
 222
 223         fdma_dcb_advance(fdma);
 224         if (!fdma_db_is_done(fdma_db_get(fdma, fdma->dcb_index, 0)))
 225                 return -EINVAL;
 226
 227         /* Get the virtual address of the dataptr for the next DB */
 228         virt_addr = ((u8 *)fdma->dcbs +
 229                      (sizeof(struct fdma_dcb) * fdma->n_dcbs) +
 230                      ((fdma->dcb_index * fdma->n_dbs) * fdma->db_size));
 231
 232         memcpy(virt_addr, ifh, IFH_LEN * 4);
 233         memcpy(virt_addr + IFH_LEN * 4, skb->data, skb->len);
 234
 235         fdma_dcb_add(fdma, fdma->dcb_index, 0,
 236                      FDMA_DCB_STATUS_SOF |
 237                      FDMA_DCB_STATUS_EOF |
 238                      FDMA_DCB_STATUS_BLOCKO(0) |
 239                      FDMA_DCB_STATUS_BLOCKL(skb->len + IFH_LEN * 4 + 4));
 240
 241         if (first_time) {
 242                 sparx5_fdma_tx_activate(sparx5, tx);
 243                 first_time = false;
 244         } else {
 245                 sparx5_fdma_reload(sparx5, fdma);
 246         }
 247         return NETDEV_TX_OK;
 248 }
 249
 250 static int sparx5_fdma_rx_alloc(struct sparx5 *sparx5)
 251 {
 252         struct sparx5_rx *rx = &sparx5->rx;
 253         struct fdma *fdma = &rx->fdma;
 254         int err;
 255
 256         err = fdma_alloc_phys(fdma);
 257         if (err)
 258                 return err;
 259
 260         fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size),
 261                        FDMA_DCB_STATUS_INTR);
 262
 263         netif_napi_add_weight(rx->ndev, &rx->napi, sparx5_fdma_napi_callback,
 264                               FDMA_WEIGHT);
 265         napi_enable(&rx->napi);
 266         sparx5_fdma_rx_activate(sparx5, rx);
 267         return 0;
 268 }
 269
 270 static int sparx5_fdma_tx_alloc(struct sparx5 *sparx5)
 271 {
 272         struct sparx5_tx *tx = &sparx5->tx;
 273         struct fdma *fdma = &tx->fdma;
 274         int err;
 275
 276         err = fdma_alloc_phys(fdma);
 277         if (err)
 278                 return err;
 279
 280         fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size),
 281                        FDMA_DCB_STATUS_DONE);
 282
 283         return 0;
 284 }
 285
 286 static void sparx5_fdma_rx_init(struct sparx5 *sparx5,
 287                                 struct sparx5_rx *rx, int channel)
 288 {
 289         struct fdma *fdma = &rx->fdma;
 290         int idx;
 291
 292         fdma->channel_id = channel;
 293         fdma->n_dcbs = FDMA_DCB_MAX;
 294         fdma->n_dbs = FDMA_RX_DCB_MAX_DBS;
 295         fdma->priv = sparx5;
 296         fdma->db_size = ALIGN(FDMA_XTR_BUFFER_SIZE, PAGE_SIZE);
 297         fdma->size = fdma_get_size(&sparx5->rx.fdma);
 298         fdma->ops.dataptr_cb = &sparx5_fdma_rx_dataptr_cb;
 299         fdma->ops.nextptr_cb = &fdma_nextptr_cb;
 300         /* Fetch a netdev for SKB and NAPI use, any will do */
 301         for (idx = 0; idx < sparx5->data->consts->n_ports; ++idx) {
 302                 struct sparx5_port *port = sparx5->ports[idx];
 303
 304                 if (port && port->ndev) {
 305                         rx->ndev = port->ndev;
 306                         break;
 307                 }
 308         }
 309 }
 310
 311 static void sparx5_fdma_tx_init(struct sparx5 *sparx5,
 312                                 struct sparx5_tx *tx, int channel)
 313 {
 314         struct fdma *fdma = &tx->fdma;
 315
 316         fdma->channel_id = channel;
 317         fdma->n_dcbs = FDMA_DCB_MAX;
 318         fdma->n_dbs = FDMA_TX_DCB_MAX_DBS;
 319         fdma->priv = sparx5;
 320         fdma->db_size = ALIGN(FDMA_XTR_BUFFER_SIZE, PAGE_SIZE);
 321         fdma->size = fdma_get_size_contiguous(&sparx5->tx.fdma);
 322         fdma->ops.dataptr_cb = &sparx5_fdma_tx_dataptr_cb;
 323         fdma->ops.nextptr_cb = &fdma_nextptr_cb;
 324 }
 325
 326 irqreturn_t sparx5_fdma_handler(int irq, void *args)
 327 {
 328         struct sparx5 *sparx5 = args;
 329         u32 db = 0, err = 0;
 330
 331         db = spx5_rd(sparx5, FDMA_INTR_DB);
 332         err = spx5_rd(sparx5, FDMA_INTR_ERR);
 333         /* Clear interrupt */
 334         if (db) {
 335                 spx5_wr(0, sparx5, FDMA_INTR_DB_ENA);
 336                 spx5_wr(db, sparx5, FDMA_INTR_DB);
 337                 napi_schedule(&sparx5->rx.napi);
 338         }
 339         if (err) {
 340                 u32 err_type = spx5_rd(sparx5, FDMA_ERRORS);
 341
 342                 dev_err_ratelimited(sparx5->dev,
 343                                     "ERR: int: %#x, type: %#x\n",
 344                                     err, err_type);
 345                 spx5_wr(err, sparx5, FDMA_INTR_ERR);
 346                 spx5_wr(err_type, sparx5, FDMA_ERRORS);
 347         }
 348         return IRQ_HANDLED;
 349 }
 350
 351 static void sparx5_fdma_injection_mode(struct sparx5 *sparx5)
 352 {
 353         const int byte_swap = 1;
 354         int portno;
 355         int urgency;
 356
 357         /* Change mode to fdma extraction and injection */
 358         spx5_wr(QS_XTR_GRP_CFG_MODE_SET(2) |
 359                 QS_XTR_GRP_CFG_STATUS_WORD_POS_SET(1) |
 360                 QS_XTR_GRP_CFG_BYTE_SWAP_SET(byte_swap),
 361                 sparx5, QS_XTR_GRP_CFG(XTR_QUEUE));
 362         spx5_wr(QS_INJ_GRP_CFG_MODE_SET(2) |
 363                 QS_INJ_GRP_CFG_BYTE_SWAP_SET(byte_swap),
 364                 sparx5, QS_INJ_GRP_CFG(INJ_QUEUE));
 365
 366         /* CPU ports capture setup */
 367         for (portno = sparx5_get_internal_port(sparx5, SPX5_PORT_CPU_0);
 368              portno <= sparx5_get_internal_port(sparx5, SPX5_PORT_CPU_1);
 369              portno++) {
 370                 /* ASM CPU port: No preamble, IFH, enable padding */
 371                 spx5_wr(ASM_PORT_CFG_PAD_ENA_SET(1) |
 372                         ASM_PORT_CFG_NO_PREAMBLE_ENA_SET(1) |
 373                         ASM_PORT_CFG_INJ_FORMAT_CFG_SET(1), /* 1 = IFH */
 374                         sparx5, ASM_PORT_CFG(portno));
 375
 376                 /* Reset WM cnt to unclog queued frames */
 377                 spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_SET(1),
 378                          DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR,
 379                          sparx5,
 380                          DSM_DEV_TX_STOP_WM_CFG(portno));
 381
 382                 /* Set Disassembler Stop Watermark level */
 383                 spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_SET(100),
 384                          DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM,
 385                          sparx5,
 386                          DSM_DEV_TX_STOP_WM_CFG(portno));
 387
 388                 /* Enable port in queue system */
 389                 urgency = sparx5_port_fwd_urg(sparx5, SPEED_2500);
 390                 spx5_rmw(QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(1) |
 391                          QFWD_SWITCH_PORT_MODE_FWD_URGENCY_SET(urgency),
 392                          QFWD_SWITCH_PORT_MODE_PORT_ENA |
 393                          QFWD_SWITCH_PORT_MODE_FWD_URGENCY,
 394                          sparx5,
 395                          QFWD_SWITCH_PORT_MODE(portno));
 396
 397                 /* Disable Disassembler buffer underrun watchdog
 398                  * to avoid truncated packets in XTR
 399                  */
 400                 spx5_rmw(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS_SET(1),
 401                          DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS,
 402                          sparx5,
 403                          DSM_BUF_CFG(portno));
 404
 405                 /* Disabling frame aging */
 406                 spx5_rmw(HSCH_PORT_MODE_AGE_DIS_SET(1),
 407                          HSCH_PORT_MODE_AGE_DIS,
 408                          sparx5,
 409                          HSCH_PORT_MODE(portno));
 410         }
 411 }
 412
 413 int sparx5_fdma_start(struct sparx5 *sparx5)
 414 {
 415         int err;
 416
 417         /* Reset FDMA state */
 418         spx5_wr(FDMA_CTRL_NRESET_SET(0), sparx5, FDMA_CTRL);
 419         spx5_wr(FDMA_CTRL_NRESET_SET(1), sparx5, FDMA_CTRL);
 420
 421         /* Force ACP caching but disable read/write allocation */
 422         spx5_rmw(CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA_SET(1) |
 423                  CPU_PROC_CTRL_ACP_AWCACHE_SET(0) |
 424                  CPU_PROC_CTRL_ACP_ARCACHE_SET(0),
 425                  CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA |
 426                  CPU_PROC_CTRL_ACP_AWCACHE |
 427                  CPU_PROC_CTRL_ACP_ARCACHE,
 428                  sparx5, CPU_PROC_CTRL);
 429
 430         sparx5_fdma_injection_mode(sparx5);
 431         sparx5_fdma_rx_init(sparx5, &sparx5->rx, FDMA_XTR_CHANNEL);
 432         sparx5_fdma_tx_init(sparx5, &sparx5->tx, FDMA_INJ_CHANNEL);
 433         err = sparx5_fdma_rx_alloc(sparx5);
 434         if (err) {
 435                 dev_err(sparx5->dev, "Could not allocate RX buffers: %d\n", err);
 436                 return err;
 437         }
 438         err = sparx5_fdma_tx_alloc(sparx5);
 439         if (err) {
 440                 dev_err(sparx5->dev, "Could not allocate TX buffers: %d\n", err);
 441                 return err;
 442         }
 443         return err;
 444 }
 445
 446 static u32 sparx5_fdma_port_ctrl(struct sparx5 *sparx5)
 447 {
 448         return spx5_rd(sparx5, FDMA_PORT_CTRL(0));
 449 }
 450
 451 int sparx5_fdma_stop(struct sparx5 *sparx5)
 452 {
 453         u32 val;
 454
 455         napi_disable(&sparx5->rx.napi);
 456         /* Stop the fdma and channel interrupts */
 457         sparx5_fdma_rx_deactivate(sparx5, &sparx5->rx);
 458         sparx5_fdma_tx_deactivate(sparx5, &sparx5->tx);
 459         /* Wait for the RX channel to stop */
 460         read_poll_timeout(sparx5_fdma_port_ctrl, val,
 461                           FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY_GET(val) == 0,
 462                           500, 10000, 0, sparx5);
 463         fdma_free_phys(&sparx5->rx.fdma);
 464         fdma_free_phys(&sparx5->tx.fdma);
 465         return 0;
 466 }