drivers/net/ethernet/hisilicon/hibmcge/hbg_txrx.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright (c) 2024 Hisilicon Limited.
   3
   4 #include <net/netdev_queues.h>
   5 #include "hbg_common.h"
   6 #include "hbg_irq.h"
   7 #include "hbg_reg.h"
   8 #include "hbg_txrx.h"
   9
  10 #define netdev_get_tx_ring(netdev) \
  11                         (&(((struct hbg_priv *)netdev_priv(netdev))->tx_ring))
  12
  13 #define buffer_to_dma_dir(buffer) (((buffer)->dir == HBG_DIR_RX) ? \
  14                                    DMA_FROM_DEVICE : DMA_TO_DEVICE)
  15
  16 #define hbg_queue_used_num(head, tail, ring) ({ \
  17         typeof(ring) _ring = (ring); \
  18         ((tail) + _ring->len - (head)) % _ring->len; })
  19 #define hbg_queue_left_num(head, tail, ring) ({ \
  20         typeof(ring) _r = (ring); \
  21         _r->len - hbg_queue_used_num((head), (tail), _r) - 1; })
  22 #define hbg_queue_is_empty(head, tail, ring) \
  23         (hbg_queue_used_num((head), (tail), (ring)) == 0)
  24 #define hbg_queue_is_full(head, tail, ring) \
  25         (hbg_queue_left_num((head), (tail), (ring)) == 0)
  26 #define hbg_queue_next_prt(p, ring) (((p) + 1) % (ring)->len)
  27 #define hbg_queue_move_next(p, ring) ({ \
  28         typeof(ring) _ring = (ring); \
  29         _ring->p = hbg_queue_next_prt(_ring->p, _ring); })
  30
  31 #define HBG_TX_STOP_THRS        2
  32 #define HBG_TX_START_THRS       (2 * HBG_TX_STOP_THRS)
  33
  34 static int hbg_dma_map(struct hbg_buffer *buffer)
  35 {
  36         struct hbg_priv *priv = buffer->priv;
  37
  38         buffer->skb_dma = dma_map_single(&priv->pdev->dev,
  39                                          buffer->skb->data, buffer->skb_len,
  40                                          buffer_to_dma_dir(buffer));
  41         if (unlikely(dma_mapping_error(&priv->pdev->dev, buffer->skb_dma)))
  42                 return -ENOMEM;
  43
  44         return 0;
  45 }
  46
  47 static void hbg_dma_unmap(struct hbg_buffer *buffer)
  48 {
  49         struct hbg_priv *priv = buffer->priv;
  50
  51         if (unlikely(!buffer->skb_dma))
  52                 return;
  53
  54         dma_unmap_single(&priv->pdev->dev, buffer->skb_dma, buffer->skb_len,
  55                          buffer_to_dma_dir(buffer));
  56         buffer->skb_dma = 0;
  57 }
  58
  59 static void hbg_init_tx_desc(struct hbg_buffer *buffer,
  60                              struct hbg_tx_desc *tx_desc)
  61 {
  62         u32 ip_offset = buffer->skb->network_header - buffer->skb->mac_header;
  63         u32 word0 = 0;
  64
  65         word0 |= FIELD_PREP(HBG_TX_DESC_W0_WB_B, HBG_STATUS_ENABLE);
  66         word0 |= FIELD_PREP(HBG_TX_DESC_W0_IP_OFF_M, ip_offset);
  67         if (likely(buffer->skb->ip_summed == CHECKSUM_PARTIAL)) {
  68                 word0 |= FIELD_PREP(HBG_TX_DESC_W0_l3_CS_B, HBG_STATUS_ENABLE);
  69                 word0 |= FIELD_PREP(HBG_TX_DESC_W0_l4_CS_B, HBG_STATUS_ENABLE);
  70         }
  71
  72         tx_desc->word0 = word0;
  73         tx_desc->word1 = FIELD_PREP(HBG_TX_DESC_W1_SEND_LEN_M,
  74                                     buffer->skb->len);
  75         tx_desc->word2 = buffer->skb_dma;
  76         tx_desc->word3 = buffer->state_dma;
  77 }
  78
  79 netdev_tx_t hbg_net_start_xmit(struct sk_buff *skb, struct net_device *netdev)
  80 {
  81         struct hbg_ring *ring = netdev_get_tx_ring(netdev);
  82         struct hbg_priv *priv = netdev_priv(netdev);
  83         /* This smp_load_acquire() pairs with smp_store_release() in
  84          * hbg_napi_tx_recycle() called in tx interrupt handle process.
  85          */
  86         u32 ntc = smp_load_acquire(&ring->ntc);
  87         struct hbg_buffer *buffer;
  88         struct hbg_tx_desc tx_desc;
  89         u32 ntu = ring->ntu;
  90
  91         if (unlikely(!skb->len ||
  92                      skb->len > hbg_spec_max_frame_len(priv, HBG_DIR_TX))) {
  93                 dev_kfree_skb_any(skb);
  94                 netdev->stats.tx_errors++;
  95                 return NETDEV_TX_OK;
  96         }
  97
  98         if (!netif_subqueue_maybe_stop(netdev, 0,
  99                                        hbg_queue_left_num(ntc, ntu, ring),
 100                                        HBG_TX_STOP_THRS, HBG_TX_START_THRS))
 101                 return NETDEV_TX_BUSY;
 102
 103         buffer = &ring->queue[ntu];
 104         buffer->skb = skb;
 105         buffer->skb_len = skb->len;
 106         if (unlikely(hbg_dma_map(buffer))) {
 107                 dev_kfree_skb_any(skb);
 108                 return NETDEV_TX_OK;
 109         }
 110
 111         buffer->state = HBG_TX_STATE_START;
 112         hbg_init_tx_desc(buffer, &tx_desc);
 113         hbg_hw_set_tx_desc(priv, &tx_desc);
 114
 115         /* This smp_store_release() pairs with smp_load_acquire() in
 116          * hbg_napi_tx_recycle() called in tx interrupt handle process.
 117          */
 118         smp_store_release(&ring->ntu, hbg_queue_next_prt(ntu, ring));
 119         dev_sw_netstats_tx_add(netdev, 1, skb->len);
 120         return NETDEV_TX_OK;
 121 }
 122
 123 static void hbg_buffer_free_skb(struct hbg_buffer *buffer)
 124 {
 125         if (unlikely(!buffer->skb))
 126                 return;
 127
 128         dev_kfree_skb_any(buffer->skb);
 129         buffer->skb = NULL;
 130 }
 131
 132 static int hbg_buffer_alloc_skb(struct hbg_buffer *buffer)
 133 {
 134         u32 len = hbg_spec_max_frame_len(buffer->priv, buffer->dir);
 135         struct hbg_priv *priv = buffer->priv;
 136
 137         buffer->skb = netdev_alloc_skb(priv->netdev, len);
 138         if (unlikely(!buffer->skb))
 139                 return -ENOMEM;
 140
 141         buffer->skb_len = len;
 142         memset(buffer->skb->data, 0, HBG_PACKET_HEAD_SIZE);
 143         return 0;
 144 }
 145
 146 static void hbg_buffer_free(struct hbg_buffer *buffer)
 147 {
 148         hbg_dma_unmap(buffer);
 149         hbg_buffer_free_skb(buffer);
 150 }
 151
 152 static int hbg_napi_tx_recycle(struct napi_struct *napi, int budget)
 153 {
 154         struct hbg_ring *ring = container_of(napi, struct hbg_ring, napi);
 155         /* This smp_load_acquire() pairs with smp_store_release() in
 156          * hbg_net_start_xmit() called in xmit process.
 157          */
 158         u32 ntu = smp_load_acquire(&ring->ntu);
 159         struct hbg_priv *priv = ring->priv;
 160         struct hbg_buffer *buffer;
 161         u32 ntc = ring->ntc;
 162         int packet_done = 0;
 163
 164         /* We need do cleanup even if budget is 0.
 165          * Per NAPI documentation budget is for Rx.
 166          * So We hardcode the amount of work Tx NAPI does to 128.
 167          */
 168         budget = 128;
 169         while (packet_done < budget) {
 170                 if (unlikely(hbg_queue_is_empty(ntc, ntu, ring)))
 171                         break;
 172
 173                 /* make sure HW write desc complete */
 174                 dma_rmb();
 175
 176                 buffer = &ring->queue[ntc];
 177                 if (buffer->state != HBG_TX_STATE_COMPLETE)
 178                         break;
 179
 180                 hbg_buffer_free(buffer);
 181                 ntc = hbg_queue_next_prt(ntc, ring);
 182                 packet_done++;
 183         }
 184
 185         /* This smp_store_release() pairs with smp_load_acquire() in
 186          * hbg_net_start_xmit() called in xmit process.
 187          */
 188         smp_store_release(&ring->ntc, ntc);
 189         netif_wake_queue(priv->netdev);
 190
 191         if (likely(packet_done < budget &&
 192                    napi_complete_done(napi, packet_done)))
 193                 hbg_hw_irq_enable(priv, HBG_INT_MSK_TX_B, true);
 194
 195         return packet_done;
 196 }
 197
 198 static int hbg_rx_fill_one_buffer(struct hbg_priv *priv)
 199 {
 200         struct hbg_ring *ring = &priv->rx_ring;
 201         struct hbg_buffer *buffer;
 202         int ret;
 203
 204         if (hbg_queue_is_full(ring->ntc, ring->ntu, ring))
 205                 return 0;
 206
 207         buffer = &ring->queue[ring->ntu];
 208         ret = hbg_buffer_alloc_skb(buffer);
 209         if (unlikely(ret))
 210                 return ret;
 211
 212         ret = hbg_dma_map(buffer);
 213         if (unlikely(ret)) {
 214                 hbg_buffer_free_skb(buffer);
 215                 return ret;
 216         }
 217
 218         hbg_hw_fill_buffer(priv, buffer->skb_dma);
 219         hbg_queue_move_next(ntu, ring);
 220         return 0;
 221 }
 222
 223 static bool hbg_sync_data_from_hw(struct hbg_priv *priv,
 224                                   struct hbg_buffer *buffer)
 225 {
 226         struct hbg_rx_desc *rx_desc;
 227
 228         /* make sure HW write desc complete */
 229         dma_rmb();
 230
 231         dma_sync_single_for_cpu(&priv->pdev->dev, buffer->skb_dma,
 232                                 buffer->skb_len, DMA_FROM_DEVICE);
 233
 234         rx_desc = (struct hbg_rx_desc *)buffer->skb->data;
 235         return FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2) != 0;
 236 }
 237
 238 static int hbg_napi_rx_poll(struct napi_struct *napi, int budget)
 239 {
 240         struct hbg_ring *ring = container_of(napi, struct hbg_ring, napi);
 241         struct hbg_priv *priv = ring->priv;
 242         struct hbg_rx_desc *rx_desc;
 243         struct hbg_buffer *buffer;
 244         u32 packet_done = 0;
 245         u32 pkt_len;
 246
 247         while (packet_done < budget) {
 248                 if (unlikely(hbg_queue_is_empty(ring->ntc, ring->ntu, ring)))
 249                         break;
 250
 251                 buffer = &ring->queue[ring->ntc];
 252                 if (unlikely(!buffer->skb))
 253                         goto next_buffer;
 254
 255                 if (unlikely(!hbg_sync_data_from_hw(priv, buffer)))
 256                         break;
 257                 rx_desc = (struct hbg_rx_desc *)buffer->skb->data;
 258                 pkt_len = FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2);
 259
 260                 hbg_dma_unmap(buffer);
 261
 262                 skb_reserve(buffer->skb, HBG_PACKET_HEAD_SIZE + NET_IP_ALIGN);
 263                 skb_put(buffer->skb, pkt_len);
 264                 buffer->skb->protocol = eth_type_trans(buffer->skb,
 265                                                        priv->netdev);
 266
 267                 dev_sw_netstats_rx_add(priv->netdev, pkt_len);
 268                 napi_gro_receive(napi, buffer->skb);
 269                 buffer->skb = NULL;
 270
 271 next_buffer:
 272                 hbg_rx_fill_one_buffer(priv);
 273                 hbg_queue_move_next(ntc, ring);
 274                 packet_done++;
 275         }
 276
 277         if (likely(packet_done < budget &&
 278                    napi_complete_done(napi, packet_done)))
 279                 hbg_hw_irq_enable(priv, HBG_INT_MSK_RX_B, true);
 280
 281         return packet_done;
 282 }
 283
 284 static void hbg_ring_uninit(struct hbg_ring *ring)
 285 {
 286         struct hbg_buffer *buffer;
 287         u32 i;
 288
 289         if (!ring->queue)
 290                 return;
 291
 292         napi_disable(&ring->napi);
 293         netif_napi_del(&ring->napi);
 294
 295         for (i = 0; i < ring->len; i++) {
 296                 buffer = &ring->queue[i];
 297                 hbg_buffer_free(buffer);
 298                 buffer->ring = NULL;
 299                 buffer->priv = NULL;
 300         }
 301
 302         dma_free_coherent(&ring->priv->pdev->dev,
 303                           ring->len * sizeof(*ring->queue),
 304                           ring->queue, ring->queue_dma);
 305         ring->queue = NULL;
 306         ring->queue_dma = 0;
 307         ring->len = 0;
 308         ring->priv = NULL;
 309 }
 310
 311 static int hbg_ring_init(struct hbg_priv *priv, struct hbg_ring *ring,
 312                          int (*napi_poll)(struct napi_struct *, int),
 313                          enum hbg_dir dir)
 314 {
 315         struct hbg_buffer *buffer;
 316         u32 i, len;
 317
 318         len = hbg_get_spec_fifo_max_num(priv, dir) + 1;
 319         ring->queue = dma_alloc_coherent(&priv->pdev->dev,
 320                                          len * sizeof(*ring->queue),
 321                                          &ring->queue_dma, GFP_KERNEL);
 322         if (!ring->queue)
 323                 return -ENOMEM;
 324
 325         for (i = 0; i < len; i++) {
 326                 buffer = &ring->queue[i];
 327                 buffer->skb_len = 0;
 328                 buffer->dir = dir;
 329                 buffer->ring = ring;
 330                 buffer->priv = priv;
 331                 buffer->state_dma = ring->queue_dma + (i * sizeof(*buffer));
 332         }
 333
 334         ring->dir = dir;
 335         ring->priv = priv;
 336         ring->ntc = 0;
 337         ring->ntu = 0;
 338         ring->len = len;
 339
 340         if (dir == HBG_DIR_TX)
 341                 netif_napi_add_tx(priv->netdev, &ring->napi, napi_poll);
 342         else
 343                 netif_napi_add(priv->netdev, &ring->napi, napi_poll);
 344
 345         napi_enable(&ring->napi);
 346         return 0;
 347 }
 348
 349 static int hbg_tx_ring_init(struct hbg_priv *priv)
 350 {
 351         struct hbg_ring *tx_ring = &priv->tx_ring;
 352
 353         if (!tx_ring->tout_log_buf)
 354                 tx_ring->tout_log_buf = devm_kmalloc(&priv->pdev->dev,
 355                                                      HBG_TX_TIMEOUT_BUF_LEN,
 356                                                      GFP_KERNEL);
 357
 358         if (!tx_ring->tout_log_buf)
 359                 return -ENOMEM;
 360
 361         return hbg_ring_init(priv, tx_ring, hbg_napi_tx_recycle, HBG_DIR_TX);
 362 }
 363
 364 static int hbg_rx_ring_init(struct hbg_priv *priv)
 365 {
 366         int ret;
 367         u32 i;
 368
 369         ret = hbg_ring_init(priv, &priv->rx_ring, hbg_napi_rx_poll, HBG_DIR_RX);
 370         if (ret)
 371                 return ret;
 372
 373         for (i = 0; i < priv->rx_ring.len - 1; i++) {
 374                 ret = hbg_rx_fill_one_buffer(priv);
 375                 if (ret) {
 376                         hbg_ring_uninit(&priv->rx_ring);
 377                         return ret;
 378                 }
 379         }
 380
 381         return 0;
 382 }
 383
 384 int hbg_txrx_init(struct hbg_priv *priv)
 385 {
 386         int ret;
 387
 388         ret = hbg_tx_ring_init(priv);
 389         if (ret) {
 390                 dev_err(&priv->pdev->dev,
 391                         "failed to init tx ring, ret = %d\n", ret);
 392                 return ret;
 393         }
 394
 395         ret = hbg_rx_ring_init(priv);
 396         if (ret) {
 397                 dev_err(&priv->pdev->dev,
 398                         "failed to init rx ring, ret = %d\n", ret);
 399                 hbg_ring_uninit(&priv->tx_ring);
 400         }
 401
 402         return ret;
 403 }
 404
 405 void hbg_txrx_uninit(struct hbg_priv *priv)
 406 {
 407         hbg_ring_uninit(&priv->tx_ring);
 408         hbg_ring_uninit(&priv->rx_ring);
 409 }