drivers/net/wireless/mediatek/mt7601u/dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2015 Jakub Kicinski <kubakici@wp.pl>
   4  */
   5
   6 #include "mt7601u.h"
   7 #include "dma.h"
   8 #include "usb.h"
   9 #include "trace.h"
  10
  11 static int mt7601u_submit_rx_buf(struct mt7601u_dev *dev,
  12                                  struct mt7601u_dma_buf_rx *e, gfp_t gfp);
  13
  14 static unsigned int ieee80211_get_hdrlen_from_buf(const u8 *data, unsigned len)
  15 {
  16         const struct ieee80211_hdr *hdr = (const struct ieee80211_hdr *)data;
  17         unsigned int hdrlen;
  18
  19         if (unlikely(len < 10))
  20                 return 0;
  21         hdrlen = ieee80211_hdrlen(hdr->frame_control);
  22         if (unlikely(hdrlen > len))
  23                 return 0;
  24         return hdrlen;
  25 }
  26
  27 static struct sk_buff *
  28 mt7601u_rx_skb_from_seg(struct mt7601u_dev *dev, struct mt7601u_rxwi *rxwi,
  29                         void *data, u32 seg_len, u32 truesize, struct page *p)
  30 {
  31         struct sk_buff *skb;
  32         u32 true_len, hdr_len = 0, copy, frag;
  33
  34         skb = alloc_skb(p ? 128 : seg_len, GFP_ATOMIC);
  35         if (!skb)
  36                 return NULL;
  37
  38         true_len = mt76_mac_process_rx(dev, skb, data, rxwi);
  39         if (!true_len || true_len > seg_len)
  40                 goto bad_frame;
  41
  42         hdr_len = ieee80211_get_hdrlen_from_buf(data, true_len);
  43         if (!hdr_len)
  44                 goto bad_frame;
  45
  46         if (rxwi->rxinfo & cpu_to_le32(MT_RXINFO_L2PAD)) {
  47                 skb_put_data(skb, data, hdr_len);
  48
  49                 data += hdr_len + 2;
  50                 true_len -= hdr_len;
  51                 hdr_len = 0;
  52         }
  53
  54         /* If not doing paged RX allocated skb will always have enough space */
  55         copy = (true_len <= skb_tailroom(skb)) ? true_len : hdr_len + 8;
  56         frag = true_len - copy;
  57
  58         skb_put_data(skb, data, copy);
  59         data += copy;
  60
  61         if (frag) {
  62                 skb_add_rx_frag(skb, 0, p, data - page_address(p),
  63                                 frag, truesize);
  64                 get_page(p);
  65         }
  66
  67         return skb;
  68
  69 bad_frame:
  70         dev_err_ratelimited(dev->dev, "Error: incorrect frame len:%u hdr:%u\n",
  71                             true_len, hdr_len);
  72         dev_kfree_skb(skb);
  73         return NULL;
  74 }
  75
  76 static void mt7601u_rx_process_seg(struct mt7601u_dev *dev, u8 *data,
  77                                    u32 seg_len, struct page *p)
  78 {
  79         struct sk_buff *skb;
  80         struct mt7601u_rxwi *rxwi;
  81         u32 fce_info, truesize = seg_len;
  82
  83         /* DMA_INFO field at the beginning of the segment contains only some of
  84          * the information, we need to read the FCE descriptor from the end.
  85          */
  86         fce_info = get_unaligned_le32(data + seg_len - MT_FCE_INFO_LEN);
  87         seg_len -= MT_FCE_INFO_LEN;
  88
  89         data += MT_DMA_HDR_LEN;
  90         seg_len -= MT_DMA_HDR_LEN;
  91
  92         rxwi = (struct mt7601u_rxwi *) data;
  93         data += sizeof(struct mt7601u_rxwi);
  94         seg_len -= sizeof(struct mt7601u_rxwi);
  95
  96         if (unlikely(rxwi->zero[0] || rxwi->zero[1] || rxwi->zero[2]))
  97                 dev_err_once(dev->dev, "Error: RXWI zero fields are set\n");
  98         if (unlikely(FIELD_GET(MT_RXD_INFO_TYPE, fce_info)))
  99                 dev_err_once(dev->dev, "Error: RX path seen a non-pkt urb\n");
 100
 101         trace_mt_rx(dev, rxwi, fce_info);
 102
 103         skb = mt7601u_rx_skb_from_seg(dev, rxwi, data, seg_len, truesize, p);
 104         if (!skb)
 105                 return;
 106
 107         spin_lock(&dev->mac_lock);
 108         ieee80211_rx(dev->hw, skb);
 109         spin_unlock(&dev->mac_lock);
 110 }
 111
 112 static u16 mt7601u_rx_next_seg_len(u8 *data, u32 data_len)
 113 {
 114         u32 min_seg_len = MT_DMA_HDR_LEN + MT_RX_INFO_LEN +
 115                 sizeof(struct mt7601u_rxwi) + MT_FCE_INFO_LEN;
 116         u16 dma_len = get_unaligned_le16(data);
 117
 118         if (data_len < min_seg_len ||
 119             WARN_ON_ONCE(!dma_len) ||
 120             WARN_ON_ONCE(dma_len + MT_DMA_HDRS > data_len) ||
 121             WARN_ON_ONCE(dma_len & 0x3))
 122                 return 0;
 123
 124         return MT_DMA_HDRS + dma_len;
 125 }
 126
 127 static void
 128 mt7601u_rx_process_entry(struct mt7601u_dev *dev, struct mt7601u_dma_buf_rx *e)
 129 {
 130         u32 seg_len, data_len = e->urb->actual_length;
 131         u8 *data = page_address(e->p);
 132         struct page *new_p = NULL;
 133         int cnt = 0;
 134
 135         if (!test_bit(MT7601U_STATE_INITIALIZED, &dev->state))
 136                 return;
 137
 138         /* Copy if there is very little data in the buffer. */
 139         if (data_len > 512)
 140                 new_p = dev_alloc_pages(MT_RX_ORDER);
 141
 142         while ((seg_len = mt7601u_rx_next_seg_len(data, data_len))) {
 143                 mt7601u_rx_process_seg(dev, data, seg_len, new_p ? e->p : NULL);
 144
 145                 data_len -= seg_len;
 146                 data += seg_len;
 147                 cnt++;
 148         }
 149
 150         if (cnt > 1)
 151                 trace_mt_rx_dma_aggr(dev, cnt, !!new_p);
 152
 153         if (new_p) {
 154                 /* we have one extra ref from the allocator */
 155                 __free_pages(e->p, MT_RX_ORDER);
 156
 157                 e->p = new_p;
 158         }
 159 }
 160
 161 static struct mt7601u_dma_buf_rx *
 162 mt7601u_rx_get_pending_entry(struct mt7601u_dev *dev)
 163 {
 164         struct mt7601u_rx_queue *q = &dev->rx_q;
 165         struct mt7601u_dma_buf_rx *buf = NULL;
 166         unsigned long flags;
 167
 168         spin_lock_irqsave(&dev->rx_lock, flags);
 169
 170         if (!q->pending)
 171                 goto out;
 172
 173         buf = &q->e[q->start];
 174         q->pending--;
 175         q->start = (q->start + 1) % q->entries;
 176 out:
 177         spin_unlock_irqrestore(&dev->rx_lock, flags);
 178
 179         return buf;
 180 }
 181
 182 static void mt7601u_complete_rx(struct urb *urb)
 183 {
 184         struct mt7601u_dev *dev = urb->context;
 185         struct mt7601u_rx_queue *q = &dev->rx_q;
 186         unsigned long flags;
 187
 188         /* do no schedule rx tasklet if urb has been unlinked
 189          * or the device has been removed
 190          */
 191         switch (urb->status) {
 192         case -ECONNRESET:
 193         case -ESHUTDOWN:
 194         case -ENOENT:
 195                 return;
 196         default:
 197                 dev_err_ratelimited(dev->dev, "rx urb failed: %d\n",
 198                                     urb->status);
 199                 fallthrough;
 200         case 0:
 201                 break;
 202         }
 203
 204         spin_lock_irqsave(&dev->rx_lock, flags);
 205         if (WARN_ONCE(q->e[q->end].urb != urb, "RX urb mismatch"))
 206                 goto out;
 207
 208         q->end = (q->end + 1) % q->entries;
 209         q->pending++;
 210         tasklet_schedule(&dev->rx_tasklet);
 211 out:
 212         spin_unlock_irqrestore(&dev->rx_lock, flags);
 213 }
 214
 215 static void mt7601u_rx_tasklet(struct tasklet_struct *t)
 216 {
 217         struct mt7601u_dev *dev = from_tasklet(dev, t, rx_tasklet);
 218         struct mt7601u_dma_buf_rx *e;
 219
 220         while ((e = mt7601u_rx_get_pending_entry(dev))) {
 221                 if (e->urb->status)
 222                         continue;
 223
 224                 mt7601u_rx_process_entry(dev, e);
 225                 mt7601u_submit_rx_buf(dev, e, GFP_ATOMIC);
 226         }
 227 }
 228
 229 static void mt7601u_complete_tx(struct urb *urb)
 230 {
 231         struct mt7601u_tx_queue *q = urb->context;
 232         struct mt7601u_dev *dev = q->dev;
 233         struct sk_buff *skb;
 234         unsigned long flags;
 235
 236         switch (urb->status) {
 237         case -ECONNRESET:
 238         case -ESHUTDOWN:
 239         case -ENOENT:
 240                 return;
 241         default:
 242                 dev_err_ratelimited(dev->dev, "tx urb failed: %d\n",
 243                                     urb->status);
 244                 fallthrough;
 245         case 0:
 246                 break;
 247         }
 248
 249         spin_lock_irqsave(&dev->tx_lock, flags);
 250         if (WARN_ONCE(q->e[q->start].urb != urb, "TX urb mismatch"))
 251                 goto out;
 252
 253         skb = q->e[q->start].skb;
 254         q->e[q->start].skb = NULL;
 255         trace_mt_tx_dma_done(dev, skb);
 256
 257         __skb_queue_tail(&dev->tx_skb_done, skb);
 258         tasklet_schedule(&dev->tx_tasklet);
 259
 260         if (q->used == q->entries - q->entries / 8)
 261                 ieee80211_wake_queue(dev->hw, skb_get_queue_mapping(skb));
 262
 263         q->start = (q->start + 1) % q->entries;
 264         q->used--;
 265 out:
 266         spin_unlock_irqrestore(&dev->tx_lock, flags);
 267 }
 268
 269 static void mt7601u_tx_tasklet(struct tasklet_struct *t)
 270 {
 271         struct mt7601u_dev *dev = from_tasklet(dev, t, tx_tasklet);
 272         struct sk_buff_head skbs;
 273         unsigned long flags;
 274
 275         __skb_queue_head_init(&skbs);
 276
 277         spin_lock_irqsave(&dev->tx_lock, flags);
 278
 279         set_bit(MT7601U_STATE_MORE_STATS, &dev->state);
 280         if (!test_and_set_bit(MT7601U_STATE_READING_STATS, &dev->state))
 281                 queue_delayed_work(dev->stat_wq, &dev->stat_work,
 282                                    msecs_to_jiffies(10));
 283
 284         skb_queue_splice_init(&dev->tx_skb_done, &skbs);
 285
 286         spin_unlock_irqrestore(&dev->tx_lock, flags);
 287
 288         while (!skb_queue_empty(&skbs)) {
 289                 struct sk_buff *skb = __skb_dequeue(&skbs);
 290
 291                 mt7601u_tx_status(dev, skb);
 292         }
 293 }
 294
 295 static int mt7601u_dma_submit_tx(struct mt7601u_dev *dev,
 296                                  struct sk_buff *skb, u8 ep)
 297 {
 298         struct usb_device *usb_dev = mt7601u_to_usb_dev(dev);
 299         unsigned snd_pipe = usb_sndbulkpipe(usb_dev, dev->out_eps[ep]);
 300         struct mt7601u_dma_buf_tx *e;
 301         struct mt7601u_tx_queue *q = &dev->tx_q[ep];
 302         unsigned long flags;
 303         int ret;
 304
 305         spin_lock_irqsave(&dev->tx_lock, flags);
 306
 307         if (WARN_ON(q->entries <= q->used)) {
 308                 ret = -ENOSPC;
 309                 goto out;
 310         }
 311
 312         e = &q->e[q->end];
 313         e->skb = skb;
 314         usb_fill_bulk_urb(e->urb, usb_dev, snd_pipe, skb->data, skb->len,
 315                           mt7601u_complete_tx, q);
 316         ret = usb_submit_urb(e->urb, GFP_ATOMIC);
 317         if (ret) {
 318                 /* Special-handle ENODEV from TX urb submission because it will
 319                  * often be the first ENODEV we see after device is removed.
 320                  */
 321                 if (ret == -ENODEV)
 322                         set_bit(MT7601U_STATE_REMOVED, &dev->state);
 323                 else
 324                         dev_err(dev->dev, "Error: TX urb submit failed:%d\n",
 325                                 ret);
 326                 goto out;
 327         }
 328
 329         q->end = (q->end + 1) % q->entries;
 330         q->used++;
 331
 332         if (q->used >= q->entries)
 333                 ieee80211_stop_queue(dev->hw, skb_get_queue_mapping(skb));
 334 out:
 335         spin_unlock_irqrestore(&dev->tx_lock, flags);
 336
 337         return ret;
 338 }
 339
 340 /* Map hardware Q to USB endpoint number */
 341 static u8 q2ep(u8 qid)
 342 {
 343         /* TODO: take management packets to queue 5 */
 344         return qid + 1;
 345 }
 346
 347 /* Map USB endpoint number to Q id in the DMA engine */
 348 static enum mt76_qsel ep2dmaq(u8 ep)
 349 {
 350         if (ep == 5)
 351                 return MT_QSEL_MGMT;
 352         return MT_QSEL_EDCA;
 353 }
 354
 355 int mt7601u_dma_enqueue_tx(struct mt7601u_dev *dev, struct sk_buff *skb,
 356                            struct mt76_wcid *wcid, int hw_q)
 357 {
 358         u8 ep = q2ep(hw_q);
 359         u32 dma_flags;
 360         int ret;
 361
 362         dma_flags = MT_TXD_PKT_INFO_80211;
 363         if (wcid->hw_key_idx == 0xff)
 364                 dma_flags |= MT_TXD_PKT_INFO_WIV;
 365
 366         ret = mt7601u_dma_skb_wrap_pkt(skb, ep2dmaq(ep), dma_flags);
 367         if (ret)
 368                 return ret;
 369
 370         ret = mt7601u_dma_submit_tx(dev, skb, ep);
 371         if (ret) {
 372                 ieee80211_free_txskb(dev->hw, skb);
 373                 return ret;
 374         }
 375
 376         return 0;
 377 }
 378
 379 static void mt7601u_kill_rx(struct mt7601u_dev *dev)
 380 {
 381         int i;
 382
 383         for (i = 0; i < dev->rx_q.entries; i++)
 384                 usb_poison_urb(dev->rx_q.e[i].urb);
 385 }
 386
 387 static int mt7601u_submit_rx_buf(struct mt7601u_dev *dev,
 388                                  struct mt7601u_dma_buf_rx *e, gfp_t gfp)
 389 {
 390         struct usb_device *usb_dev = mt7601u_to_usb_dev(dev);
 391         u8 *buf = page_address(e->p);
 392         unsigned pipe;
 393         int ret;
 394
 395         pipe = usb_rcvbulkpipe(usb_dev, dev->in_eps[MT_EP_IN_PKT_RX]);
 396
 397         usb_fill_bulk_urb(e->urb, usb_dev, pipe, buf, MT_RX_URB_SIZE,
 398                           mt7601u_complete_rx, dev);
 399
 400         trace_mt_submit_urb(dev, e->urb);
 401         ret = usb_submit_urb(e->urb, gfp);
 402         if (ret)
 403                 dev_err(dev->dev, "Error: submit RX URB failed:%d\n", ret);
 404
 405         return ret;
 406 }
 407
 408 static int mt7601u_submit_rx(struct mt7601u_dev *dev)
 409 {
 410         int i, ret;
 411
 412         for (i = 0; i < dev->rx_q.entries; i++) {
 413                 ret = mt7601u_submit_rx_buf(dev, &dev->rx_q.e[i], GFP_KERNEL);
 414                 if (ret)
 415                         return ret;
 416         }
 417
 418         return 0;
 419 }
 420
 421 static void mt7601u_free_rx(struct mt7601u_dev *dev)
 422 {
 423         int i;
 424
 425         for (i = 0; i < dev->rx_q.entries; i++) {
 426                 __free_pages(dev->rx_q.e[i].p, MT_RX_ORDER);
 427                 usb_free_urb(dev->rx_q.e[i].urb);
 428         }
 429 }
 430
 431 static int mt7601u_alloc_rx(struct mt7601u_dev *dev)
 432 {
 433         int i;
 434
 435         memset(&dev->rx_q, 0, sizeof(dev->rx_q));
 436         dev->rx_q.dev = dev;
 437         dev->rx_q.entries = N_RX_ENTRIES;
 438
 439         for (i = 0; i < N_RX_ENTRIES; i++) {
 440                 dev->rx_q.e[i].urb = usb_alloc_urb(0, GFP_KERNEL);
 441                 dev->rx_q.e[i].p = dev_alloc_pages(MT_RX_ORDER);
 442
 443                 if (!dev->rx_q.e[i].urb || !dev->rx_q.e[i].p)
 444                         return -ENOMEM;
 445         }
 446
 447         return 0;
 448 }
 449
 450 static void mt7601u_free_tx_queue(struct mt7601u_tx_queue *q)
 451 {
 452         int i;
 453
 454         for (i = 0; i < q->entries; i++)  {
 455                 usb_poison_urb(q->e[i].urb);
 456                 if (q->e[i].skb)
 457                         mt7601u_tx_status(q->dev, q->e[i].skb);
 458                 usb_free_urb(q->e[i].urb);
 459         }
 460 }
 461
 462 static void mt7601u_free_tx(struct mt7601u_dev *dev)
 463 {
 464         int i;
 465
 466         if (!dev->tx_q)
 467                 return;
 468
 469         for (i = 0; i < __MT_EP_OUT_MAX; i++)
 470                 mt7601u_free_tx_queue(&dev->tx_q[i]);
 471 }
 472
 473 static int mt7601u_alloc_tx_queue(struct mt7601u_dev *dev,
 474                                   struct mt7601u_tx_queue *q)
 475 {
 476         int i;
 477
 478         q->dev = dev;
 479         q->entries = N_TX_ENTRIES;
 480
 481         for (i = 0; i < N_TX_ENTRIES; i++) {
 482                 q->e[i].urb = usb_alloc_urb(0, GFP_KERNEL);
 483                 if (!q->e[i].urb)
 484                         return -ENOMEM;
 485         }
 486
 487         return 0;
 488 }
 489
 490 static int mt7601u_alloc_tx(struct mt7601u_dev *dev)
 491 {
 492         int i;
 493
 494         dev->tx_q = devm_kcalloc(dev->dev, __MT_EP_OUT_MAX,
 495                                  sizeof(*dev->tx_q), GFP_KERNEL);
 496         if (!dev->tx_q)
 497                 return -ENOMEM;
 498
 499         for (i = 0; i < __MT_EP_OUT_MAX; i++)
 500                 if (mt7601u_alloc_tx_queue(dev, &dev->tx_q[i]))
 501                         return -ENOMEM;
 502
 503         return 0;
 504 }
 505
 506 int mt7601u_dma_init(struct mt7601u_dev *dev)
 507 {
 508         int ret = -ENOMEM;
 509
 510         tasklet_setup(&dev->tx_tasklet, mt7601u_tx_tasklet);
 511         tasklet_setup(&dev->rx_tasklet, mt7601u_rx_tasklet);
 512
 513         ret = mt7601u_alloc_tx(dev);
 514         if (ret)
 515                 goto err;
 516         ret = mt7601u_alloc_rx(dev);
 517         if (ret)
 518                 goto err;
 519
 520         ret = mt7601u_submit_rx(dev);
 521         if (ret)
 522                 goto err;
 523
 524         return 0;
 525 err:
 526         mt7601u_dma_cleanup(dev);
 527         return ret;
 528 }
 529
 530 void mt7601u_dma_cleanup(struct mt7601u_dev *dev)
 531 {
 532         mt7601u_kill_rx(dev);
 533
 534         tasklet_kill(&dev->rx_tasklet);
 535
 536         mt7601u_free_rx(dev);
 537         mt7601u_free_tx(dev);
 538
 539         tasklet_kill(&dev->tx_tasklet);
 540 }