drivers/net/wireless/mediatek/mt76/dma.c

   1 /*
   2  * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
   3  *
   4  * Permission to use, copy, modify, and/or distribute this software for any
   5  * purpose with or without fee is hereby granted, provided that the above
   6  * copyright notice and this permission notice appear in all copies.
   7  *
   8  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  11  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  13  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  14  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15  */
  16
  17 #include <linux/dma-mapping.h>
  18 #include "mt76.h"
  19 #include "dma.h"
  20
  21 #define DMA_DUMMY_TXWI  ((void *) ~0)
  22
  23 static int
  24 mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q)
  25 {
  26         int size;
  27         int i;
  28
  29         spin_lock_init(&q->lock);
  30         INIT_LIST_HEAD(&q->swq);
  31
  32         size = q->ndesc * sizeof(struct mt76_desc);
  33         q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
  34         if (!q->desc)
  35                 return -ENOMEM;
  36
  37         size = q->ndesc * sizeof(*q->entry);
  38         q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
  39         if (!q->entry)
  40                 return -ENOMEM;
  41
  42         /* clear descriptors */
  43         for (i = 0; i < q->ndesc; i++)
  44                 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  45
  46         iowrite32(q->desc_dma, &q->regs->desc_base);
  47         iowrite32(0, &q->regs->cpu_idx);
  48         iowrite32(0, &q->regs->dma_idx);
  49         iowrite32(q->ndesc, &q->regs->ring_size);
  50
  51         return 0;
  52 }
  53
  54 static int
  55 mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
  56                  struct mt76_queue_buf *buf, int nbufs, u32 info,
  57                  struct sk_buff *skb, void *txwi)
  58 {
  59         struct mt76_desc *desc;
  60         u32 ctrl;
  61         int i, idx = -1;
  62
  63         if (txwi)
  64                 q->entry[q->head].txwi = DMA_DUMMY_TXWI;
  65
  66         for (i = 0; i < nbufs; i += 2, buf += 2) {
  67                 u32 buf0 = buf[0].addr, buf1 = 0;
  68
  69                 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
  70                 if (i < nbufs - 1) {
  71                         buf1 = buf[1].addr;
  72                         ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
  73                 }
  74
  75                 if (i == nbufs - 1)
  76                         ctrl |= MT_DMA_CTL_LAST_SEC0;
  77                 else if (i == nbufs - 2)
  78                         ctrl |= MT_DMA_CTL_LAST_SEC1;
  79
  80                 idx = q->head;
  81                 q->head = (q->head + 1) % q->ndesc;
  82
  83                 desc = &q->desc[idx];
  84
  85                 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
  86                 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
  87                 WRITE_ONCE(desc->info, cpu_to_le32(info));
  88                 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
  89
  90                 q->queued++;
  91         }
  92
  93         q->entry[idx].txwi = txwi;
  94         q->entry[idx].skb = skb;
  95
  96         return idx;
  97 }
  98
  99 static void
 100 mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 101                         struct mt76_queue_entry *prev_e)
 102 {
 103         struct mt76_queue_entry *e = &q->entry[idx];
 104         __le32 __ctrl = READ_ONCE(q->desc[idx].ctrl);
 105         u32 ctrl = le32_to_cpu(__ctrl);
 106
 107         if (!e->txwi || !e->skb) {
 108                 __le32 addr = READ_ONCE(q->desc[idx].buf0);
 109                 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
 110
 111                 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
 112                                  DMA_TO_DEVICE);
 113         }
 114
 115         if (!(ctrl & MT_DMA_CTL_LAST_SEC0)) {
 116                 __le32 addr = READ_ONCE(q->desc[idx].buf1);
 117                 u32 len = FIELD_GET(MT_DMA_CTL_SD_LEN1, ctrl);
 118
 119                 dma_unmap_single(dev->dev, le32_to_cpu(addr), len,
 120                                  DMA_TO_DEVICE);
 121         }
 122
 123         if (e->txwi == DMA_DUMMY_TXWI)
 124                 e->txwi = NULL;
 125
 126         *prev_e = *e;
 127         memset(e, 0, sizeof(*e));
 128 }
 129
 130 static void
 131 mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
 132 {
 133         q->head = ioread32(&q->regs->dma_idx);
 134         q->tail = q->head;
 135         iowrite32(q->head, &q->regs->cpu_idx);
 136 }
 137
 138 static void
 139 mt76_dma_tx_cleanup(struct mt76_dev *dev, enum mt76_txq_id qid, bool flush)
 140 {
 141         struct mt76_queue *q = &dev->q_tx[qid];
 142         struct mt76_queue_entry entry;
 143         bool wake = false;
 144         int last;
 145
 146         if (!q->ndesc)
 147                 return;
 148
 149         spin_lock_bh(&q->lock);
 150         if (flush)
 151                 last = -1;
 152         else
 153                 last = ioread32(&q->regs->dma_idx);
 154
 155         while (q->queued && q->tail != last) {
 156                 mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
 157                 if (entry.schedule)
 158                         q->swq_queued--;
 159
 160                 if (entry.skb)
 161                         dev->drv->tx_complete_skb(dev, q, &entry, flush);
 162
 163                 if (entry.txwi) {
 164                         mt76_put_txwi(dev, entry.txwi);
 165                         wake = true;
 166                 }
 167
 168                 q->tail = (q->tail + 1) % q->ndesc;
 169                 q->queued--;
 170
 171                 if (!flush && q->tail == last)
 172                         last = ioread32(&q->regs->dma_idx);
 173         }
 174
 175         if (!flush)
 176                 mt76_txq_schedule(dev, q);
 177         else
 178                 mt76_dma_sync_idx(dev, q);
 179
 180         wake = wake && qid < IEEE80211_NUM_ACS && q->queued < q->ndesc - 8;
 181
 182         if (!q->queued)
 183                 wake_up(&dev->tx_wait);
 184
 185         spin_unlock_bh(&q->lock);
 186
 187         if (wake)
 188                 ieee80211_wake_queue(dev->hw, qid);
 189 }
 190
 191 static void *
 192 mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
 193                  int *len, u32 *info, bool *more)
 194 {
 195         struct mt76_queue_entry *e = &q->entry[idx];
 196         struct mt76_desc *desc = &q->desc[idx];
 197         dma_addr_t buf_addr;
 198         void *buf = e->buf;
 199         int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
 200
 201         buf_addr = le32_to_cpu(READ_ONCE(desc->buf0));
 202         if (len) {
 203                 u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
 204                 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
 205                 *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
 206         }
 207
 208         if (info)
 209                 *info = le32_to_cpu(desc->info);
 210
 211         dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
 212         e->buf = NULL;
 213
 214         return buf;
 215 }
 216
 217 static void *
 218 mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
 219                  int *len, u32 *info, bool *more)
 220 {
 221         int idx = q->tail;
 222
 223         *more = false;
 224         if (!q->queued)
 225                 return NULL;
 226
 227         if (!flush && !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
 228                 return NULL;
 229
 230         q->tail = (q->tail + 1) % q->ndesc;
 231         q->queued--;
 232
 233         return mt76_dma_get_buf(dev, q, idx, len, info, more);
 234 }
 235
 236 static void
 237 mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
 238 {
 239         iowrite32(q->head, &q->regs->cpu_idx);
 240 }
 241
 242 static int
 243 mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q, bool napi)
 244 {
 245         dma_addr_t addr;
 246         void *buf;
 247         int frames = 0;
 248         int len = SKB_WITH_OVERHEAD(q->buf_size);
 249         int offset = q->buf_offset;
 250         int idx;
 251         void *(*alloc)(unsigned int fragsz);
 252
 253         if (napi)
 254                 alloc = napi_alloc_frag;
 255         else
 256                 alloc = netdev_alloc_frag;
 257
 258         spin_lock_bh(&q->lock);
 259
 260         while (q->queued < q->ndesc - 1) {
 261                 struct mt76_queue_buf qbuf;
 262
 263                 buf = alloc(q->buf_size);
 264                 if (!buf)
 265                         break;
 266
 267                 addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
 268                 if (dma_mapping_error(dev->dev, addr)) {
 269                         skb_free_frag(buf);
 270                         break;
 271                 }
 272
 273                 qbuf.addr = addr + offset;
 274                 qbuf.len = len - offset;
 275                 idx = mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
 276                 frames++;
 277         }
 278
 279         if (frames)
 280                 mt76_dma_kick_queue(dev, q);
 281
 282         spin_unlock_bh(&q->lock);
 283
 284         return frames;
 285 }
 286
 287 static void
 288 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 289 {
 290         void *buf;
 291         bool more;
 292
 293         spin_lock_bh(&q->lock);
 294         do {
 295                 buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
 296                 if (!buf)
 297                         break;
 298
 299                 skb_free_frag(buf);
 300         } while (1);
 301         spin_unlock_bh(&q->lock);
 302 }
 303
 304 static void
 305 mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
 306 {
 307         struct mt76_queue *q = &dev->q_rx[qid];
 308         int i;
 309
 310         for (i = 0; i < q->ndesc; i++)
 311                 q->desc[i].ctrl &= ~cpu_to_le32(MT_DMA_CTL_DMA_DONE);
 312
 313         mt76_dma_rx_cleanup(dev, q);
 314         mt76_dma_sync_idx(dev, q);
 315         mt76_dma_rx_fill(dev, q, false);
 316 }
 317
 318 static void
 319 mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
 320                   int len, bool more)
 321 {
 322         struct page *page = virt_to_head_page(data);
 323         int offset = data - page_address(page);
 324         struct sk_buff *skb = q->rx_head;
 325
 326         offset += q->buf_offset;
 327         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, len,
 328                         q->buf_size);
 329
 330         if (more)
 331                 return;
 332
 333         q->rx_head = NULL;
 334         dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 335 }
 336
 337 static int
 338 mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 339 {
 340         struct sk_buff *skb;
 341         unsigned char *data;
 342         int len;
 343         int done = 0;
 344         bool more;
 345
 346         while (done < budget) {
 347                 u32 info;
 348
 349                 data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
 350                 if (!data)
 351                         break;
 352
 353                 if (q->rx_head) {
 354                         mt76_add_fragment(dev, q, data, len, more);
 355                         continue;
 356                 }
 357
 358                 skb = build_skb(data, q->buf_size);
 359                 if (!skb) {
 360                         skb_free_frag(data);
 361                         continue;
 362                 }
 363
 364                 skb_reserve(skb, q->buf_offset);
 365                 if (skb->tail + len > skb->end) {
 366                         dev_kfree_skb(skb);
 367                         continue;
 368                 }
 369
 370                 if (q == &dev->q_rx[MT_RXQ_MCU]) {
 371                         u32 *rxfce = (u32 *) skb->cb;
 372                         *rxfce = info;
 373                 }
 374
 375                 __skb_put(skb, len);
 376                 done++;
 377
 378                 if (more) {
 379                         q->rx_head = skb;
 380                         continue;
 381                 }
 382
 383                 dev->drv->rx_skb(dev, q - dev->q_rx, skb);
 384         }
 385
 386         mt76_dma_rx_fill(dev, q, true);
 387         return done;
 388 }
 389
 390 static int
 391 mt76_dma_rx_poll(struct napi_struct *napi, int budget)
 392 {
 393         struct mt76_dev *dev;
 394         int qid, done = 0, cur;
 395
 396         dev = container_of(napi->dev, struct mt76_dev, napi_dev);
 397         qid = napi - dev->napi;
 398
 399         rcu_read_lock();
 400
 401         do {
 402                 cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
 403                 mt76_rx_poll_complete(dev, qid);
 404                 done += cur;
 405         } while (cur && done < budget);
 406
 407         rcu_read_unlock();
 408
 409         if (done < budget) {
 410                 napi_complete(napi);
 411                 dev->drv->rx_poll_complete(dev, qid);
 412         }
 413
 414         return done;
 415 }
 416
 417 static int
 418 mt76_dma_init(struct mt76_dev *dev)
 419 {
 420         int i;
 421
 422         init_dummy_netdev(&dev->napi_dev);
 423
 424         for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
 425                 netif_napi_add(&dev->napi_dev, &dev->napi[i], mt76_dma_rx_poll,
 426                                64);
 427                 mt76_dma_rx_fill(dev, &dev->q_rx[i], false);
 428                 skb_queue_head_init(&dev->rx_skb[i]);
 429                 napi_enable(&dev->napi[i]);
 430         }
 431
 432         return 0;
 433 }
 434
 435 static const struct mt76_queue_ops mt76_dma_ops = {
 436         .init = mt76_dma_init,
 437         .alloc = mt76_dma_alloc_queue,
 438         .add_buf = mt76_dma_add_buf,
 439         .tx_cleanup = mt76_dma_tx_cleanup,
 440         .rx_reset = mt76_dma_rx_reset,
 441         .kick = mt76_dma_kick_queue,
 442 };
 443
 444 int mt76_dma_attach(struct mt76_dev *dev)
 445 {
 446         dev->queue_ops = &mt76_dma_ops;
 447         return 0;
 448 }
 449 EXPORT_SYMBOL_GPL(mt76_dma_attach);
 450
 451 void mt76_dma_cleanup(struct mt76_dev *dev)
 452 {
 453         int i;
 454
 455         for (i = 0; i < ARRAY_SIZE(dev->q_tx); i++)
 456                 mt76_dma_tx_cleanup(dev, i, true);
 457
 458         for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) {
 459                 netif_napi_del(&dev->napi[i]);
 460                 mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
 461         }
 462 }
 463 EXPORT_SYMBOL_GPL(mt76_dma_cleanup);