drivers/usb/mtu3/mtu3_qmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * mtu3_qmu.c - Queue Management Unit driver for device controller
   4  *
   5  * Copyright (C) 2016 MediaTek Inc.
   6  *
   7  * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
   8  */
   9
  10 /*
  11  * Queue Management Unit (QMU) is designed to unload SW effort
  12  * to serve DMA interrupts.
  13  * By preparing General Purpose Descriptor (GPD) and Buffer Descriptor (BD),
  14  * SW links data buffers and triggers QMU to send / receive data to
  15  * host / from device at a time.
  16  * And now only GPD is supported.
  17  *
  18  * For more detailed information, please refer to QMU Programming Guide
  19  */
  20
  21 #include <linux/dmapool.h>
  22 #include <linux/iopoll.h>
  23
  24 #include "mtu3.h"
  25
  26 #define QMU_CHECKSUM_LEN        16
  27
  28 #define GPD_FLAGS_HWO   BIT(0)
  29 #define GPD_FLAGS_BDP   BIT(1)
  30 #define GPD_FLAGS_BPS   BIT(2)
  31 #define GPD_FLAGS_IOC   BIT(7)
  32
  33 #define GPD_EXT_FLAG_ZLP        BIT(5)
  34 #define GPD_EXT_NGP(x)          (((x) & 0xf) << 4)
  35 #define GPD_EXT_BUF(x)          (((x) & 0xf) << 0)
  36
  37 #define HILO_GEN64(hi, lo) (((u64)(hi) << 32) + (lo))
  38 #define HILO_DMA(hi, lo)        \
  39         ((dma_addr_t)HILO_GEN64((le32_to_cpu(hi)), (le32_to_cpu(lo))))
  40
  41 static dma_addr_t read_txq_cur_addr(void __iomem *mbase, u8 epnum)
  42 {
  43         u32 txcpr;
  44         u32 txhiar;
  45
  46         txcpr = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));
  47         txhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));
  48
  49         return HILO_DMA(QMU_CUR_GPD_ADDR_HI(txhiar), txcpr);
  50 }
  51
  52 static dma_addr_t read_rxq_cur_addr(void __iomem *mbase, u8 epnum)
  53 {
  54         u32 rxcpr;
  55         u32 rxhiar;
  56
  57         rxcpr = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));
  58         rxhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));
  59
  60         return HILO_DMA(QMU_CUR_GPD_ADDR_HI(rxhiar), rxcpr);
  61 }
  62
  63 static void write_txq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)
  64 {
  65         u32 tqhiar;
  66
  67         mtu3_writel(mbase, USB_QMU_TQSAR(epnum),
  68                     cpu_to_le32(lower_32_bits(dma)));
  69         tqhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));
  70         tqhiar &= ~QMU_START_ADDR_HI_MSK;
  71         tqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma));
  72         mtu3_writel(mbase, USB_QMU_TQHIAR(epnum), tqhiar);
  73 }
  74
  75 static void write_rxq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)
  76 {
  77         u32 rqhiar;
  78
  79         mtu3_writel(mbase, USB_QMU_RQSAR(epnum),
  80                     cpu_to_le32(lower_32_bits(dma)));
  81         rqhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));
  82         rqhiar &= ~QMU_START_ADDR_HI_MSK;
  83         rqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma));
  84         mtu3_writel(mbase, USB_QMU_RQHIAR(epnum), rqhiar);
  85 }
  86
  87 static struct qmu_gpd *gpd_dma_to_virt(struct mtu3_gpd_ring *ring,
  88                 dma_addr_t dma_addr)
  89 {
  90         dma_addr_t dma_base = ring->dma;
  91         struct qmu_gpd *gpd_head = ring->start;
  92         u32 offset = (dma_addr - dma_base) / sizeof(*gpd_head);
  93
  94         if (offset >= MAX_GPD_NUM)
  95                 return NULL;
  96
  97         return gpd_head + offset;
  98 }
  99
 100 static dma_addr_t gpd_virt_to_dma(struct mtu3_gpd_ring *ring,
 101                 struct qmu_gpd *gpd)
 102 {
 103         dma_addr_t dma_base = ring->dma;
 104         struct qmu_gpd *gpd_head = ring->start;
 105         u32 offset;
 106
 107         offset = gpd - gpd_head;
 108         if (offset >= MAX_GPD_NUM)
 109                 return 0;
 110
 111         return dma_base + (offset * sizeof(*gpd));
 112 }
 113
 114 static void gpd_ring_init(struct mtu3_gpd_ring *ring, struct qmu_gpd *gpd)
 115 {
 116         ring->start = gpd;
 117         ring->enqueue = gpd;
 118         ring->dequeue = gpd;
 119         ring->end = gpd + MAX_GPD_NUM - 1;
 120 }
 121
 122 static void reset_gpd_list(struct mtu3_ep *mep)
 123 {
 124         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 125         struct qmu_gpd *gpd = ring->start;
 126
 127         if (gpd) {
 128                 gpd->flag &= ~GPD_FLAGS_HWO;
 129                 gpd_ring_init(ring, gpd);
 130         }
 131 }
 132
 133 int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)
 134 {
 135         struct qmu_gpd *gpd;
 136         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 137
 138         /* software own all gpds as default */
 139         gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);
 140         if (gpd == NULL)
 141                 return -ENOMEM;
 142
 143         gpd_ring_init(ring, gpd);
 144
 145         return 0;
 146 }
 147
 148 void mtu3_gpd_ring_free(struct mtu3_ep *mep)
 149 {
 150         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 151
 152         dma_pool_free(mep->mtu->qmu_gpd_pool,
 153                         ring->start, ring->dma);
 154         memset(ring, 0, sizeof(*ring));
 155 }
 156
 157 /*
 158  * calculate check sum of a gpd or bd
 159  * add "noinline" and "mb" to prevent wrong calculation
 160  */
 161 static noinline u8 qmu_calc_checksum(u8 *data)
 162 {
 163         u8 chksum = 0;
 164         int i;
 165
 166         data[1] = 0x0;  /* set checksum to 0 */
 167
 168         mb();   /* ensure the gpd/bd is really up-to-date */
 169         for (i = 0; i < QMU_CHECKSUM_LEN; i++)
 170                 chksum += data[i];
 171
 172         /* Default: HWO=1, @flag[bit0] */
 173         chksum += 1;
 174
 175         return 0xFF - chksum;
 176 }
 177
 178 void mtu3_qmu_resume(struct mtu3_ep *mep)
 179 {
 180         struct mtu3 *mtu = mep->mtu;
 181         void __iomem *mbase = mtu->mac_base;
 182         int epnum = mep->epnum;
 183         u32 offset;
 184
 185         offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
 186
 187         mtu3_writel(mbase, offset, QMU_Q_RESUME);
 188         if (!(mtu3_readl(mbase, offset) & QMU_Q_ACTIVE))
 189                 mtu3_writel(mbase, offset, QMU_Q_RESUME);
 190 }
 191
 192 static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring)
 193 {
 194         if (ring->enqueue < ring->end)
 195                 ring->enqueue++;
 196         else
 197                 ring->enqueue = ring->start;
 198
 199         return ring->enqueue;
 200 }
 201
 202 static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
 203 {
 204         if (ring->dequeue < ring->end)
 205                 ring->dequeue++;
 206         else
 207                 ring->dequeue = ring->start;
 208
 209         return ring->dequeue;
 210 }
 211
 212 /* check if a ring is emtpy */
 213 static int gpd_ring_empty(struct mtu3_gpd_ring *ring)
 214 {
 215         struct qmu_gpd *enq = ring->enqueue;
 216         struct qmu_gpd *next;
 217
 218         if (ring->enqueue < ring->end)
 219                 next = enq + 1;
 220         else
 221                 next = ring->start;
 222
 223         /* one gpd is reserved to simplify gpd preparation */
 224         return next == ring->dequeue;
 225 }
 226
 227 int mtu3_prepare_transfer(struct mtu3_ep *mep)
 228 {
 229         return gpd_ring_empty(&mep->gpd_ring);
 230 }
 231
 232 static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 233 {
 234         struct qmu_gpd *enq;
 235         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 236         struct qmu_gpd *gpd = ring->enqueue;
 237         struct usb_request *req = &mreq->request;
 238         dma_addr_t enq_dma;
 239         u16 ext_addr;
 240
 241         /* set all fields to zero as default value */
 242         memset(gpd, 0, sizeof(*gpd));
 243
 244         gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
 245         ext_addr = GPD_EXT_BUF(upper_32_bits(req->dma));
 246         gpd->buf_len = cpu_to_le16(req->length);
 247         gpd->flag |= GPD_FLAGS_IOC;
 248
 249         /* get the next GPD */
 250         enq = advance_enq_gpd(ring);
 251         enq_dma = gpd_virt_to_dma(ring, enq);
 252         dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
 253                 mep->epnum, gpd, enq, &enq_dma);
 254
 255         enq->flag &= ~GPD_FLAGS_HWO;
 256         gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
 257         ext_addr |= GPD_EXT_NGP(upper_32_bits(enq_dma));
 258         gpd->tx_ext_addr = cpu_to_le16(ext_addr);
 259
 260         if (req->zero)
 261                 gpd->ext_flag |= GPD_EXT_FLAG_ZLP;
 262
 263         gpd->chksum = qmu_calc_checksum((u8 *)gpd);
 264         gpd->flag |= GPD_FLAGS_HWO;
 265
 266         mreq->gpd = gpd;
 267
 268         return 0;
 269 }
 270
 271 static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 272 {
 273         struct qmu_gpd *enq;
 274         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 275         struct qmu_gpd *gpd = ring->enqueue;
 276         struct usb_request *req = &mreq->request;
 277         dma_addr_t enq_dma;
 278         u16 ext_addr;
 279
 280         /* set all fields to zero as default value */
 281         memset(gpd, 0, sizeof(*gpd));
 282
 283         gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
 284         ext_addr = GPD_EXT_BUF(upper_32_bits(req->dma));
 285         gpd->data_buf_len = cpu_to_le16(req->length);
 286         gpd->flag |= GPD_FLAGS_IOC;
 287
 288         /* get the next GPD */
 289         enq = advance_enq_gpd(ring);
 290         enq_dma = gpd_virt_to_dma(ring, enq);
 291         dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
 292                 mep->epnum, gpd, enq, &enq_dma);
 293
 294         enq->flag &= ~GPD_FLAGS_HWO;
 295         gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
 296         ext_addr |= GPD_EXT_NGP(upper_32_bits(enq_dma));
 297         gpd->rx_ext_addr = cpu_to_le16(ext_addr);
 298         gpd->chksum = qmu_calc_checksum((u8 *)gpd);
 299         gpd->flag |= GPD_FLAGS_HWO;
 300
 301         mreq->gpd = gpd;
 302
 303         return 0;
 304 }
 305
 306 void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
 307 {
 308
 309         if (mep->is_in)
 310                 mtu3_prepare_tx_gpd(mep, mreq);
 311         else
 312                 mtu3_prepare_rx_gpd(mep, mreq);
 313 }
 314
 315 int mtu3_qmu_start(struct mtu3_ep *mep)
 316 {
 317         struct mtu3 *mtu = mep->mtu;
 318         void __iomem *mbase = mtu->mac_base;
 319         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 320         u8 epnum = mep->epnum;
 321
 322         if (mep->is_in) {
 323                 /* set QMU start address */
 324                 write_txq_start_addr(mbase, epnum, ring->dma);
 325                 mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN);
 326                 mtu3_setbits(mbase, U3D_QCR0, QMU_TX_CS_EN(epnum));
 327                 /* send zero length packet according to ZLP flag in GPD */
 328                 mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));
 329                 mtu3_writel(mbase, U3D_TQERRIESR0,
 330                                 QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));
 331
 332                 if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {
 333                         dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);
 334                         return 0;
 335                 }
 336                 mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);
 337
 338         } else {
 339                 write_rxq_start_addr(mbase, epnum, ring->dma);
 340                 mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN);
 341                 mtu3_setbits(mbase, U3D_QCR0, QMU_RX_CS_EN(epnum));
 342                 /* don't expect ZLP */
 343                 mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));
 344                 /* move to next GPD when receive ZLP */
 345                 mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));
 346                 mtu3_writel(mbase, U3D_RQERRIESR0,
 347                                 QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));
 348                 mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));
 349
 350                 if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {
 351                         dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);
 352                         return 0;
 353                 }
 354                 mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);
 355         }
 356
 357         return 0;
 358 }
 359
 360 /* may called in atomic context */
 361 void mtu3_qmu_stop(struct mtu3_ep *mep)
 362 {
 363         struct mtu3 *mtu = mep->mtu;
 364         void __iomem *mbase = mtu->mac_base;
 365         int epnum = mep->epnum;
 366         u32 value = 0;
 367         u32 qcsr;
 368         int ret;
 369
 370         qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
 371
 372         if (!(mtu3_readl(mbase, qcsr) & QMU_Q_ACTIVE)) {
 373                 dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);
 374                 return;
 375         }
 376         mtu3_writel(mbase, qcsr, QMU_Q_STOP);
 377
 378         ret = readl_poll_timeout_atomic(mbase + qcsr, value,
 379                         !(value & QMU_Q_ACTIVE), 1, 1000);
 380         if (ret) {
 381                 dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);
 382                 return;
 383         }
 384
 385         dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);
 386 }
 387
 388 void mtu3_qmu_flush(struct mtu3_ep *mep)
 389 {
 390
 391         dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,
 392                 ((mep->is_in) ? "TX" : "RX"));
 393
 394         /*Stop QMU */
 395         mtu3_qmu_stop(mep);
 396         reset_gpd_list(mep);
 397 }
 398
 399 /*
 400  * QMU can't transfer zero length packet directly (a hardware limit
 401  * on old SoCs), so when needs to send ZLP, we intentionally trigger
 402  * a length error interrupt, and in the ISR sends a ZLP by BMU.
 403  */
 404 static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
 405 {
 406         struct mtu3_ep *mep = mtu->in_eps + epnum;
 407         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 408         void __iomem *mbase = mtu->mac_base;
 409         struct qmu_gpd *gpd_current = NULL;
 410         struct usb_request *req = NULL;
 411         struct mtu3_request *mreq;
 412         dma_addr_t cur_gpd_dma;
 413         u32 txcsr = 0;
 414         int ret;
 415
 416         mreq = next_request(mep);
 417         if (mreq && mreq->request.length == 0)
 418                 req = &mreq->request;
 419         else
 420                 return;
 421
 422         cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
 423         gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
 424
 425         if (le16_to_cpu(gpd_current->buf_len) != 0) {
 426                 dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);
 427                 return;
 428         }
 429
 430         dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);
 431
 432         mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
 433
 434         ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),
 435                         txcsr, !(txcsr & TX_FIFOFULL), 1, 1000);
 436         if (ret) {
 437                 dev_err(mtu->dev, "%s wait for fifo empty fail\n", __func__);
 438                 return;
 439         }
 440         mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);
 441
 442         /* by pass the current GDP */
 443         gpd_current->flag |= GPD_FLAGS_BPS;
 444         gpd_current->chksum = qmu_calc_checksum((u8 *)gpd_current);
 445         gpd_current->flag |= GPD_FLAGS_HWO;
 446
 447         /*enable DMAREQEN, switch back to QMU mode */
 448         mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
 449         mtu3_qmu_resume(mep);
 450 }
 451
 452 /*
 453  * NOTE: request list maybe is already empty as following case:
 454  * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
 455  * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
 456  * tasklet process both of them)-->qmu_interrupt for second one.
 457  * To avoid upper case, put qmu_done_tx in ISR directly to process it.
 458  */
 459 static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
 460 {
 461         struct mtu3_ep *mep = mtu->in_eps + epnum;
 462         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 463         void __iomem *mbase = mtu->mac_base;
 464         struct qmu_gpd *gpd = ring->dequeue;
 465         struct qmu_gpd *gpd_current = NULL;
 466         struct usb_request *request = NULL;
 467         struct mtu3_request *mreq;
 468         dma_addr_t cur_gpd_dma;
 469
 470         /*transfer phy address got from QMU register to virtual address */
 471         cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
 472         gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
 473
 474         dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
 475                 __func__, epnum, gpd, gpd_current, ring->enqueue);
 476
 477         while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
 478
 479                 mreq = next_request(mep);
 480
 481                 if (mreq == NULL || mreq->gpd != gpd) {
 482                         dev_err(mtu->dev, "no correct TX req is found\n");
 483                         break;
 484                 }
 485
 486                 request = &mreq->request;
 487                 request->actual = le16_to_cpu(gpd->buf_len);
 488                 mtu3_req_complete(mep, request, 0);
 489
 490                 gpd = advance_deq_gpd(ring);
 491         }
 492
 493         dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
 494                 __func__, epnum, ring->dequeue, ring->enqueue);
 495
 496 }
 497
 498 static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
 499 {
 500         struct mtu3_ep *mep = mtu->out_eps + epnum;
 501         struct mtu3_gpd_ring *ring = &mep->gpd_ring;
 502         void __iomem *mbase = mtu->mac_base;
 503         struct qmu_gpd *gpd = ring->dequeue;
 504         struct qmu_gpd *gpd_current = NULL;
 505         struct usb_request *req = NULL;
 506         struct mtu3_request *mreq;
 507         dma_addr_t cur_gpd_dma;
 508
 509         cur_gpd_dma = read_rxq_cur_addr(mbase, epnum);
 510         gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
 511
 512         dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
 513                 __func__, epnum, gpd, gpd_current, ring->enqueue);
 514
 515         while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) {
 516
 517                 mreq = next_request(mep);
 518
 519                 if (mreq == NULL || mreq->gpd != gpd) {
 520                         dev_err(mtu->dev, "no correct RX req is found\n");
 521                         break;
 522                 }
 523                 req = &mreq->request;
 524
 525                 req->actual = le16_to_cpu(gpd->buf_len);
 526                 mtu3_req_complete(mep, req, 0);
 527
 528                 gpd = advance_deq_gpd(ring);
 529         }
 530
 531         dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
 532                 __func__, epnum, ring->dequeue, ring->enqueue);
 533 }
 534
 535 static void qmu_done_isr(struct mtu3 *mtu, u32 done_status)
 536 {
 537         int i;
 538
 539         for (i = 1; i < mtu->num_eps; i++) {
 540                 if (done_status & QMU_RX_DONE_INT(i))
 541                         qmu_done_rx(mtu, i);
 542                 if (done_status & QMU_TX_DONE_INT(i))
 543                         qmu_done_tx(mtu, i);
 544         }
 545 }
 546
 547 static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)
 548 {
 549         void __iomem *mbase = mtu->mac_base;
 550         u32 errval;
 551         int i;
 552
 553         if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) {
 554                 errval = mtu3_readl(mbase, U3D_RQERRIR0);
 555                 for (i = 1; i < mtu->num_eps; i++) {
 556                         if (errval & QMU_RX_CS_ERR(i))
 557                                 dev_err(mtu->dev, "Rx %d CS error!\n", i);
 558
 559                         if (errval & QMU_RX_LEN_ERR(i))
 560                                 dev_err(mtu->dev, "RX %d Length error\n", i);
 561                 }
 562                 mtu3_writel(mbase, U3D_RQERRIR0, errval);
 563         }
 564
 565         if (qmu_status & RXQ_ZLPERR_INT) {
 566                 errval = mtu3_readl(mbase, U3D_RQERRIR1);
 567                 for (i = 1; i < mtu->num_eps; i++) {
 568                         if (errval & QMU_RX_ZLP_ERR(i))
 569                                 dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);
 570                 }
 571                 mtu3_writel(mbase, U3D_RQERRIR1, errval);
 572         }
 573
 574         if ((qmu_status & TXQ_CSERR_INT) || (qmu_status & TXQ_LENERR_INT)) {
 575                 errval = mtu3_readl(mbase, U3D_TQERRIR0);
 576                 for (i = 1; i < mtu->num_eps; i++) {
 577                         if (errval & QMU_TX_CS_ERR(i))
 578                                 dev_err(mtu->dev, "Tx %d checksum error!\n", i);
 579
 580                         if (errval & QMU_TX_LEN_ERR(i))
 581                                 qmu_tx_zlp_error_handler(mtu, i);
 582                 }
 583                 mtu3_writel(mbase, U3D_TQERRIR0, errval);
 584         }
 585 }
 586
 587 irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)
 588 {
 589         void __iomem *mbase = mtu->mac_base;
 590         u32 qmu_status;
 591         u32 qmu_done_status;
 592
 593         /* U3D_QISAR1 is read update */
 594         qmu_status = mtu3_readl(mbase, U3D_QISAR1);
 595         qmu_status &= mtu3_readl(mbase, U3D_QIER1);
 596
 597         qmu_done_status = mtu3_readl(mbase, U3D_QISAR0);
 598         qmu_done_status &= mtu3_readl(mbase, U3D_QIER0);
 599         mtu3_writel(mbase, U3D_QISAR0, qmu_done_status); /* W1C */
 600         dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",
 601                 (qmu_done_status & 0xFFFF), qmu_done_status >> 16,
 602                 qmu_status);
 603
 604         if (qmu_done_status)
 605                 qmu_done_isr(mtu, qmu_done_status);
 606
 607         if (qmu_status)
 608                 qmu_exception_isr(mtu, qmu_status);
 609
 610         return IRQ_HANDLED;
 611 }
 612
 613 int mtu3_qmu_init(struct mtu3 *mtu)
 614 {
 615
 616         compiletime_assert(QMU_GPD_SIZE == 16, "QMU_GPD size SHOULD be 16B");
 617
 618         mtu->qmu_gpd_pool = dma_pool_create("QMU_GPD", mtu->dev,
 619                         QMU_GPD_RING_SIZE, QMU_GPD_SIZE, 0);
 620
 621         if (!mtu->qmu_gpd_pool)
 622                 return -ENOMEM;
 623
 624         return 0;
 625 }
 626
 627 void mtu3_qmu_exit(struct mtu3 *mtu)
 628 {
 629         dma_pool_destroy(mtu->qmu_gpd_pool);
 630 }