drivers/usb/renesas_usbhs/fifo.c

   1 /*
   2  * Renesas USB driver
   3  *
   4  * Copyright (C) 2011 Renesas Solutions Corp.
   5  * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * along with this program; if not, write to the Free Software
  14  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  15  *
  16  */
  17 #include <linux/delay.h>
  18 #include <linux/io.h>
  19 #include <linux/scatterlist.h>
  20 #include "./common.h"
  21 #include "./pipe.h"
  22
  23 #define usbhsf_get_cfifo(p)     (&((p)->fifo_info.cfifo))
  24 #define usbhsf_get_d0fifo(p)    (&((p)->fifo_info.d0fifo))
  25 #define usbhsf_get_d1fifo(p)    (&((p)->fifo_info.d1fifo))
  26
  27 #define usbhsf_fifo_is_busy(f)  ((f)->pipe) /* see usbhs_pipe_select_fifo */
  28
  29 /*
  30  *              packet initialize
  31  */
  32 void usbhs_pkt_init(struct usbhs_pkt *pkt)
  33 {
  34         pkt->dma = DMA_ADDR_INVALID;
  35         INIT_LIST_HEAD(&pkt->node);
  36 }
  37
  38 /*
  39  *              packet control function
  40  */
  41 static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done)
  42 {
  43         struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);
  44         struct device *dev = usbhs_priv_to_dev(priv);
  45
  46         dev_err(dev, "null handler\n");
  47
  48         return -EINVAL;
  49 }
  50
  51 static struct usbhs_pkt_handle usbhsf_null_handler = {
  52         .prepare = usbhsf_null_handle,
  53         .try_run = usbhsf_null_handle,
  54 };
  55
  56 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,
  57                     struct usbhs_pkt_handle *handler,
  58                     void *buf, int len, int zero)
  59 {
  60         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
  61         struct device *dev = usbhs_priv_to_dev(priv);
  62         unsigned long flags;
  63
  64         /********************  spin lock ********************/
  65         usbhs_lock(priv, flags);
  66
  67         if (!handler) {
  68                 dev_err(dev, "no handler function\n");
  69                 handler = &usbhsf_null_handler;
  70         }
  71
  72         list_del_init(&pkt->node);
  73         list_add_tail(&pkt->node, &pipe->list);
  74
  75         pkt->pipe       = pipe;
  76         pkt->buf        = buf;
  77         pkt->handler    = handler;
  78         pkt->length     = len;
  79         pkt->zero       = zero;
  80         pkt->actual     = 0;
  81
  82         usbhs_unlock(priv, flags);
  83         /********************  spin unlock ******************/
  84
  85         usbhs_pkt_start(pipe);
  86 }
  87
  88 static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)
  89 {
  90         list_del_init(&pkt->node);
  91 }
  92
  93 static struct usbhs_pkt *__usbhsf_pkt_get(struct usbhs_pipe *pipe)
  94 {
  95         if (list_empty(&pipe->list))
  96                 return NULL;
  97
  98         return list_entry(pipe->list.next, struct usbhs_pkt, node);
  99 }
 100
 101 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)
 102 {
 103         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 104         unsigned long flags;
 105
 106         /********************  spin lock ********************/
 107         usbhs_lock(priv, flags);
 108
 109         if (!pkt)
 110                 pkt = __usbhsf_pkt_get(pipe);
 111
 112         if (pkt)
 113                 __usbhsf_pkt_del(pkt);
 114
 115         usbhs_unlock(priv, flags);
 116         /********************  spin unlock ******************/
 117
 118         return pkt;
 119 }
 120
 121 int __usbhs_pkt_handler(struct usbhs_pipe *pipe, int type)
 122 {
 123         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 124         struct usbhs_pipe_info *info = usbhs_priv_to_pipeinfo(priv);
 125         struct usbhs_pkt *pkt;
 126         struct device *dev = usbhs_priv_to_dev(priv);
 127         int (*func)(struct usbhs_pkt *pkt, int *is_done);
 128         unsigned long flags;
 129         int ret = 0;
 130         int is_done = 0;
 131
 132         /********************  spin lock ********************/
 133         usbhs_lock(priv, flags);
 134
 135         pkt = __usbhsf_pkt_get(pipe);
 136         if (!pkt)
 137                 goto __usbhs_pkt_handler_end;
 138
 139         switch (type) {
 140         case USBHSF_PKT_PREPARE:
 141                 func = pkt->handler->prepare;
 142                 break;
 143         case USBHSF_PKT_TRY_RUN:
 144                 func = pkt->handler->try_run;
 145                 break;
 146         case USBHSF_PKT_DMA_DONE:
 147                 func = pkt->handler->dma_done;
 148                 break;
 149         default:
 150                 dev_err(dev, "unknown pkt hander\n");
 151                 goto __usbhs_pkt_handler_end;
 152         }
 153
 154         ret = func(pkt, &is_done);
 155
 156         if (is_done)
 157                 __usbhsf_pkt_del(pkt);
 158
 159 __usbhs_pkt_handler_end:
 160         usbhs_unlock(priv, flags);
 161         /********************  spin unlock ******************/
 162
 163         if (is_done) {
 164                 info->done(pkt);
 165                 usbhs_pkt_start(pipe);
 166         }
 167
 168         return ret;
 169 }
 170
 171 /*
 172  *              irq enable/disable function
 173  */
 174 #define usbhsf_irq_empty_ctrl(p, e) usbhsf_irq_callback_ctrl(p, bempsts, e)
 175 #define usbhsf_irq_ready_ctrl(p, e) usbhsf_irq_callback_ctrl(p, brdysts, e)
 176 #define usbhsf_irq_callback_ctrl(pipe, status, enable)                  \
 177         ({                                                              \
 178                 struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);     \
 179                 struct usbhs_mod *mod = usbhs_mod_get_current(priv);    \
 180                 u16 status = (1 << usbhs_pipe_number(pipe));            \
 181                 if (!mod)                                               \
 182                         return;                                         \
 183                 if (enable)                                             \
 184                         mod->irq_##status |= status;                    \
 185                 else                                                    \
 186                         mod->irq_##status &= ~status;                   \
 187                 usbhs_irq_callback_update(priv, mod);                   \
 188         })
 189
 190 static void usbhsf_tx_irq_ctrl(struct usbhs_pipe *pipe, int enable)
 191 {
 192         /*
 193          * And DCP pipe can NOT use "ready interrupt" for "send"
 194          * it should use "empty" interrupt.
 195          * see
 196          *   "Operation" - "Interrupt Function" - "BRDY Interrupt"
 197          *
 198          * on the other hand, normal pipe can use "ready interrupt" for "send"
 199          * even though it is single/double buffer
 200          */
 201         if (usbhs_pipe_is_dcp(pipe))
 202                 usbhsf_irq_empty_ctrl(pipe, enable);
 203         else
 204                 usbhsf_irq_ready_ctrl(pipe, enable);
 205 }
 206
 207 static void usbhsf_rx_irq_ctrl(struct usbhs_pipe *pipe, int enable)
 208 {
 209         usbhsf_irq_ready_ctrl(pipe, enable);
 210 }
 211
 212 /*
 213  *              FIFO ctrl
 214  */
 215 static void usbhsf_send_terminator(struct usbhs_pipe *pipe,
 216                                    struct usbhs_fifo *fifo)
 217 {
 218         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 219
 220         usbhs_bset(priv, fifo->ctr, BVAL, BVAL);
 221 }
 222
 223 static int usbhsf_fifo_barrier(struct usbhs_priv *priv,
 224                                struct usbhs_fifo *fifo)
 225 {
 226         int timeout = 1024;
 227
 228         do {
 229                 /* The FIFO port is accessible */
 230                 if (usbhs_read(priv, fifo->ctr) & FRDY)
 231                         return 0;
 232
 233                 udelay(10);
 234         } while (timeout--);
 235
 236         return -EBUSY;
 237 }
 238
 239 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe,
 240                               struct usbhs_fifo *fifo)
 241 {
 242         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 243
 244         if (!usbhs_pipe_is_dcp(pipe))
 245                 usbhsf_fifo_barrier(priv, fifo);
 246
 247         usbhs_write(priv, fifo->ctr, BCLR);
 248 }
 249
 250 static int usbhsf_fifo_rcv_len(struct usbhs_priv *priv,
 251                                struct usbhs_fifo *fifo)
 252 {
 253         return usbhs_read(priv, fifo->ctr) & DTLN_MASK;
 254 }
 255
 256 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,
 257                                  struct usbhs_fifo *fifo)
 258 {
 259         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 260
 261         usbhs_pipe_select_fifo(pipe, NULL);
 262         usbhs_write(priv, fifo->sel, 0);
 263 }
 264
 265 static int usbhsf_fifo_select(struct usbhs_pipe *pipe,
 266                               struct usbhs_fifo *fifo,
 267                               int write)
 268 {
 269         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 270         struct device *dev = usbhs_priv_to_dev(priv);
 271         int timeout = 1024;
 272         u16 mask = ((1 << 5) | 0xF);            /* mask of ISEL | CURPIPE */
 273         u16 base = usbhs_pipe_number(pipe);     /* CURPIPE */
 274
 275         if (usbhs_pipe_is_busy(pipe) ||
 276             usbhsf_fifo_is_busy(fifo))
 277                 return -EBUSY;
 278
 279         if (usbhs_pipe_is_dcp(pipe))
 280                 base |= (1 == write) << 5;      /* ISEL */
 281
 282         /* "base" will be used below  */
 283         usbhs_write(priv, fifo->sel, base | MBW_32);
 284
 285         /* check ISEL and CURPIPE value */
 286         while (timeout--) {
 287                 if (base == (mask & usbhs_read(priv, fifo->sel))) {
 288                         usbhs_pipe_select_fifo(pipe, fifo);
 289                         return 0;
 290                 }
 291                 udelay(10);
 292         }
 293
 294         dev_err(dev, "fifo select error\n");
 295
 296         return -EIO;
 297 }
 298
 299 /*
 300  *              PIO push handler
 301  */
 302 static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done)
 303 {
 304         struct usbhs_pipe *pipe = pkt->pipe;
 305         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 306         struct device *dev = usbhs_priv_to_dev(priv);
 307         struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */
 308         void __iomem *addr = priv->base + fifo->port;
 309         u8 *buf;
 310         int maxp = usbhs_pipe_get_maxpacket(pipe);
 311         int total_len;
 312         int i, ret, len;
 313         int is_short;
 314
 315         ret = usbhsf_fifo_select(pipe, fifo, 1);
 316         if (ret < 0)
 317                 return 0;
 318
 319         ret = usbhs_pipe_is_accessible(pipe);
 320         if (ret < 0) {
 321                 /* inaccessible pipe is not an error */
 322                 ret = 0;
 323                 goto usbhs_fifo_write_busy;
 324         }
 325
 326         ret = usbhsf_fifo_barrier(priv, fifo);
 327         if (ret < 0)
 328                 goto usbhs_fifo_write_busy;
 329
 330         buf             = pkt->buf    + pkt->actual;
 331         len             = pkt->length - pkt->actual;
 332         len             = min(len, maxp);
 333         total_len       = len;
 334         is_short        = total_len < maxp;
 335
 336         /*
 337          * FIXME
 338          *
 339          * 32-bit access only
 340          */
 341         if (len >= 4 && !((unsigned long)buf & 0x03)) {
 342                 iowrite32_rep(addr, buf, len / 4);
 343                 len %= 4;
 344                 buf += total_len - len;
 345         }
 346
 347         /* the rest operation */
 348         for (i = 0; i < len; i++)
 349                 iowrite8(buf[i], addr + (0x03 - (i & 0x03)));
 350
 351         /*
 352          * variable update
 353          */
 354         pkt->actual += total_len;
 355
 356         if (pkt->actual < pkt->length)
 357                 *is_done = 0;           /* there are remainder data */
 358         else if (is_short)
 359                 *is_done = 1;           /* short packet */
 360         else
 361                 *is_done = !pkt->zero;  /* send zero packet ? */
 362
 363         /*
 364          * pipe/irq handling
 365          */
 366         if (is_short)
 367                 usbhsf_send_terminator(pipe, fifo);
 368
 369         usbhsf_tx_irq_ctrl(pipe, !*is_done);
 370         usbhs_pipe_enable(pipe);
 371
 372         dev_dbg(dev, "  send %d (%d/ %d/ %d/ %d)\n",
 373                 usbhs_pipe_number(pipe),
 374                 pkt->length, pkt->actual, *is_done, pkt->zero);
 375
 376         /*
 377          * Transmission end
 378          */
 379         if (*is_done) {
 380                 if (usbhs_pipe_is_dcp(pipe))
 381                         usbhs_dcp_control_transfer_done(pipe);
 382         }
 383
 384         usbhsf_fifo_unselect(pipe, fifo);
 385
 386         return 0;
 387
 388 usbhs_fifo_write_busy:
 389         usbhsf_fifo_unselect(pipe, fifo);
 390
 391         /*
 392          * pipe is busy.
 393          * retry in interrupt
 394          */
 395         usbhsf_tx_irq_ctrl(pipe, 1);
 396
 397         return ret;
 398 }
 399
 400 struct usbhs_pkt_handle usbhs_fifo_pio_push_handler = {
 401         .prepare = usbhsf_pio_try_push,
 402         .try_run = usbhsf_pio_try_push,
 403 };
 404
 405 /*
 406  *              PIO pop handler
 407  */
 408 static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done)
 409 {
 410         struct usbhs_pipe *pipe = pkt->pipe;
 411
 412         if (usbhs_pipe_is_busy(pipe))
 413                 return 0;
 414
 415         /*
 416          * pipe enable to prepare packet receive
 417          */
 418
 419         usbhs_pipe_enable(pipe);
 420         usbhsf_rx_irq_ctrl(pipe, 1);
 421
 422         return 0;
 423 }
 424
 425 static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done)
 426 {
 427         struct usbhs_pipe *pipe = pkt->pipe;
 428         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 429         struct device *dev = usbhs_priv_to_dev(priv);
 430         struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */
 431         void __iomem *addr = priv->base + fifo->port;
 432         u8 *buf;
 433         u32 data = 0;
 434         int maxp = usbhs_pipe_get_maxpacket(pipe);
 435         int rcv_len, len;
 436         int i, ret;
 437         int total_len = 0;
 438
 439         ret = usbhsf_fifo_select(pipe, fifo, 0);
 440         if (ret < 0)
 441                 return 0;
 442
 443         ret = usbhsf_fifo_barrier(priv, fifo);
 444         if (ret < 0)
 445                 goto usbhs_fifo_read_busy;
 446
 447         rcv_len = usbhsf_fifo_rcv_len(priv, fifo);
 448
 449         buf             = pkt->buf    + pkt->actual;
 450         len             = pkt->length - pkt->actual;
 451         len             = min(len, rcv_len);
 452         total_len       = len;
 453
 454         /*
 455          * Buffer clear if Zero-Length packet
 456          *
 457          * see
 458          * "Operation" - "FIFO Buffer Memory" - "FIFO Port Function"
 459          */
 460         if (0 == rcv_len) {
 461                 usbhsf_fifo_clear(pipe, fifo);
 462                 goto usbhs_fifo_read_end;
 463         }
 464
 465         /*
 466          * FIXME
 467          *
 468          * 32-bit access only
 469          */
 470         if (len >= 4 && !((unsigned long)buf & 0x03)) {
 471                 ioread32_rep(addr, buf, len / 4);
 472                 len %= 4;
 473                 buf += total_len - len;
 474         }
 475
 476         /* the rest operation */
 477         for (i = 0; i < len; i++) {
 478                 if (!(i & 0x03))
 479                         data = ioread32(addr);
 480
 481                 buf[i] = (data >> ((i & 0x03) * 8)) & 0xff;
 482         }
 483
 484         pkt->actual += total_len;
 485
 486 usbhs_fifo_read_end:
 487         if ((pkt->actual == pkt->length) ||     /* receive all data */
 488             (total_len < maxp)) {               /* short packet */
 489                 *is_done = 1;
 490                 usbhsf_rx_irq_ctrl(pipe, 0);
 491                 usbhs_pipe_disable(pipe);
 492         }
 493
 494         dev_dbg(dev, "  recv %d (%d/ %d/ %d/ %d)\n",
 495                 usbhs_pipe_number(pipe),
 496                 pkt->length, pkt->actual, *is_done, pkt->zero);
 497
 498 usbhs_fifo_read_busy:
 499         usbhsf_fifo_unselect(pipe, fifo);
 500
 501         return ret;
 502 }
 503
 504 struct usbhs_pkt_handle usbhs_fifo_pio_pop_handler = {
 505         .prepare = usbhsf_prepare_pop,
 506         .try_run = usbhsf_pio_try_pop,
 507 };
 508
 509 /*
 510  *              DCP ctrol statge handler
 511  */
 512 static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done)
 513 {
 514         usbhs_dcp_control_transfer_done(pkt->pipe);
 515
 516         *is_done = 1;
 517
 518         return 0;
 519 }
 520
 521 struct usbhs_pkt_handle usbhs_ctrl_stage_end_handler = {
 522         .prepare = usbhsf_ctrl_stage_end,
 523         .try_run = usbhsf_ctrl_stage_end,
 524 };
 525
 526 /*
 527  *              DMA fifo functions
 528  */
 529 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,
 530                                             struct usbhs_pkt *pkt)
 531 {
 532         if (&usbhs_fifo_dma_push_handler == pkt->handler)
 533                 return fifo->tx_chan;
 534
 535         if (&usbhs_fifo_dma_pop_handler == pkt->handler)
 536                 return fifo->rx_chan;
 537
 538         return NULL;
 539 }
 540
 541 static struct usbhs_fifo *usbhsf_get_dma_fifo(struct usbhs_priv *priv,
 542                                               struct usbhs_pkt *pkt)
 543 {
 544         struct usbhs_fifo *fifo;
 545
 546         /* DMA :: D0FIFO */
 547         fifo = usbhsf_get_d0fifo(priv);
 548         if (usbhsf_dma_chan_get(fifo, pkt) &&
 549             !usbhsf_fifo_is_busy(fifo))
 550                 return fifo;
 551
 552         /* DMA :: D1FIFO */
 553         fifo = usbhsf_get_d1fifo(priv);
 554         if (usbhsf_dma_chan_get(fifo, pkt) &&
 555             !usbhsf_fifo_is_busy(fifo))
 556                 return fifo;
 557
 558         return NULL;
 559 }
 560
 561 #define usbhsf_dma_start(p, f)  __usbhsf_dma_ctrl(p, f, DREQE)
 562 #define usbhsf_dma_stop(p, f)   __usbhsf_dma_ctrl(p, f, 0)
 563 static void __usbhsf_dma_ctrl(struct usbhs_pipe *pipe,
 564                               struct usbhs_fifo *fifo,
 565                               u16 dreqe)
 566 {
 567         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 568
 569         usbhs_bset(priv, fifo->sel, DREQE, dreqe);
 570 }
 571
 572 #define usbhsf_dma_map(p)       __usbhsf_dma_map_ctrl(p, 1)
 573 #define usbhsf_dma_unmap(p)     __usbhsf_dma_map_ctrl(p, 0)
 574 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map)
 575 {
 576         struct usbhs_pipe *pipe = pkt->pipe;
 577         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 578         struct usbhs_pipe_info *info = usbhs_priv_to_pipeinfo(priv);
 579
 580         return info->dma_map_ctrl(pkt, map);
 581 }
 582
 583 static void usbhsf_dma_complete(void *arg);
 584 static void usbhsf_dma_prepare_tasklet(unsigned long data)
 585 {
 586         struct usbhs_pkt *pkt = (struct usbhs_pkt *)data;
 587         struct usbhs_pipe *pipe = pkt->pipe;
 588         struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);
 589         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 590         struct scatterlist sg;
 591         struct dma_async_tx_descriptor *desc;
 592         struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);
 593         struct device *dev = usbhs_priv_to_dev(priv);
 594         enum dma_data_direction dir;
 595         dma_cookie_t cookie;
 596
 597         dir = usbhs_pipe_is_dir_in(pipe) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
 598
 599         sg_init_table(&sg, 1);
 600         sg_set_page(&sg, virt_to_page(pkt->dma),
 601                     pkt->length, offset_in_page(pkt->dma));
 602         sg_dma_address(&sg) = pkt->dma + pkt->actual;
 603         sg_dma_len(&sg) = pkt->trans;
 604
 605         desc = chan->device->device_prep_slave_sg(chan, &sg, 1, dir,
 606                                                   DMA_PREP_INTERRUPT |
 607                                                   DMA_CTRL_ACK);
 608         if (!desc)
 609                 return;
 610
 611         desc->callback          = usbhsf_dma_complete;
 612         desc->callback_param    = pipe;
 613
 614         cookie = desc->tx_submit(desc);
 615         if (cookie < 0) {
 616                 dev_err(dev, "Failed to submit dma descriptor\n");
 617                 return;
 618         }
 619
 620         dev_dbg(dev, "  %s %d (%d/ %d)\n",
 621                 fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);
 622
 623         usbhsf_dma_start(pipe, fifo);
 624         dma_async_issue_pending(chan);
 625 }
 626
 627 /*
 628  *              DMA push handler
 629  */
 630 static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)
 631 {
 632         struct usbhs_pipe *pipe = pkt->pipe;
 633         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 634         struct usbhs_fifo *fifo;
 635         int len = pkt->length - pkt->actual;
 636         int ret;
 637
 638         if (usbhs_pipe_is_busy(pipe))
 639                 return 0;
 640
 641         /* use PIO if packet is less than pio_dma_border or pipe is DCP */
 642         if ((len < usbhs_get_dparam(priv, pio_dma_border)) ||
 643             usbhs_pipe_is_dcp(pipe))
 644                 goto usbhsf_pio_prepare_push;
 645
 646         if (len % 4) /* 32bit alignment */
 647                 goto usbhsf_pio_prepare_push;
 648
 649         if (((u32)pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */
 650                 goto usbhsf_pio_prepare_push;
 651
 652         /* get enable DMA fifo */
 653         fifo = usbhsf_get_dma_fifo(priv, pkt);
 654         if (!fifo)
 655                 goto usbhsf_pio_prepare_push;
 656
 657         if (usbhsf_dma_map(pkt) < 0)
 658                 goto usbhsf_pio_prepare_push;
 659
 660         ret = usbhsf_fifo_select(pipe, fifo, 0);
 661         if (ret < 0)
 662                 goto usbhsf_pio_prepare_push_unmap;
 663
 664         pkt->trans = len;
 665
 666         tasklet_init(&fifo->tasklet,
 667                      usbhsf_dma_prepare_tasklet,
 668                      (unsigned long)pkt);
 669
 670         tasklet_schedule(&fifo->tasklet);
 671
 672         return 0;
 673
 674 usbhsf_pio_prepare_push_unmap:
 675         usbhsf_dma_unmap(pkt);
 676 usbhsf_pio_prepare_push:
 677         /*
 678          * change handler to PIO
 679          */
 680         pkt->handler = &usbhs_fifo_pio_push_handler;
 681
 682         return pkt->handler->prepare(pkt, is_done);
 683 }
 684
 685 static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done)
 686 {
 687         struct usbhs_pipe *pipe = pkt->pipe;
 688
 689         pkt->actual = pkt->trans;
 690
 691         *is_done = !pkt->zero;  /* send zero packet ? */
 692
 693         usbhsf_dma_stop(pipe, pipe->fifo);
 694         usbhsf_dma_unmap(pkt);
 695         usbhsf_fifo_unselect(pipe, pipe->fifo);
 696
 697         return 0;
 698 }
 699
 700 struct usbhs_pkt_handle usbhs_fifo_dma_push_handler = {
 701         .prepare        = usbhsf_dma_prepare_push,
 702         .dma_done       = usbhsf_dma_push_done,
 703 };
 704
 705 /*
 706  *              DMA pop handler
 707  */
 708 static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done)
 709 {
 710         struct usbhs_pipe *pipe = pkt->pipe;
 711         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 712         struct usbhs_fifo *fifo;
 713         int len, ret;
 714
 715         if (usbhs_pipe_is_busy(pipe))
 716                 return 0;
 717
 718         if (usbhs_pipe_is_dcp(pipe))
 719                 goto usbhsf_pio_prepare_pop;
 720
 721         /* get enable DMA fifo */
 722         fifo = usbhsf_get_dma_fifo(priv, pkt);
 723         if (!fifo)
 724                 goto usbhsf_pio_prepare_pop;
 725
 726         if (((u32)pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */
 727                 goto usbhsf_pio_prepare_pop;
 728
 729         ret = usbhsf_fifo_select(pipe, fifo, 0);
 730         if (ret < 0)
 731                 goto usbhsf_pio_prepare_pop;
 732
 733         /* use PIO if packet is less than pio_dma_border */
 734         len = usbhsf_fifo_rcv_len(priv, fifo);
 735         len = min(pkt->length - pkt->actual, len);
 736         if (len % 4) /* 32bit alignment */
 737                 goto usbhsf_pio_prepare_pop_unselect;
 738
 739         if (len < usbhs_get_dparam(priv, pio_dma_border))
 740                 goto usbhsf_pio_prepare_pop_unselect;
 741
 742         ret = usbhsf_fifo_barrier(priv, fifo);
 743         if (ret < 0)
 744                 goto usbhsf_pio_prepare_pop_unselect;
 745
 746         if (usbhsf_dma_map(pkt) < 0)
 747                 goto usbhsf_pio_prepare_pop_unselect;
 748
 749         /* DMA */
 750
 751         /*
 752          * usbhs_fifo_dma_pop_handler :: prepare
 753          * enabled irq to come here.
 754          * but it is no longer needed for DMA. disable it.
 755          */
 756         usbhsf_rx_irq_ctrl(pipe, 0);
 757
 758         pkt->trans = len;
 759
 760         tasklet_init(&fifo->tasklet,
 761                      usbhsf_dma_prepare_tasklet,
 762                      (unsigned long)pkt);
 763
 764         tasklet_schedule(&fifo->tasklet);
 765
 766         return 0;
 767
 768 usbhsf_pio_prepare_pop_unselect:
 769         usbhsf_fifo_unselect(pipe, fifo);
 770 usbhsf_pio_prepare_pop:
 771
 772         /*
 773          * change handler to PIO
 774          */
 775         pkt->handler = &usbhs_fifo_pio_pop_handler;
 776
 777         return pkt->handler->try_run(pkt, is_done);
 778 }
 779
 780 static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done)
 781 {
 782         struct usbhs_pipe *pipe = pkt->pipe;
 783         int maxp = usbhs_pipe_get_maxpacket(pipe);
 784
 785         usbhsf_dma_stop(pipe, pipe->fifo);
 786         usbhsf_dma_unmap(pkt);
 787         usbhsf_fifo_unselect(pipe, pipe->fifo);
 788
 789         pkt->actual += pkt->trans;
 790
 791         if ((pkt->actual == pkt->length) ||     /* receive all data */
 792             (pkt->trans < maxp)) {              /* short packet */
 793                 *is_done = 1;
 794         } else {
 795                 /* re-enable */
 796                 usbhsf_prepare_pop(pkt, is_done);
 797         }
 798
 799         return 0;
 800 }
 801
 802 struct usbhs_pkt_handle usbhs_fifo_dma_pop_handler = {
 803         .prepare        = usbhsf_prepare_pop,
 804         .try_run        = usbhsf_dma_try_pop,
 805         .dma_done       = usbhsf_dma_pop_done
 806 };
 807
 808 /*
 809  *              DMA setting
 810  */
 811 static bool usbhsf_dma_filter(struct dma_chan *chan, void *param)
 812 {
 813         struct sh_dmae_slave *slave = param;
 814
 815         /*
 816          * FIXME
 817          *
 818          * usbhs doesn't recognize id = 0 as valid DMA
 819          */
 820         if (0 == slave->slave_id)
 821                 return false;
 822
 823         chan->private = slave;
 824
 825         return true;
 826 }
 827
 828 static void usbhsf_dma_quit(struct usbhs_priv *priv, struct usbhs_fifo *fifo)
 829 {
 830         if (fifo->tx_chan)
 831                 dma_release_channel(fifo->tx_chan);
 832         if (fifo->rx_chan)
 833                 dma_release_channel(fifo->rx_chan);
 834
 835         fifo->tx_chan = NULL;
 836         fifo->rx_chan = NULL;
 837 }
 838
 839 static void usbhsf_dma_init(struct usbhs_priv *priv,
 840                             struct usbhs_fifo *fifo)
 841 {
 842         struct device *dev = usbhs_priv_to_dev(priv);
 843         dma_cap_mask_t mask;
 844
 845         dma_cap_zero(mask);
 846         dma_cap_set(DMA_SLAVE, mask);
 847         fifo->tx_chan = dma_request_channel(mask, usbhsf_dma_filter,
 848                                             &fifo->tx_slave);
 849
 850         dma_cap_zero(mask);
 851         dma_cap_set(DMA_SLAVE, mask);
 852         fifo->rx_chan = dma_request_channel(mask, usbhsf_dma_filter,
 853                                             &fifo->rx_slave);
 854
 855         if (fifo->tx_chan || fifo->rx_chan)
 856                 dev_dbg(dev, "enable DMAEngine (%s%s%s)\n",
 857                          fifo->name,
 858                          fifo->tx_chan ? "[TX]" : "    ",
 859                          fifo->rx_chan ? "[RX]" : "    ");
 860 }
 861
 862 /*
 863  *              irq functions
 864  */
 865 static int usbhsf_irq_empty(struct usbhs_priv *priv,
 866                             struct usbhs_irq_state *irq_state)
 867 {
 868         struct usbhs_pipe *pipe;
 869         struct device *dev = usbhs_priv_to_dev(priv);
 870         int i, ret;
 871
 872         if (!irq_state->bempsts) {
 873                 dev_err(dev, "debug %s !!\n", __func__);
 874                 return -EIO;
 875         }
 876
 877         dev_dbg(dev, "irq empty [0x%04x]\n", irq_state->bempsts);
 878
 879         /*
 880          * search interrupted "pipe"
 881          * not "uep".
 882          */
 883         usbhs_for_each_pipe_with_dcp(pipe, priv, i) {
 884                 if (!(irq_state->bempsts & (1 << i)))
 885                         continue;
 886
 887                 ret = usbhs_pkt_run(pipe);
 888                 if (ret < 0)
 889                         dev_err(dev, "irq_empty run_error %d : %d\n", i, ret);
 890         }
 891
 892         return 0;
 893 }
 894
 895 static int usbhsf_irq_ready(struct usbhs_priv *priv,
 896                             struct usbhs_irq_state *irq_state)
 897 {
 898         struct usbhs_pipe *pipe;
 899         struct device *dev = usbhs_priv_to_dev(priv);
 900         int i, ret;
 901
 902         if (!irq_state->brdysts) {
 903                 dev_err(dev, "debug %s !!\n", __func__);
 904                 return -EIO;
 905         }
 906
 907         dev_dbg(dev, "irq ready [0x%04x]\n", irq_state->brdysts);
 908
 909         /*
 910          * search interrupted "pipe"
 911          * not "uep".
 912          */
 913         usbhs_for_each_pipe_with_dcp(pipe, priv, i) {
 914                 if (!(irq_state->brdysts & (1 << i)))
 915                         continue;
 916
 917                 ret = usbhs_pkt_run(pipe);
 918                 if (ret < 0)
 919                         dev_err(dev, "irq_ready run_error %d : %d\n", i, ret);
 920         }
 921
 922         return 0;
 923 }
 924
 925 static void usbhsf_dma_complete(void *arg)
 926 {
 927         struct usbhs_pipe *pipe = arg;
 928         struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);
 929         struct device *dev = usbhs_priv_to_dev(priv);
 930         int ret;
 931
 932         ret = usbhs_pkt_dmadone(pipe);
 933         if (ret < 0)
 934                 dev_err(dev, "dma_complete run_error %d : %d\n",
 935                         usbhs_pipe_number(pipe), ret);
 936 }
 937
 938 /*
 939  *              fifo init
 940  */
 941 void usbhs_fifo_init(struct usbhs_priv *priv)
 942 {
 943         struct usbhs_mod *mod = usbhs_mod_get_current(priv);
 944         struct usbhs_fifo *cfifo = usbhsf_get_cfifo(priv);
 945         struct usbhs_fifo *d0fifo = usbhsf_get_d0fifo(priv);
 946         struct usbhs_fifo *d1fifo = usbhsf_get_d1fifo(priv);
 947
 948         mod->irq_empty          = usbhsf_irq_empty;
 949         mod->irq_ready          = usbhsf_irq_ready;
 950         mod->irq_bempsts        = 0;
 951         mod->irq_brdysts        = 0;
 952
 953         cfifo->pipe     = NULL;
 954         cfifo->tx_chan  = NULL;
 955         cfifo->rx_chan  = NULL;
 956
 957         d0fifo->pipe    = NULL;
 958         d0fifo->tx_chan = NULL;
 959         d0fifo->rx_chan = NULL;
 960
 961         d1fifo->pipe    = NULL;
 962         d1fifo->tx_chan = NULL;
 963         d1fifo->rx_chan = NULL;
 964
 965         usbhsf_dma_init(priv, usbhsf_get_d0fifo(priv));
 966         usbhsf_dma_init(priv, usbhsf_get_d1fifo(priv));
 967 }
 968
 969 void usbhs_fifo_quit(struct usbhs_priv *priv)
 970 {
 971         struct usbhs_mod *mod = usbhs_mod_get_current(priv);
 972
 973         mod->irq_empty          = NULL;
 974         mod->irq_ready          = NULL;
 975         mod->irq_bempsts        = 0;
 976         mod->irq_brdysts        = 0;
 977
 978         usbhsf_dma_quit(priv, usbhsf_get_d0fifo(priv));
 979         usbhsf_dma_quit(priv, usbhsf_get_d1fifo(priv));
 980 }
 981
 982 int usbhs_fifo_probe(struct usbhs_priv *priv)
 983 {
 984         struct usbhs_fifo *fifo;
 985
 986         /* CFIFO */
 987         fifo = usbhsf_get_cfifo(priv);
 988         fifo->name      = "CFIFO";
 989         fifo->port      = CFIFO;
 990         fifo->sel       = CFIFOSEL;
 991         fifo->ctr       = CFIFOCTR;
 992
 993         /* D0FIFO */
 994         fifo = usbhsf_get_d0fifo(priv);
 995         fifo->name      = "D0FIFO";
 996         fifo->port      = D0FIFO;
 997         fifo->sel       = D0FIFOSEL;
 998         fifo->ctr       = D0FIFOCTR;
 999         fifo->tx_slave.slave_id = usbhs_get_dparam(priv, d0_tx_id);
1000         fifo->rx_slave.slave_id = usbhs_get_dparam(priv, d0_rx_id);
1001
1002         /* D1FIFO */
1003         fifo = usbhsf_get_d1fifo(priv);
1004         fifo->name      = "D1FIFO";
1005         fifo->port      = D1FIFO;
1006         fifo->sel       = D1FIFOSEL;
1007         fifo->ctr       = D1FIFOCTR;
1008         fifo->tx_slave.slave_id = usbhs_get_dparam(priv, d1_tx_id);
1009         fifo->rx_slave.slave_id = usbhs_get_dparam(priv, d1_rx_id);
1010
1011         return 0;
1012 }
1013
1014 void usbhs_fifo_remove(struct usbhs_priv *priv)
1015 {
1016 }