drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 /**
  31  * dwc3_gadget_set_test_mode - enables usb2 test modes
  32  * @dwc: pointer to our context structure
  33  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  34  *
  35  * Caller should take care of locking. This function will return 0 on
  36  * success or -EINVAL if wrong Test Selector is passed.
  37  */
  38 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  39 {
  40         u32             reg;
  41
  42         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  43         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  44
  45         switch (mode) {
  46         case TEST_J:
  47         case TEST_K:
  48         case TEST_SE0_NAK:
  49         case TEST_PACKET:
  50         case TEST_FORCE_EN:
  51                 reg |= mode << 1;
  52                 break;
  53         default:
  54                 return -EINVAL;
  55         }
  56
  57         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  58
  59         return 0;
  60 }
  61
  62 /**
  63  * dwc3_gadget_get_link_state - gets current state of usb link
  64  * @dwc: pointer to our context structure
  65  *
  66  * Caller should take care of locking. This function will
  67  * return the link state on success (>= 0) or -ETIMEDOUT.
  68  */
  69 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  70 {
  71         u32             reg;
  72
  73         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  74
  75         return DWC3_DSTS_USBLNKST(reg);
  76 }
  77
  78 /**
  79  * dwc3_gadget_set_link_state - sets usb link to a particular state
  80  * @dwc: pointer to our context structure
  81  * @state: the state to put link into
  82  *
  83  * Caller should take care of locking. This function will
  84  * return 0 on success or -ETIMEDOUT.
  85  */
  86 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  87 {
  88         int             retries = 10000;
  89         u32             reg;
  90
  91         /*
  92          * Wait until device controller is ready. Only applies to 1.94a and
  93          * later RTL.
  94          */
  95         if (dwc->revision >= DWC3_REVISION_194A) {
  96                 while (--retries) {
  97                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  98                         if (reg & DWC3_DSTS_DCNRD)
  99                                 udelay(5);
 100                         else
 101                                 break;
 102                 }
 103
 104                 if (retries <= 0)
 105                         return -ETIMEDOUT;
 106         }
 107
 108         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 109         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 110
 111         /* set requested state */
 112         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 113         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 114
 115         /*
 116          * The following code is racy when called from dwc3_gadget_wakeup,
 117          * and is not needed, at least on newer versions
 118          */
 119         if (dwc->revision >= DWC3_REVISION_194A)
 120                 return 0;
 121
 122         /* wait for a change in DSTS */
 123         retries = 10000;
 124         while (--retries) {
 125                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 126
 127                 if (DWC3_DSTS_USBLNKST(reg) == state)
 128                         return 0;
 129
 130                 udelay(5);
 131         }
 132
 133         return -ETIMEDOUT;
 134 }
 135
 136 /**
 137  * dwc3_ep_inc_trb - increment a trb index.
 138  * @index: Pointer to the TRB index to increment.
 139  *
 140  * The index should never point to the link TRB. After incrementing,
 141  * if it is point to the link TRB, wrap around to the beginning. The
 142  * link TRB is always at the last TRB entry.
 143  */
 144 static void dwc3_ep_inc_trb(u8 *index)
 145 {
 146         (*index)++;
 147         if (*index == (DWC3_TRB_NUM - 1))
 148                 *index = 0;
 149 }
 150
 151 /**
 152  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 153  * @dep: The endpoint whose enqueue pointer we're incrementing
 154  */
 155 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 156 {
 157         dwc3_ep_inc_trb(&dep->trb_enqueue);
 158 }
 159
 160 /**
 161  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 162  * @dep: The endpoint whose enqueue pointer we're incrementing
 163  */
 164 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 165 {
 166         dwc3_ep_inc_trb(&dep->trb_dequeue);
 167 }
 168
 169 void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 170                 struct dwc3_request *req, int status)
 171 {
 172         struct dwc3                     *dwc = dep->dwc;
 173
 174         req->started = false;
 175         list_del(&req->list);
 176         req->remaining = 0;
 177
 178         if (req->request.status == -EINPROGRESS)
 179                 req->request.status = status;
 180
 181         if (req->trb)
 182                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 183                                 &req->request, req->direction);
 184
 185         req->trb = NULL;
 186         trace_dwc3_gadget_giveback(req);
 187
 188         if (dep->number > 1)
 189                 pm_runtime_put(dwc->dev);
 190 }
 191
 192 /**
 193  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 194  * @dep: The endpoint to whom the request belongs to
 195  * @req: The request we're giving back
 196  * @status: completion code for the request
 197  *
 198  * Must be called with controller's lock held and interrupts disabled. This
 199  * function will unmap @req and call its ->complete() callback to notify upper
 200  * layers that it has completed.
 201  */
 202 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 203                 int status)
 204 {
 205         struct dwc3                     *dwc = dep->dwc;
 206
 207         dwc3_gadget_del_and_unmap_request(dep, req, status);
 208
 209         spin_unlock(&dwc->lock);
 210         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 211         spin_lock(&dwc->lock);
 212 }
 213
 214 /**
 215  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 216  * @dwc: pointer to the controller context
 217  * @cmd: the command to be issued
 218  * @param: command parameter
 219  *
 220  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 221  * and wait for its completion.
 222  */
 223 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 224 {
 225         u32             timeout = 500;
 226         int             status = 0;
 227         int             ret = 0;
 228         u32             reg;
 229
 230         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 231         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 232
 233         do {
 234                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 235                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 236                         status = DWC3_DGCMD_STATUS(reg);
 237                         if (status)
 238                                 ret = -EINVAL;
 239                         break;
 240                 }
 241         } while (--timeout);
 242
 243         if (!timeout) {
 244                 ret = -ETIMEDOUT;
 245                 status = -ETIMEDOUT;
 246         }
 247
 248         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 249
 250         return ret;
 251 }
 252
 253 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 254
 255 /**
 256  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 257  * @dep: the endpoint to which the command is going to be issued
 258  * @cmd: the command to be issued
 259  * @params: parameters to the command
 260  *
 261  * Caller should handle locking. This function will issue @cmd with given
 262  * @params to @dep and wait for its completion.
 263  */
 264 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 265                 struct dwc3_gadget_ep_cmd_params *params)
 266 {
 267         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 268         struct dwc3             *dwc = dep->dwc;
 269         u32                     timeout = 1000;
 270         u32                     reg;
 271
 272         int                     cmd_status = 0;
 273         int                     susphy = false;
 274         int                     ret = -EINVAL;
 275
 276         /*
 277          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 278          * we're issuing an endpoint command, we must check if
 279          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 280          *
 281          * We will also set SUSPHY bit to what it was before returning as stated
 282          * by the same section on Synopsys databook.
 283          */
 284         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 285                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 286                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 287                         susphy = true;
 288                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 289                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 290                 }
 291         }
 292
 293         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 294                 int             needs_wakeup;
 295
 296                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 297                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 298                                 dwc->link_state == DWC3_LINK_STATE_U3);
 299
 300                 if (unlikely(needs_wakeup)) {
 301                         ret = __dwc3_gadget_wakeup(dwc);
 302                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 303                                         ret);
 304                 }
 305         }
 306
 307         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 308         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 309         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 310
 311         /*
 312          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 313          * not relying on XferNotReady, we can make use of a special "No
 314          * Response Update Transfer" command where we should clear both CmdAct
 315          * and CmdIOC bits.
 316          *
 317          * With this, we don't need to wait for command completion and can
 318          * straight away issue further commands to the endpoint.
 319          *
 320          * NOTICE: We're making an assumption that control endpoints will never
 321          * make use of Update Transfer command. This is a safe assumption
 322          * because we can never have more than one request at a time with
 323          * Control Endpoints. If anybody changes that assumption, this chunk
 324          * needs to be updated accordingly.
 325          */
 326         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 327                         !usb_endpoint_xfer_isoc(desc))
 328                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 329         else
 330                 cmd |= DWC3_DEPCMD_CMDACT;
 331
 332         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 333         do {
 334                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 335                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 336                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 337
 338                         switch (cmd_status) {
 339                         case 0:
 340                                 ret = 0;
 341                                 break;
 342                         case DEPEVT_TRANSFER_NO_RESOURCE:
 343                                 ret = -EINVAL;
 344                                 break;
 345                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 346                                 /*
 347                                  * SW issues START TRANSFER command to
 348                                  * isochronous ep with future frame interval. If
 349                                  * future interval time has already passed when
 350                                  * core receives the command, it will respond
 351                                  * with an error status of 'Bus Expiry'.
 352                                  *
 353                                  * Instead of always returning -EINVAL, let's
 354                                  * give a hint to the gadget driver that this is
 355                                  * the case by returning -EAGAIN.
 356                                  */
 357                                 ret = -EAGAIN;
 358                                 break;
 359                         default:
 360                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 361                         }
 362
 363                         break;
 364                 }
 365         } while (--timeout);
 366
 367         if (timeout == 0) {
 368                 ret = -ETIMEDOUT;
 369                 cmd_status = -ETIMEDOUT;
 370         }
 371
 372         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 373
 374         if (ret == 0) {
 375                 switch (DWC3_DEPCMD_CMD(cmd)) {
 376                 case DWC3_DEPCMD_STARTTRANSFER:
 377                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 378                         break;
 379                 case DWC3_DEPCMD_ENDTRANSFER:
 380                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 381                         break;
 382                 default:
 383                         /* nothing */
 384                         break;
 385                 }
 386         }
 387
 388         if (unlikely(susphy)) {
 389                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 390                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 391                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 392         }
 393
 394         return ret;
 395 }
 396
 397 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 398 {
 399         struct dwc3 *dwc = dep->dwc;
 400         struct dwc3_gadget_ep_cmd_params params;
 401         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 402
 403         /*
 404          * As of core revision 2.60a the recommended programming model
 405          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 406          * command for IN endpoints. This is to prevent an issue where
 407          * some (non-compliant) hosts may not send ACK TPs for pending
 408          * IN transfers due to a mishandled error condition. Synopsys
 409          * STAR 9000614252.
 410          */
 411         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 412             (dwc->gadget.speed >= USB_SPEED_SUPER))
 413                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 414
 415         memset(&params, 0, sizeof(params));
 416
 417         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 418 }
 419
 420 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 421                 struct dwc3_trb *trb)
 422 {
 423         u32             offset = (char *) trb - (char *) dep->trb_pool;
 424
 425         return dep->trb_pool_dma + offset;
 426 }
 427
 428 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 429 {
 430         struct dwc3             *dwc = dep->dwc;
 431
 432         if (dep->trb_pool)
 433                 return 0;
 434
 435         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 436                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 437                         &dep->trb_pool_dma, GFP_KERNEL);
 438         if (!dep->trb_pool) {
 439                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 440                                 dep->name);
 441                 return -ENOMEM;
 442         }
 443
 444         return 0;
 445 }
 446
 447 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 448 {
 449         struct dwc3             *dwc = dep->dwc;
 450
 451         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 452                         dep->trb_pool, dep->trb_pool_dma);
 453
 454         dep->trb_pool = NULL;
 455         dep->trb_pool_dma = 0;
 456 }
 457
 458 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);
 459
 460 /**
 461  * dwc3_gadget_start_config - configure ep resources
 462  * @dwc: pointer to our controller context structure
 463  * @dep: endpoint that is being enabled
 464  *
 465  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 466  * completion, it will set Transfer Resource for all available endpoints.
 467  *
 468  * The assignment of transfer resources cannot perfectly follow the data book
 469  * due to the fact that the controller driver does not have all knowledge of the
 470  * configuration in advance. It is given this information piecemeal by the
 471  * composite gadget framework after every SET_CONFIGURATION and
 472  * SET_INTERFACE. Trying to follow the databook programming model in this
 473  * scenario can cause errors. For two reasons:
 474  *
 475  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 476  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 477  * incorrect in the scenario of multiple interfaces.
 478  *
 479  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 480  * endpoint on alt setting (8.1.6).
 481  *
 482  * The following simplified method is used instead:
 483  *
 484  * All hardware endpoints can be assigned a transfer resource and this setting
 485  * will stay persistent until either a core reset or hibernation. So whenever we
 486  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 487  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 488  * guaranteed that there are as many transfer resources as endpoints.
 489  *
 490  * This function is called for each endpoint when it is being enabled but is
 491  * triggered only when called for EP0-out, which always happens first, and which
 492  * should only happen in one of the above conditions.
 493  */
 494 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 495 {
 496         struct dwc3_gadget_ep_cmd_params params;
 497         u32                     cmd;
 498         int                     i;
 499         int                     ret;
 500
 501         if (dep->number)
 502                 return 0;
 503
 504         memset(&params, 0x00, sizeof(params));
 505         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 506
 507         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 508         if (ret)
 509                 return ret;
 510
 511         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 512                 struct dwc3_ep *dep = dwc->eps[i];
 513
 514                 if (!dep)
 515                         continue;
 516
 517                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 518                 if (ret)
 519                         return ret;
 520         }
 521
 522         return 0;
 523 }
 524
 525 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 526                 bool modify, bool restore)
 527 {
 528         const struct usb_ss_ep_comp_descriptor *comp_desc;
 529         const struct usb_endpoint_descriptor *desc;
 530         struct dwc3_gadget_ep_cmd_params params;
 531
 532         if (dev_WARN_ONCE(dwc->dev, modify && restore,
 533                                         "Can't modify and restore\n"))
 534                 return -EINVAL;
 535
 536         comp_desc = dep->endpoint.comp_desc;
 537         desc = dep->endpoint.desc;
 538
 539         memset(&params, 0x00, sizeof(params));
 540
 541         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 542                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 543
 544         /* Burst size is only needed in SuperSpeed mode */
 545         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 546                 u32 burst = dep->endpoint.maxburst;
 547                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 548         }
 549
 550         if (modify) {
 551                 params.param0 |= DWC3_DEPCFG_ACTION_MODIFY;
 552         } else if (restore) {
 553                 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
 554                 params.param2 |= dep->saved_state;
 555         } else {
 556                 params.param0 |= DWC3_DEPCFG_ACTION_INIT;
 557         }
 558
 559         if (usb_endpoint_xfer_control(desc))
 560                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 561
 562         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 563                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 564
 565         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 566                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 567                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 568                 dep->stream_capable = true;
 569         }
 570
 571         if (!usb_endpoint_xfer_control(desc))
 572                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 573
 574         /*
 575          * We are doing 1:1 mapping for endpoints, meaning
 576          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 577          * so on. We consider the direction bit as part of the physical
 578          * endpoint number. So USB endpoint 0x81 is 0x03.
 579          */
 580         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 581
 582         /*
 583          * We must use the lower 16 TX FIFOs even though
 584          * HW might have more
 585          */
 586         if (dep->direction)
 587                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 588
 589         if (desc->bInterval) {
 590                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 591                 dep->interval = 1 << (desc->bInterval - 1);
 592         }
 593
 594         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 595 }
 596
 597 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 598 {
 599         struct dwc3_gadget_ep_cmd_params params;
 600
 601         memset(&params, 0x00, sizeof(params));
 602
 603         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 604
 605         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 606                         &params);
 607 }
 608
 609 /**
 610  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 611  * @dep: endpoint to be initialized
 612  * @modify: if true, modify existing endpoint configuration
 613  * @restore: if true, restore endpoint configuration from scratch buffer
 614  *
 615  * Caller should take care of locking. Execute all necessary commands to
 616  * initialize a HW endpoint so it can be used by a gadget driver.
 617  */
 618 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 619                 bool modify, bool restore)
 620 {
 621         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 622         struct dwc3             *dwc = dep->dwc;
 623
 624         u32                     reg;
 625         int                     ret;
 626
 627         if (!(dep->flags & DWC3_EP_ENABLED)) {
 628                 ret = dwc3_gadget_start_config(dwc, dep);
 629                 if (ret)
 630                         return ret;
 631         }
 632
 633         ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore);
 634         if (ret)
 635                 return ret;
 636
 637         if (!(dep->flags & DWC3_EP_ENABLED)) {
 638                 struct dwc3_trb *trb_st_hw;
 639                 struct dwc3_trb *trb_link;
 640
 641                 dep->type = usb_endpoint_type(desc);
 642                 dep->flags |= DWC3_EP_ENABLED;
 643                 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 644
 645                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 646                 reg |= DWC3_DALEPENA_EP(dep->number);
 647                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 648
 649                 init_waitqueue_head(&dep->wait_end_transfer);
 650
 651                 if (usb_endpoint_xfer_control(desc))
 652                         goto out;
 653
 654                 /* Initialize the TRB ring */
 655                 dep->trb_dequeue = 0;
 656                 dep->trb_enqueue = 0;
 657                 memset(dep->trb_pool, 0,
 658                        sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 659
 660                 /* Link TRB. The HWO bit is never reset */
 661                 trb_st_hw = &dep->trb_pool[0];
 662
 663                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 664                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 665                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 666                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 667                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 668         }
 669
 670         /*
 671          * Issue StartTransfer here with no-op TRB so we can always rely on No
 672          * Response Update Transfer command.
 673          */
 674         if (usb_endpoint_xfer_bulk(desc)) {
 675                 struct dwc3_gadget_ep_cmd_params params;
 676                 struct dwc3_trb *trb;
 677                 dma_addr_t trb_dma;
 678                 u32 cmd;
 679
 680                 memset(&params, 0, sizeof(params));
 681                 trb = &dep->trb_pool[0];
 682                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 683
 684                 params.param0 = upper_32_bits(trb_dma);
 685                 params.param1 = lower_32_bits(trb_dma);
 686
 687                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 688
 689                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 690                 if (ret < 0)
 691                         return ret;
 692
 693                 dep->flags |= DWC3_EP_BUSY;
 694
 695                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
 696                 WARN_ON_ONCE(!dep->resource_index);
 697         }
 698
 699
 700 out:
 701         trace_dwc3_gadget_ep_enable(dep);
 702
 703         return 0;
 704 }
 705
 706 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
 707 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 708 {
 709         struct dwc3_request             *req;
 710
 711         dwc3_stop_active_transfer(dwc, dep->number, true);
 712
 713         /* - giveback all requests to gadget driver */
 714         while (!list_empty(&dep->started_list)) {
 715                 req = next_request(&dep->started_list);
 716
 717                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 718         }
 719
 720         while (!list_empty(&dep->pending_list)) {
 721                 req = next_request(&dep->pending_list);
 722
 723                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 724         }
 725 }
 726
 727 /**
 728  * __dwc3_gadget_ep_disable - disables a hw endpoint
 729  * @dep: the endpoint to disable
 730  *
 731  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 732  * requests which are currently being processed by the hardware and those which
 733  * are not yet scheduled.
 734  *
 735  * Caller should take care of locking.
 736  */
 737 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 738 {
 739         struct dwc3             *dwc = dep->dwc;
 740         u32                     reg;
 741
 742         trace_dwc3_gadget_ep_disable(dep);
 743
 744         dwc3_remove_requests(dwc, dep);
 745
 746         /* make sure HW endpoint isn't stalled */
 747         if (dep->flags & DWC3_EP_STALL)
 748                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 749
 750         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 751         reg &= ~DWC3_DALEPENA_EP(dep->number);
 752         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 753
 754         dep->stream_capable = false;
 755         dep->type = 0;
 756         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
 757
 758         /* Clear out the ep descriptors for non-ep0 */
 759         if (dep->number > 1) {
 760                 dep->endpoint.comp_desc = NULL;
 761                 dep->endpoint.desc = NULL;
 762         }
 763
 764         return 0;
 765 }
 766
 767 /* -------------------------------------------------------------------------- */
 768
 769 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 770                 const struct usb_endpoint_descriptor *desc)
 771 {
 772         return -EINVAL;
 773 }
 774
 775 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 776 {
 777         return -EINVAL;
 778 }
 779
 780 /* -------------------------------------------------------------------------- */
 781
 782 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 783                 const struct usb_endpoint_descriptor *desc)
 784 {
 785         struct dwc3_ep                  *dep;
 786         struct dwc3                     *dwc;
 787         unsigned long                   flags;
 788         int                             ret;
 789
 790         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 791                 pr_debug("dwc3: invalid parameters\n");
 792                 return -EINVAL;
 793         }
 794
 795         if (!desc->wMaxPacketSize) {
 796                 pr_debug("dwc3: missing wMaxPacketSize\n");
 797                 return -EINVAL;
 798         }
 799
 800         dep = to_dwc3_ep(ep);
 801         dwc = dep->dwc;
 802
 803         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 804                                         "%s is already enabled\n",
 805                                         dep->name))
 806                 return 0;
 807
 808         spin_lock_irqsave(&dwc->lock, flags);
 809         ret = __dwc3_gadget_ep_enable(dep, false, false);
 810         spin_unlock_irqrestore(&dwc->lock, flags);
 811
 812         return ret;
 813 }
 814
 815 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 816 {
 817         struct dwc3_ep                  *dep;
 818         struct dwc3                     *dwc;
 819         unsigned long                   flags;
 820         int                             ret;
 821
 822         if (!ep) {
 823                 pr_debug("dwc3: invalid parameters\n");
 824                 return -EINVAL;
 825         }
 826
 827         dep = to_dwc3_ep(ep);
 828         dwc = dep->dwc;
 829
 830         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 831                                         "%s is already disabled\n",
 832                                         dep->name))
 833                 return 0;
 834
 835         spin_lock_irqsave(&dwc->lock, flags);
 836         ret = __dwc3_gadget_ep_disable(dep);
 837         spin_unlock_irqrestore(&dwc->lock, flags);
 838
 839         return ret;
 840 }
 841
 842 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 843         gfp_t gfp_flags)
 844 {
 845         struct dwc3_request             *req;
 846         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 847
 848         req = kzalloc(sizeof(*req), gfp_flags);
 849         if (!req)
 850                 return NULL;
 851
 852         req->epnum      = dep->number;
 853         req->dep        = dep;
 854
 855         dep->allocated_requests++;
 856
 857         trace_dwc3_alloc_request(req);
 858
 859         return &req->request;
 860 }
 861
 862 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 863                 struct usb_request *request)
 864 {
 865         struct dwc3_request             *req = to_dwc3_request(request);
 866         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 867
 868         dep->allocated_requests--;
 869         trace_dwc3_free_request(req);
 870         kfree(req);
 871 }
 872
 873 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
 874
 875 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 876                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 877                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 878 {
 879         struct dwc3             *dwc = dep->dwc;
 880         struct usb_gadget       *gadget = &dwc->gadget;
 881         enum usb_device_speed   speed = gadget->speed;
 882
 883         dwc3_ep_inc_enq(dep);
 884
 885         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 886         trb->bpl = lower_32_bits(dma);
 887         trb->bph = upper_32_bits(dma);
 888
 889         switch (usb_endpoint_type(dep->endpoint.desc)) {
 890         case USB_ENDPOINT_XFER_CONTROL:
 891                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 892                 break;
 893
 894         case USB_ENDPOINT_XFER_ISOC:
 895                 if (!node) {
 896                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 897
 898                         /*
 899                          * USB Specification 2.0 Section 5.9.2 states that: "If
 900                          * there is only a single transaction in the microframe,
 901                          * only a DATA0 data packet PID is used.  If there are
 902                          * two transactions per microframe, DATA1 is used for
 903                          * the first transaction data packet and DATA0 is used
 904                          * for the second transaction data packet.  If there are
 905                          * three transactions per microframe, DATA2 is used for
 906                          * the first transaction data packet, DATA1 is used for
 907                          * the second, and DATA0 is used for the third."
 908                          *
 909                          * IOW, we should satisfy the following cases:
 910                          *
 911                          * 1) length <= maxpacket
 912                          *      - DATA0
 913                          *
 914                          * 2) maxpacket < length <= (2 * maxpacket)
 915                          *      - DATA1, DATA0
 916                          *
 917                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 918                          *      - DATA2, DATA1, DATA0
 919                          */
 920                         if (speed == USB_SPEED_HIGH) {
 921                                 struct usb_ep *ep = &dep->endpoint;
 922                                 unsigned int mult = 2;
 923                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 924
 925                                 if (length <= (2 * maxp))
 926                                         mult--;
 927
 928                                 if (length <= maxp)
 929                                         mult--;
 930
 931                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 932                         }
 933                 } else {
 934                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 935                 }
 936
 937                 /* always enable Interrupt on Missed ISOC */
 938                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 939                 break;
 940
 941         case USB_ENDPOINT_XFER_BULK:
 942         case USB_ENDPOINT_XFER_INT:
 943                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 944                 break;
 945         default:
 946                 /*
 947                  * This is only possible with faulty memory because we
 948                  * checked it already :)
 949                  */
 950                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 951                                 usb_endpoint_type(dep->endpoint.desc));
 952         }
 953
 954         /* always enable Continue on Short Packet */
 955         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
 956                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
 957
 958                 if (short_not_ok)
 959                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 960         }
 961
 962         if ((!no_interrupt && !chain) ||
 963                         (dwc3_calc_trbs_left(dep) == 0))
 964                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
 965
 966         if (chain)
 967                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 968
 969         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 970                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
 971
 972         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 973
 974         trace_dwc3_prepare_trb(dep, trb);
 975 }
 976
 977 /**
 978  * dwc3_prepare_one_trb - setup one TRB from one request
 979  * @dep: endpoint for which this request is prepared
 980  * @req: dwc3_request pointer
 981  * @chain: should this TRB be chained to the next?
 982  * @node: only for isochronous endpoints. First TRB needs different type.
 983  */
 984 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 985                 struct dwc3_request *req, unsigned chain, unsigned node)
 986 {
 987         struct dwc3_trb         *trb;
 988         unsigned                length = req->request.length;
 989         unsigned                stream_id = req->request.stream_id;
 990         unsigned                short_not_ok = req->request.short_not_ok;
 991         unsigned                no_interrupt = req->request.no_interrupt;
 992         dma_addr_t              dma = req->request.dma;
 993
 994         trb = &dep->trb_pool[dep->trb_enqueue];
 995
 996         if (!req->trb) {
 997                 dwc3_gadget_move_started_request(req);
 998                 req->trb = trb;
 999                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1000                 dep->queued_requests++;
1001         }
1002
1003         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1004                         stream_id, short_not_ok, no_interrupt);
1005 }
1006
1007 /**
1008  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1009  * @dep: The endpoint with the TRB ring
1010  * @index: The index of the current TRB in the ring
1011  *
1012  * Returns the TRB prior to the one pointed to by the index. If the
1013  * index is 0, we will wrap backwards, skip the link TRB, and return
1014  * the one just before that.
1015  */
1016 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1017 {
1018         u8 tmp = index;
1019
1020         if (!tmp)
1021                 tmp = DWC3_TRB_NUM - 1;
1022
1023         return &dep->trb_pool[tmp - 1];
1024 }
1025
1026 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1027 {
1028         struct dwc3_trb         *tmp;
1029         u8                      trbs_left;
1030
1031         /*
1032          * If enqueue & dequeue are equal than it is either full or empty.
1033          *
1034          * One way to know for sure is if the TRB right before us has HWO bit
1035          * set or not. If it has, then we're definitely full and can't fit any
1036          * more transfers in our ring.
1037          */
1038         if (dep->trb_enqueue == dep->trb_dequeue) {
1039                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1040                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
1041                         return 0;
1042
1043                 return DWC3_TRB_NUM - 1;
1044         }
1045
1046         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1047         trbs_left &= (DWC3_TRB_NUM - 1);
1048
1049         if (dep->trb_dequeue < dep->trb_enqueue)
1050                 trbs_left--;
1051
1052         return trbs_left;
1053 }
1054
1055 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1056                 struct dwc3_request *req)
1057 {
1058         struct scatterlist *sg = req->sg;
1059         struct scatterlist *s;
1060         int             i;
1061
1062         for_each_sg(sg, s, req->num_pending_sgs, i) {
1063                 unsigned int length = req->request.length;
1064                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1065                 unsigned int rem = length % maxp;
1066                 unsigned chain = true;
1067
1068                 if (sg_is_last(s))
1069                         chain = false;
1070
1071                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1072                         struct dwc3     *dwc = dep->dwc;
1073                         struct dwc3_trb *trb;
1074
1075                         req->unaligned = true;
1076
1077                         /* prepare normal TRB */
1078                         dwc3_prepare_one_trb(dep, req, true, i);
1079
1080                         /* Now prepare one extra TRB to align transfer size */
1081                         trb = &dep->trb_pool[dep->trb_enqueue];
1082                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1083                                         maxp - rem, false, 0,
1084                                         req->request.stream_id,
1085                                         req->request.short_not_ok,
1086                                         req->request.no_interrupt);
1087                 } else {
1088                         dwc3_prepare_one_trb(dep, req, chain, i);
1089                 }
1090
1091                 if (!dwc3_calc_trbs_left(dep))
1092                         break;
1093         }
1094 }
1095
1096 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1097                 struct dwc3_request *req)
1098 {
1099         unsigned int length = req->request.length;
1100         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1101         unsigned int rem = length % maxp;
1102
1103         if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1104                 struct dwc3     *dwc = dep->dwc;
1105                 struct dwc3_trb *trb;
1106
1107                 req->unaligned = true;
1108
1109                 /* prepare normal TRB */
1110                 dwc3_prepare_one_trb(dep, req, true, 0);
1111
1112                 /* Now prepare one extra TRB to align transfer size */
1113                 trb = &dep->trb_pool[dep->trb_enqueue];
1114                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1115                                 false, 0, req->request.stream_id,
1116                                 req->request.short_not_ok,
1117                                 req->request.no_interrupt);
1118         } else if (req->request.zero && req->request.length &&
1119                    (IS_ALIGNED(req->request.length,dep->endpoint.maxpacket))) {
1120                 struct dwc3     *dwc = dep->dwc;
1121                 struct dwc3_trb *trb;
1122
1123                 req->zero = true;
1124
1125                 /* prepare normal TRB */
1126                 dwc3_prepare_one_trb(dep, req, true, 0);
1127
1128                 /* Now prepare one extra TRB to handle ZLP */
1129                 trb = &dep->trb_pool[dep->trb_enqueue];
1130                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1131                                 false, 0, req->request.stream_id,
1132                                 req->request.short_not_ok,
1133                                 req->request.no_interrupt);
1134         } else {
1135                 dwc3_prepare_one_trb(dep, req, false, 0);
1136         }
1137 }
1138
1139 /*
1140  * dwc3_prepare_trbs - setup TRBs from requests
1141  * @dep: endpoint for which requests are being prepared
1142  *
1143  * The function goes through the requests list and sets up TRBs for the
1144  * transfers. The function returns once there are no more TRBs available or
1145  * it runs out of requests.
1146  */
1147 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1148 {
1149         struct dwc3_request     *req, *n;
1150
1151         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1152
1153         /*
1154          * We can get in a situation where there's a request in the started list
1155          * but there weren't enough TRBs to fully kick it in the first time
1156          * around, so it has been waiting for more TRBs to be freed up.
1157          *
1158          * In that case, we should check if we have a request with pending_sgs
1159          * in the started list and prepare TRBs for that request first,
1160          * otherwise we will prepare TRBs completely out of order and that will
1161          * break things.
1162          */
1163         list_for_each_entry(req, &dep->started_list, list) {
1164                 if (req->num_pending_sgs > 0)
1165                         dwc3_prepare_one_trb_sg(dep, req);
1166
1167                 if (!dwc3_calc_trbs_left(dep))
1168                         return;
1169         }
1170
1171         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1172                 struct dwc3     *dwc = dep->dwc;
1173                 int             ret;
1174
1175                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1176                                                     dep->direction);
1177                 if (ret)
1178                         return;
1179
1180                 req->sg                 = req->request.sg;
1181                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1182
1183                 if (req->num_pending_sgs > 0)
1184                         dwc3_prepare_one_trb_sg(dep, req);
1185                 else
1186                         dwc3_prepare_one_trb_linear(dep, req);
1187
1188                 if (!dwc3_calc_trbs_left(dep))
1189                         return;
1190         }
1191 }
1192
1193 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1194 {
1195         struct dwc3_gadget_ep_cmd_params params;
1196         struct dwc3_request             *req;
1197         int                             starting;
1198         int                             ret;
1199         u32                             cmd;
1200
1201         if (!dwc3_calc_trbs_left(dep))
1202                 return 0;
1203
1204         starting = !(dep->flags & DWC3_EP_BUSY);
1205
1206         dwc3_prepare_trbs(dep);
1207         req = next_request(&dep->started_list);
1208         if (!req) {
1209                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1210                 return 0;
1211         }
1212
1213         memset(&params, 0, sizeof(params));
1214
1215         if (starting) {
1216                 params.param0 = upper_32_bits(req->trb_dma);
1217                 params.param1 = lower_32_bits(req->trb_dma);
1218                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1219
1220                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1221                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1222         } else {
1223                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1224                         DWC3_DEPCMD_PARAM(dep->resource_index);
1225         }
1226
1227         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1228         if (ret < 0) {
1229                 /*
1230                  * FIXME we need to iterate over the list of requests
1231                  * here and stop, unmap, free and del each of the linked
1232                  * requests instead of what we do now.
1233                  */
1234                 if (req->trb)
1235                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1236                 dep->queued_requests--;
1237                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1238                 return ret;
1239         }
1240
1241         dep->flags |= DWC3_EP_BUSY;
1242
1243         if (starting) {
1244                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
1245                 WARN_ON_ONCE(!dep->resource_index);
1246         }
1247
1248         return 0;
1249 }
1250
1251 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1252 {
1253         u32                     reg;
1254
1255         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1256         return DWC3_DSTS_SOFFN(reg);
1257 }
1258
1259 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1260                 struct dwc3_ep *dep, u32 cur_uf)
1261 {
1262         if (list_empty(&dep->pending_list)) {
1263                 dev_info(dwc->dev, "%s: ran out of requests\n",
1264                                 dep->name);
1265                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1266                 return;
1267         }
1268
1269         /*
1270          * Schedule the first trb for one interval in the future or at
1271          * least 4 microframes.
1272          */
1273         dep->frame_number = cur_uf + max_t(u32, 4, dep->interval);
1274         __dwc3_gadget_kick_transfer(dep);
1275 }
1276
1277 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1278                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1279 {
1280         u32 cur_uf, mask;
1281
1282         mask = ~(dep->interval - 1);
1283         cur_uf = event->parameters & mask;
1284
1285         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1286 }
1287
1288 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1289 {
1290         struct dwc3             *dwc = dep->dwc;
1291
1292         if (!dep->endpoint.desc) {
1293                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1294                                 dep->name);
1295                 return -ESHUTDOWN;
1296         }
1297
1298         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1299                                 &req->request, req->dep->name))
1300                 return -EINVAL;
1301
1302         pm_runtime_get(dwc->dev);
1303
1304         req->request.actual     = 0;
1305         req->request.status     = -EINPROGRESS;
1306         req->direction          = dep->direction;
1307         req->epnum              = dep->number;
1308
1309         trace_dwc3_ep_queue(req);
1310
1311         list_add_tail(&req->list, &dep->pending_list);
1312
1313         /*
1314          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1315          * wait for a XferNotReady event so we will know what's the current
1316          * (micro-)frame number.
1317          *
1318          * Without this trick, we are very, very likely gonna get Bus Expiry
1319          * errors which will force us issue EndTransfer command.
1320          */
1321         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1322                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1323                         if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1324                                 dwc3_stop_active_transfer(dwc, dep->number, true);
1325                                 dep->flags = DWC3_EP_ENABLED;
1326                         } else {
1327                                 u32 cur_uf;
1328
1329                                 cur_uf = __dwc3_gadget_get_frame(dwc);
1330                                 __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1331                                 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1332                         }
1333                         return 0;
1334                 }
1335
1336                 if ((dep->flags & DWC3_EP_BUSY) &&
1337                     !(dep->flags & DWC3_EP_MISSED_ISOC))
1338                         goto out;
1339
1340                 return 0;
1341         }
1342
1343 out:
1344         return __dwc3_gadget_kick_transfer(dep);
1345 }
1346
1347 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1348         gfp_t gfp_flags)
1349 {
1350         struct dwc3_request             *req = to_dwc3_request(request);
1351         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1352         struct dwc3                     *dwc = dep->dwc;
1353
1354         unsigned long                   flags;
1355
1356         int                             ret;
1357
1358         spin_lock_irqsave(&dwc->lock, flags);
1359         ret = __dwc3_gadget_ep_queue(dep, req);
1360         spin_unlock_irqrestore(&dwc->lock, flags);
1361
1362         return ret;
1363 }
1364
1365 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1366                 struct usb_request *request)
1367 {
1368         struct dwc3_request             *req = to_dwc3_request(request);
1369         struct dwc3_request             *r = NULL;
1370
1371         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1372         struct dwc3                     *dwc = dep->dwc;
1373
1374         unsigned long                   flags;
1375         int                             ret = 0;
1376
1377         trace_dwc3_ep_dequeue(req);
1378
1379         spin_lock_irqsave(&dwc->lock, flags);
1380
1381         list_for_each_entry(r, &dep->pending_list, list) {
1382                 if (r == req)
1383                         break;
1384         }
1385
1386         if (r != req) {
1387                 list_for_each_entry(r, &dep->started_list, list) {
1388                         if (r == req)
1389                                 break;
1390                 }
1391                 if (r == req) {
1392                         /* wait until it is processed */
1393                         dwc3_stop_active_transfer(dwc, dep->number, true);
1394
1395                         /*
1396                          * If request was already started, this means we had to
1397                          * stop the transfer. With that we also need to ignore
1398                          * all TRBs used by the request, however TRBs can only
1399                          * be modified after completion of END_TRANSFER
1400                          * command. So what we do here is that we wait for
1401                          * END_TRANSFER completion and only after that, we jump
1402                          * over TRBs by clearing HWO and incrementing dequeue
1403                          * pointer.
1404                          *
1405                          * Note that we have 2 possible types of transfers here:
1406                          *
1407                          * i) Linear buffer request
1408                          * ii) SG-list based request
1409                          *
1410                          * SG-list based requests will have r->num_pending_sgs
1411                          * set to a valid number (> 0). Linear requests,
1412                          * normally use a single TRB.
1413                          *
1414                          * For each of these two cases, if r->unaligned flag is
1415                          * set, one extra TRB has been used to align transfer
1416                          * size to wMaxPacketSize.
1417                          *
1418                          * All of these cases need to be taken into
1419                          * consideration so we don't mess up our TRB ring
1420                          * pointers.
1421                          */
1422                         wait_event_lock_irq(dep->wait_end_transfer,
1423                                         !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1424                                         dwc->lock);
1425
1426                         if (!r->trb)
1427                                 goto out0;
1428
1429                         if (r->num_pending_sgs) {
1430                                 struct dwc3_trb *trb;
1431                                 int i = 0;
1432
1433                                 for (i = 0; i < r->num_pending_sgs; i++) {
1434                                         trb = r->trb + i;
1435                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1436                                         dwc3_ep_inc_deq(dep);
1437                                 }
1438
1439                                 if (r->unaligned || r->zero) {
1440                                         trb = r->trb + r->num_pending_sgs + 1;
1441                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1442                                         dwc3_ep_inc_deq(dep);
1443                                 }
1444                         } else {
1445                                 struct dwc3_trb *trb = r->trb;
1446
1447                                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1448                                 dwc3_ep_inc_deq(dep);
1449
1450                                 if (r->unaligned || r->zero) {
1451                                         trb = r->trb + 1;
1452                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1453                                         dwc3_ep_inc_deq(dep);
1454                                 }
1455                         }
1456                         goto out1;
1457                 }
1458                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1459                                 request, ep->name);
1460                 ret = -EINVAL;
1461                 goto out0;
1462         }
1463
1464 out1:
1465         /* giveback the request */
1466         dep->queued_requests--;
1467         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1468
1469 out0:
1470         spin_unlock_irqrestore(&dwc->lock, flags);
1471
1472         return ret;
1473 }
1474
1475 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1476 {
1477         struct dwc3_gadget_ep_cmd_params        params;
1478         struct dwc3                             *dwc = dep->dwc;
1479         int                                     ret;
1480
1481         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1482                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1483                 return -EINVAL;
1484         }
1485
1486         memset(&params, 0x00, sizeof(params));
1487
1488         if (value) {
1489                 struct dwc3_trb *trb;
1490
1491                 unsigned transfer_in_flight;
1492                 unsigned started;
1493
1494                 if (dep->flags & DWC3_EP_STALL)
1495                         return 0;
1496
1497                 if (dep->number > 1)
1498                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1499                 else
1500                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1501
1502                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1503                 started = !list_empty(&dep->started_list);
1504
1505                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1506                                 (!dep->direction && started))) {
1507                         return -EAGAIN;
1508                 }
1509
1510                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1511                                 &params);
1512                 if (ret)
1513                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1514                                         dep->name);
1515                 else
1516                         dep->flags |= DWC3_EP_STALL;
1517         } else {
1518                 if (!(dep->flags & DWC3_EP_STALL))
1519                         return 0;
1520
1521                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1522                 if (ret)
1523                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1524                                         dep->name);
1525                 else
1526                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1527         }
1528
1529         return ret;
1530 }
1531
1532 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1533 {
1534         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1535         struct dwc3                     *dwc = dep->dwc;
1536
1537         unsigned long                   flags;
1538
1539         int                             ret;
1540
1541         spin_lock_irqsave(&dwc->lock, flags);
1542         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1543         spin_unlock_irqrestore(&dwc->lock, flags);
1544
1545         return ret;
1546 }
1547
1548 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1549 {
1550         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1551         struct dwc3                     *dwc = dep->dwc;
1552         unsigned long                   flags;
1553         int                             ret;
1554
1555         spin_lock_irqsave(&dwc->lock, flags);
1556         dep->flags |= DWC3_EP_WEDGE;
1557
1558         if (dep->number == 0 || dep->number == 1)
1559                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1560         else
1561                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1562         spin_unlock_irqrestore(&dwc->lock, flags);
1563
1564         return ret;
1565 }
1566
1567 /* -------------------------------------------------------------------------- */
1568
1569 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1570         .bLength        = USB_DT_ENDPOINT_SIZE,
1571         .bDescriptorType = USB_DT_ENDPOINT,
1572         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1573 };
1574
1575 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1576         .enable         = dwc3_gadget_ep0_enable,
1577         .disable        = dwc3_gadget_ep0_disable,
1578         .alloc_request  = dwc3_gadget_ep_alloc_request,
1579         .free_request   = dwc3_gadget_ep_free_request,
1580         .queue          = dwc3_gadget_ep0_queue,
1581         .dequeue        = dwc3_gadget_ep_dequeue,
1582         .set_halt       = dwc3_gadget_ep0_set_halt,
1583         .set_wedge      = dwc3_gadget_ep_set_wedge,
1584 };
1585
1586 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1587         .enable         = dwc3_gadget_ep_enable,
1588         .disable        = dwc3_gadget_ep_disable,
1589         .alloc_request  = dwc3_gadget_ep_alloc_request,
1590         .free_request   = dwc3_gadget_ep_free_request,
1591         .queue          = dwc3_gadget_ep_queue,
1592         .dequeue        = dwc3_gadget_ep_dequeue,
1593         .set_halt       = dwc3_gadget_ep_set_halt,
1594         .set_wedge      = dwc3_gadget_ep_set_wedge,
1595 };
1596
1597 /* -------------------------------------------------------------------------- */
1598
1599 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1600 {
1601         struct dwc3             *dwc = gadget_to_dwc(g);
1602
1603         return __dwc3_gadget_get_frame(dwc);
1604 }
1605
1606 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1607 {
1608         int                     retries;
1609
1610         int                     ret;
1611         u32                     reg;
1612
1613         u8                      link_state;
1614         u8                      speed;
1615
1616         /*
1617          * According to the Databook Remote wakeup request should
1618          * be issued only when the device is in early suspend state.
1619          *
1620          * We can check that via USB Link State bits in DSTS register.
1621          */
1622         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1623
1624         speed = reg & DWC3_DSTS_CONNECTSPD;
1625         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1626             (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1627                 return 0;
1628
1629         link_state = DWC3_DSTS_USBLNKST(reg);
1630
1631         switch (link_state) {
1632         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1633         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1634                 break;
1635         default:
1636                 return -EINVAL;
1637         }
1638
1639         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1640         if (ret < 0) {
1641                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1642                 return ret;
1643         }
1644
1645         /* Recent versions do this automatically */
1646         if (dwc->revision < DWC3_REVISION_194A) {
1647                 /* write zeroes to Link Change Request */
1648                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1649                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1650                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1651         }
1652
1653         /* poll until Link State changes to ON */
1654         retries = 20000;
1655
1656         while (retries--) {
1657                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1658
1659                 /* in HS, means ON */
1660                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1661                         break;
1662         }
1663
1664         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1665                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1666                 return -EINVAL;
1667         }
1668
1669         return 0;
1670 }
1671
1672 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1673 {
1674         struct dwc3             *dwc = gadget_to_dwc(g);
1675         unsigned long           flags;
1676         int                     ret;
1677
1678         spin_lock_irqsave(&dwc->lock, flags);
1679         ret = __dwc3_gadget_wakeup(dwc);
1680         spin_unlock_irqrestore(&dwc->lock, flags);
1681
1682         return ret;
1683 }
1684
1685 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1686                 int is_selfpowered)
1687 {
1688         struct dwc3             *dwc = gadget_to_dwc(g);
1689         unsigned long           flags;
1690
1691         spin_lock_irqsave(&dwc->lock, flags);
1692         g->is_selfpowered = !!is_selfpowered;
1693         spin_unlock_irqrestore(&dwc->lock, flags);
1694
1695         return 0;
1696 }
1697
1698 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1699 {
1700         u32                     reg;
1701         u32                     timeout = 500;
1702
1703         if (pm_runtime_suspended(dwc->dev))
1704                 return 0;
1705
1706         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1707         if (is_on) {
1708                 if (dwc->revision <= DWC3_REVISION_187A) {
1709                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1710                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1711                 }
1712
1713                 if (dwc->revision >= DWC3_REVISION_194A)
1714                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1715                 reg |= DWC3_DCTL_RUN_STOP;
1716
1717                 if (dwc->has_hibernation)
1718                         reg |= DWC3_DCTL_KEEP_CONNECT;
1719
1720                 dwc->pullups_connected = true;
1721         } else {
1722                 reg &= ~DWC3_DCTL_RUN_STOP;
1723
1724                 if (dwc->has_hibernation && !suspend)
1725                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1726
1727                 dwc->pullups_connected = false;
1728         }
1729
1730         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1731
1732         do {
1733                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1734                 reg &= DWC3_DSTS_DEVCTRLHLT;
1735         } while (--timeout && !(!is_on ^ !reg));
1736
1737         if (!timeout)
1738                 return -ETIMEDOUT;
1739
1740         return 0;
1741 }
1742
1743 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1744 {
1745         struct dwc3             *dwc = gadget_to_dwc(g);
1746         unsigned long           flags;
1747         int                     ret;
1748
1749         is_on = !!is_on;
1750
1751         /*
1752          * Per databook, when we want to stop the gadget, if a control transfer
1753          * is still in process, complete it and get the core into setup phase.
1754          */
1755         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1756                 reinit_completion(&dwc->ep0_in_setup);
1757
1758                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1759                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1760                 if (ret == 0) {
1761                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1762                         return -ETIMEDOUT;
1763                 }
1764         }
1765
1766         spin_lock_irqsave(&dwc->lock, flags);
1767         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1768         spin_unlock_irqrestore(&dwc->lock, flags);
1769
1770         return ret;
1771 }
1772
1773 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1774 {
1775         u32                     reg;
1776
1777         /* Enable all but Start and End of Frame IRQs */
1778         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1779                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1780                         DWC3_DEVTEN_CMDCMPLTEN |
1781                         DWC3_DEVTEN_ERRTICERREN |
1782                         DWC3_DEVTEN_WKUPEVTEN |
1783                         DWC3_DEVTEN_CONNECTDONEEN |
1784                         DWC3_DEVTEN_USBRSTEN |
1785                         DWC3_DEVTEN_DISCONNEVTEN);
1786
1787         if (dwc->revision < DWC3_REVISION_250A)
1788                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1789
1790         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1791 }
1792
1793 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1794 {
1795         /* mask all interrupts */
1796         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1797 }
1798
1799 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1800 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1801
1802 /**
1803  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1804  * @dwc: pointer to our context structure
1805  *
1806  * The following looks like complex but it's actually very simple. In order to
1807  * calculate the number of packets we can burst at once on OUT transfers, we're
1808  * gonna use RxFIFO size.
1809  *
1810  * To calculate RxFIFO size we need two numbers:
1811  * MDWIDTH = size, in bits, of the internal memory bus
1812  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1813  *
1814  * Given these two numbers, the formula is simple:
1815  *
1816  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1817  *
1818  * 24 bytes is for 3x SETUP packets
1819  * 16 bytes is a clock domain crossing tolerance
1820  *
1821  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1822  */
1823 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1824 {
1825         u32 ram2_depth;
1826         u32 mdwidth;
1827         u32 nump;
1828         u32 reg;
1829
1830         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1831         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1832
1833         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1834         nump = min_t(u32, nump, 16);
1835
1836         /* update NumP */
1837         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1838         reg &= ~DWC3_DCFG_NUMP_MASK;
1839         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1840         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1841 }
1842
1843 static int __dwc3_gadget_start(struct dwc3 *dwc)
1844 {
1845         struct dwc3_ep          *dep;
1846         int                     ret = 0;
1847         u32                     reg;
1848
1849         /*
1850          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1851          * the core supports IMOD, disable it.
1852          */
1853         if (dwc->imod_interval) {
1854                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1855                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1856         } else if (dwc3_has_imod(dwc)) {
1857                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1858         }
1859
1860         /*
1861          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1862          * field instead of letting dwc3 itself calculate that automatically.
1863          *
1864          * This way, we maximize the chances that we'll be able to get several
1865          * bursts of data without going through any sort of endpoint throttling.
1866          */
1867         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1868         reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1869         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1870
1871         dwc3_gadget_setup_nump(dwc);
1872
1873         /* Start with SuperSpeed Default */
1874         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1875
1876         dep = dwc->eps[0];
1877         ret = __dwc3_gadget_ep_enable(dep, false, false);
1878         if (ret) {
1879                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1880                 goto err0;
1881         }
1882
1883         dep = dwc->eps[1];
1884         ret = __dwc3_gadget_ep_enable(dep, false, false);
1885         if (ret) {
1886                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1887                 goto err1;
1888         }
1889
1890         /* begin to receive SETUP packets */
1891         dwc->ep0state = EP0_SETUP_PHASE;
1892         dwc3_ep0_out_start(dwc);
1893
1894         dwc3_gadget_enable_irq(dwc);
1895
1896         return 0;
1897
1898 err1:
1899         __dwc3_gadget_ep_disable(dwc->eps[0]);
1900
1901 err0:
1902         return ret;
1903 }
1904
1905 static int dwc3_gadget_start(struct usb_gadget *g,
1906                 struct usb_gadget_driver *driver)
1907 {
1908         struct dwc3             *dwc = gadget_to_dwc(g);
1909         unsigned long           flags;
1910         int                     ret = 0;
1911         int                     irq;
1912
1913         irq = dwc->irq_gadget;
1914         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1915                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1916         if (ret) {
1917                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1918                                 irq, ret);
1919                 goto err0;
1920         }
1921
1922         spin_lock_irqsave(&dwc->lock, flags);
1923         if (dwc->gadget_driver) {
1924                 dev_err(dwc->dev, "%s is already bound to %s\n",
1925                                 dwc->gadget.name,
1926                                 dwc->gadget_driver->driver.name);
1927                 ret = -EBUSY;
1928                 goto err1;
1929         }
1930
1931         dwc->gadget_driver      = driver;
1932
1933         if (pm_runtime_active(dwc->dev))
1934                 __dwc3_gadget_start(dwc);
1935
1936         spin_unlock_irqrestore(&dwc->lock, flags);
1937
1938         return 0;
1939
1940 err1:
1941         spin_unlock_irqrestore(&dwc->lock, flags);
1942         free_irq(irq, dwc);
1943
1944 err0:
1945         return ret;
1946 }
1947
1948 static void __dwc3_gadget_stop(struct dwc3 *dwc)
1949 {
1950         dwc3_gadget_disable_irq(dwc);
1951         __dwc3_gadget_ep_disable(dwc->eps[0]);
1952         __dwc3_gadget_ep_disable(dwc->eps[1]);
1953 }
1954
1955 static int dwc3_gadget_stop(struct usb_gadget *g)
1956 {
1957         struct dwc3             *dwc = gadget_to_dwc(g);
1958         unsigned long           flags;
1959         int                     epnum;
1960
1961         spin_lock_irqsave(&dwc->lock, flags);
1962
1963         if (pm_runtime_suspended(dwc->dev))
1964                 goto out;
1965
1966         __dwc3_gadget_stop(dwc);
1967
1968         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1969                 struct dwc3_ep  *dep = dwc->eps[epnum];
1970
1971                 if (!dep)
1972                         continue;
1973
1974                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1975                         continue;
1976
1977                 wait_event_lock_irq(dep->wait_end_transfer,
1978                                     !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1979                                     dwc->lock);
1980         }
1981
1982 out:
1983         dwc->gadget_driver      = NULL;
1984         spin_unlock_irqrestore(&dwc->lock, flags);
1985
1986         free_irq(dwc->irq_gadget, dwc->ev_buf);
1987
1988         return 0;
1989 }
1990
1991 static void dwc3_gadget_set_speed(struct usb_gadget *g,
1992                                   enum usb_device_speed speed)
1993 {
1994         struct dwc3             *dwc = gadget_to_dwc(g);
1995         unsigned long           flags;
1996         u32                     reg;
1997
1998         spin_lock_irqsave(&dwc->lock, flags);
1999         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2000         reg &= ~(DWC3_DCFG_SPEED_MASK);
2001
2002         /*
2003          * WORKAROUND: DWC3 revision < 2.20a have an issue
2004          * which would cause metastability state on Run/Stop
2005          * bit if we try to force the IP to USB2-only mode.
2006          *
2007          * Because of that, we cannot configure the IP to any
2008          * speed other than the SuperSpeed
2009          *
2010          * Refers to:
2011          *
2012          * STAR#9000525659: Clock Domain Crossing on DCTL in
2013          * USB 2.0 Mode
2014          */
2015         if (dwc->revision < DWC3_REVISION_220A &&
2016             !dwc->dis_metastability_quirk) {
2017                 reg |= DWC3_DCFG_SUPERSPEED;
2018         } else {
2019                 switch (speed) {
2020                 case USB_SPEED_LOW:
2021                         reg |= DWC3_DCFG_LOWSPEED;
2022                         break;
2023                 case USB_SPEED_FULL:
2024                         reg |= DWC3_DCFG_FULLSPEED;
2025                         break;
2026                 case USB_SPEED_HIGH:
2027                         reg |= DWC3_DCFG_HIGHSPEED;
2028                         break;
2029                 case USB_SPEED_SUPER:
2030                         reg |= DWC3_DCFG_SUPERSPEED;
2031                         break;
2032                 case USB_SPEED_SUPER_PLUS:
2033                         reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2034                         break;
2035                 default:
2036                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2037
2038                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2039                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2040                         else
2041                                 reg |= DWC3_DCFG_SUPERSPEED;
2042                 }
2043         }
2044         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2045
2046         spin_unlock_irqrestore(&dwc->lock, flags);
2047 }
2048
2049 static const struct usb_gadget_ops dwc3_gadget_ops = {
2050         .get_frame              = dwc3_gadget_get_frame,
2051         .wakeup                 = dwc3_gadget_wakeup,
2052         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2053         .pullup                 = dwc3_gadget_pullup,
2054         .udc_start              = dwc3_gadget_start,
2055         .udc_stop               = dwc3_gadget_stop,
2056         .udc_set_speed          = dwc3_gadget_set_speed,
2057 };
2058
2059 /* -------------------------------------------------------------------------- */
2060
2061 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2062 {
2063         struct dwc3_ep                  *dep;
2064         u8                              epnum;
2065
2066         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2067
2068         for (epnum = 0; epnum < total; epnum++) {
2069                 bool                    direction = epnum & 1;
2070                 u8                      num = epnum >> 1;
2071
2072                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2073                 if (!dep)
2074                         return -ENOMEM;
2075
2076                 dep->dwc = dwc;
2077                 dep->number = epnum;
2078                 dep->direction = direction;
2079                 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2080                 dwc->eps[epnum] = dep;
2081
2082                 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2083                                 direction ? "in" : "out");
2084
2085                 dep->endpoint.name = dep->name;
2086
2087                 if (!(dep->number > 1)) {
2088                         dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2089                         dep->endpoint.comp_desc = NULL;
2090                 }
2091
2092                 spin_lock_init(&dep->lock);
2093
2094                 if (num == 0) {
2095                         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2096                         dep->endpoint.maxburst = 1;
2097                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2098                         if (!direction)
2099                                 dwc->gadget.ep0 = &dep->endpoint;
2100                 } else if (direction) {
2101                         int mdwidth;
2102                         int kbytes;
2103                         int size;
2104                         int ret;
2105
2106                         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2107                         /* MDWIDTH is represented in bits, we need it in bytes */
2108                         mdwidth /= 8;
2109
2110                         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num));
2111                         size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2112
2113                         /* FIFO Depth is in MDWDITH bytes. Multiply */
2114                         size *= mdwidth;
2115
2116                         kbytes = size / 1024;
2117                         if (kbytes == 0)
2118                                 kbytes = 1;
2119
2120                         /*
2121                          * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2122                          * internal overhead. We don't really know how these are used,
2123                          * but documentation say it exists.
2124                          */
2125                         size -= mdwidth * (kbytes + 1);
2126                         size /= kbytes;
2127
2128                         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2129
2130                         dep->endpoint.max_streams = 15;
2131                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2132                         list_add_tail(&dep->endpoint.ep_list,
2133                                         &dwc->gadget.ep_list);
2134
2135                         ret = dwc3_alloc_trb_pool(dep);
2136                         if (ret)
2137                                 return ret;
2138                 } else {
2139                         int             ret;
2140
2141                         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2142                         dep->endpoint.max_streams = 15;
2143                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2144                         list_add_tail(&dep->endpoint.ep_list,
2145                                         &dwc->gadget.ep_list);
2146
2147                         ret = dwc3_alloc_trb_pool(dep);
2148                         if (ret)
2149                                 return ret;
2150                 }
2151
2152                 if (num == 0) {
2153                         dep->endpoint.caps.type_control = true;
2154                 } else {
2155                         dep->endpoint.caps.type_iso = true;
2156                         dep->endpoint.caps.type_bulk = true;
2157                         dep->endpoint.caps.type_int = true;
2158                 }
2159
2160                 dep->endpoint.caps.dir_in = direction;
2161                 dep->endpoint.caps.dir_out = !direction;
2162
2163                 INIT_LIST_HEAD(&dep->pending_list);
2164                 INIT_LIST_HEAD(&dep->started_list);
2165         }
2166
2167         return 0;
2168 }
2169
2170 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2171 {
2172         struct dwc3_ep                  *dep;
2173         u8                              epnum;
2174
2175         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2176                 dep = dwc->eps[epnum];
2177                 if (!dep)
2178                         continue;
2179                 /*
2180                  * Physical endpoints 0 and 1 are special; they form the
2181                  * bi-directional USB endpoint 0.
2182                  *
2183                  * For those two physical endpoints, we don't allocate a TRB
2184                  * pool nor do we add them the endpoints list. Due to that, we
2185                  * shouldn't do these two operations otherwise we would end up
2186                  * with all sorts of bugs when removing dwc3.ko.
2187                  */
2188                 if (epnum != 0 && epnum != 1) {
2189                         dwc3_free_trb_pool(dep);
2190                         list_del(&dep->endpoint.ep_list);
2191                 }
2192
2193                 kfree(dep);
2194         }
2195 }
2196
2197 /* -------------------------------------------------------------------------- */
2198
2199 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
2200                 struct dwc3_request *req, struct dwc3_trb *trb,
2201                 const struct dwc3_event_depevt *event, int status,
2202                 int chain)
2203 {
2204         unsigned int            count;
2205         unsigned int            s_pkt = 0;
2206         unsigned int            trb_status;
2207
2208         dwc3_ep_inc_deq(dep);
2209
2210         if (req->trb == trb)
2211                 dep->queued_requests--;
2212
2213         trace_dwc3_complete_trb(dep, trb);
2214
2215         /*
2216          * If we're in the middle of series of chained TRBs and we
2217          * receive a short transfer along the way, DWC3 will skip
2218          * through all TRBs including the last TRB in the chain (the
2219          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2220          * bit and SW has to do it manually.
2221          *
2222          * We're going to do that here to avoid problems of HW trying
2223          * to use bogus TRBs for transfers.
2224          */
2225         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2226                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2227
2228         /*
2229          * If we're dealing with unaligned size OUT transfer, we will be left
2230          * with one TRB pending in the ring. We need to manually clear HWO bit
2231          * from that TRB.
2232          */
2233         if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2234                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2235                 return 1;
2236         }
2237
2238         count = trb->size & DWC3_TRB_SIZE_MASK;
2239         req->remaining += count;
2240
2241         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2242                 return 1;
2243
2244         if (dep->direction) {
2245                 if (count) {
2246                         trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
2247                         if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
2248                                 /*
2249                                  * If missed isoc occurred and there is
2250                                  * no request queued then issue END
2251                                  * TRANSFER, so that core generates
2252                                  * next xfernotready and we will issue
2253                                  * a fresh START TRANSFER.
2254                                  * If there are still queued request
2255                                  * then wait, do not issue either END
2256                                  * or UPDATE TRANSFER, just attach next
2257                                  * request in pending_list during
2258                                  * giveback.If any future queued request
2259                                  * is successfully transferred then we
2260                                  * will issue UPDATE TRANSFER for all
2261                                  * request in the pending_list.
2262                                  */
2263                                 dep->flags |= DWC3_EP_MISSED_ISOC;
2264                         } else {
2265                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
2266                                                 dep->name);
2267                                 status = -ECONNRESET;
2268                         }
2269                 } else {
2270                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
2271                 }
2272         } else {
2273                 if (count && (event->status & DEPEVT_STATUS_SHORT))
2274                         s_pkt = 1;
2275         }
2276
2277         if (s_pkt && !chain)
2278                 return 1;
2279
2280         if ((event->status & DEPEVT_STATUS_IOC) &&
2281                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
2282                 return 1;
2283
2284         return 0;
2285 }
2286
2287 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
2288                 const struct dwc3_event_depevt *event, int status)
2289 {
2290         struct dwc3_request     *req, *n;
2291         struct dwc3_trb         *trb;
2292         bool                    ioc = false;
2293         int                     ret = 0;
2294
2295         list_for_each_entry_safe(req, n, &dep->started_list, list) {
2296                 unsigned length;
2297                 int chain;
2298
2299                 length = req->request.length;
2300                 chain = req->num_pending_sgs > 0;
2301                 if (chain) {
2302                         struct scatterlist *sg = req->sg;
2303                         struct scatterlist *s;
2304                         unsigned int pending = req->num_pending_sgs;
2305                         unsigned int i;
2306
2307                         for_each_sg(sg, s, pending, i) {
2308                                 trb = &dep->trb_pool[dep->trb_dequeue];
2309
2310                                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2311                                         break;
2312
2313                                 req->sg = sg_next(s);
2314                                 req->num_pending_sgs--;
2315
2316                                 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2317                                                 event, status, chain);
2318                                 if (ret)
2319                                         break;
2320                         }
2321                 } else {
2322                         trb = &dep->trb_pool[dep->trb_dequeue];
2323                         ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2324                                         event, status, chain);
2325                 }
2326
2327                 if (req->unaligned || req->zero) {
2328                         trb = &dep->trb_pool[dep->trb_dequeue];
2329                         ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
2330                                         event, status, false);
2331                         req->unaligned = false;
2332                         req->zero = false;
2333                 }
2334
2335                 req->request.actual = length - req->remaining;
2336
2337                 if ((req->request.actual < length) && req->num_pending_sgs)
2338                         return __dwc3_gadget_kick_transfer(dep);
2339
2340                 dwc3_gadget_giveback(dep, req, status);
2341
2342                 if (ret) {
2343                         if ((event->status & DEPEVT_STATUS_IOC) &&
2344                             (trb->ctrl & DWC3_TRB_CTRL_IOC))
2345                                 ioc = true;
2346                         break;
2347                 }
2348         }
2349
2350         /*
2351          * Our endpoint might get disabled by another thread during
2352          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2353          * early on so DWC3_EP_BUSY flag gets cleared
2354          */
2355         if (!dep->endpoint.desc)
2356                 return 1;
2357
2358         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2359                         list_empty(&dep->started_list)) {
2360                 if (list_empty(&dep->pending_list)) {
2361                         /*
2362                          * If there is no entry in request list then do
2363                          * not issue END TRANSFER now. Just set PENDING
2364                          * flag, so that END TRANSFER is issued when an
2365                          * entry is added into request list.
2366                          */
2367                         dep->flags = DWC3_EP_PENDING_REQUEST;
2368                 } else {
2369                         dwc3_stop_active_transfer(dwc, dep->number, true);
2370                         dep->flags = DWC3_EP_ENABLED;
2371                 }
2372                 return 1;
2373         }
2374
2375         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc)
2376                 return 0;
2377
2378         return 1;
2379 }
2380
2381 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
2382                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
2383 {
2384         unsigned                status = 0;
2385         int                     clean_busy;
2386         u32                     is_xfer_complete;
2387
2388         is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
2389
2390         if (event->status & DEPEVT_STATUS_BUSERR)
2391                 status = -ECONNRESET;
2392
2393         clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
2394         if (clean_busy && (!dep->endpoint.desc || is_xfer_complete ||
2395                                 usb_endpoint_xfer_isoc(dep->endpoint.desc)))
2396                 dep->flags &= ~DWC3_EP_BUSY;
2397
2398         /*
2399          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2400          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2401          */
2402         if (dwc->revision < DWC3_REVISION_183A) {
2403                 u32             reg;
2404                 int             i;
2405
2406                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2407                         dep = dwc->eps[i];
2408
2409                         if (!(dep->flags & DWC3_EP_ENABLED))
2410                                 continue;
2411
2412                         if (!list_empty(&dep->started_list))
2413                                 return;
2414                 }
2415
2416                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2417                 reg |= dwc->u1u2;
2418                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2419
2420                 dwc->u1u2 = 0;
2421         }
2422
2423         /*
2424          * Our endpoint might get disabled by another thread during
2425          * dwc3_gadget_giveback(). If that happens, we're just gonna return 1
2426          * early on so DWC3_EP_BUSY flag gets cleared
2427          */
2428         if (!dep->endpoint.desc)
2429                 return;
2430
2431         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc))
2432                 __dwc3_gadget_kick_transfer(dep);
2433 }
2434
2435 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2436                 const struct dwc3_event_depevt *event)
2437 {
2438         struct dwc3_ep          *dep;
2439         u8                      epnum = event->endpoint_number;
2440         u8                      cmd;
2441
2442         dep = dwc->eps[epnum];
2443
2444         if (!(dep->flags & DWC3_EP_ENABLED)) {
2445                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2446                         return;
2447
2448                 /* Handle only EPCMDCMPLT when EP disabled */
2449                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2450                         return;
2451         }
2452
2453         if (epnum == 0 || epnum == 1) {
2454                 dwc3_ep0_interrupt(dwc, event);
2455                 return;
2456         }
2457
2458         switch (event->endpoint_event) {
2459         case DWC3_DEPEVT_XFERCOMPLETE:
2460                 dep->resource_index = 0;
2461
2462                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2463                         dev_err(dwc->dev, "XferComplete for Isochronous endpoint\n");
2464                         return;
2465                 }
2466
2467                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2468                 break;
2469         case DWC3_DEPEVT_XFERINPROGRESS:
2470                 dwc3_endpoint_transfer_complete(dwc, dep, event);
2471                 break;
2472         case DWC3_DEPEVT_XFERNOTREADY:
2473                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
2474                         dwc3_gadget_start_isoc(dwc, dep, event);
2475                 else
2476                         __dwc3_gadget_kick_transfer(dep);
2477
2478                 break;
2479         case DWC3_DEPEVT_STREAMEVT:
2480                 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2481                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
2482                                         dep->name);
2483                         return;
2484                 }
2485                 break;
2486         case DWC3_DEPEVT_EPCMDCMPLT:
2487                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2488
2489                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2490                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2491                         wake_up(&dep->wait_end_transfer);
2492                 }
2493                 break;
2494         case DWC3_DEPEVT_RXTXFIFOEVT:
2495                 break;
2496         }
2497 }
2498
2499 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2500 {
2501         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2502                 spin_unlock(&dwc->lock);
2503                 dwc->gadget_driver->disconnect(&dwc->gadget);
2504                 spin_lock(&dwc->lock);
2505         }
2506 }
2507
2508 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2509 {
2510         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2511                 spin_unlock(&dwc->lock);
2512                 dwc->gadget_driver->suspend(&dwc->gadget);
2513                 spin_lock(&dwc->lock);
2514         }
2515 }
2516
2517 static void dwc3_resume_gadget(struct dwc3 *dwc)
2518 {
2519         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2520                 spin_unlock(&dwc->lock);
2521                 dwc->gadget_driver->resume(&dwc->gadget);
2522                 spin_lock(&dwc->lock);
2523         }
2524 }
2525
2526 static void dwc3_reset_gadget(struct dwc3 *dwc)
2527 {
2528         if (!dwc->gadget_driver)
2529                 return;
2530
2531         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2532                 spin_unlock(&dwc->lock);
2533                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2534                 spin_lock(&dwc->lock);
2535         }
2536 }
2537
2538 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2539 {
2540         struct dwc3_ep *dep;
2541         struct dwc3_gadget_ep_cmd_params params;
2542         u32 cmd;
2543         int ret;
2544
2545         dep = dwc->eps[epnum];
2546
2547         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2548             !dep->resource_index)
2549                 return;
2550
2551         /*
2552          * NOTICE: We are violating what the Databook says about the
2553          * EndTransfer command. Ideally we would _always_ wait for the
2554          * EndTransfer Command Completion IRQ, but that's causing too
2555          * much trouble synchronizing between us and gadget driver.
2556          *
2557          * We have discussed this with the IP Provider and it was
2558          * suggested to giveback all requests here, but give HW some
2559          * extra time to synchronize with the interconnect. We're using
2560          * an arbitrary 100us delay for that.
2561          *
2562          * Note also that a similar handling was tested by Synopsys
2563          * (thanks a lot Paul) and nothing bad has come out of it.
2564          * In short, what we're doing is:
2565          *
2566          * - Issue EndTransfer WITH CMDIOC bit set
2567          * - Wait 100us
2568          *
2569          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2570          * supports a mode to work around the above limitation. The
2571          * software can poll the CMDACT bit in the DEPCMD register
2572          * after issuing a EndTransfer command. This mode is enabled
2573          * by writing GUCTL2[14]. This polling is already done in the
2574          * dwc3_send_gadget_ep_cmd() function so if the mode is
2575          * enabled, the EndTransfer command will have completed upon
2576          * returning from this function and we don't need to delay for
2577          * 100us.
2578          *
2579          * This mode is NOT available on the DWC_usb31 IP.
2580          */
2581
2582         cmd = DWC3_DEPCMD_ENDTRANSFER;
2583         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2584         cmd |= DWC3_DEPCMD_CMDIOC;
2585         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2586         memset(&params, 0, sizeof(params));
2587         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2588         WARN_ON_ONCE(ret);
2589         dep->resource_index = 0;
2590         dep->flags &= ~DWC3_EP_BUSY;
2591
2592         if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2593                 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2594                 udelay(100);
2595         }
2596 }
2597
2598 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2599 {
2600         u32 epnum;
2601
2602         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2603                 struct dwc3_ep *dep;
2604                 int ret;
2605
2606                 dep = dwc->eps[epnum];
2607                 if (!dep)
2608                         continue;
2609
2610                 if (!(dep->flags & DWC3_EP_STALL))
2611                         continue;
2612
2613                 dep->flags &= ~DWC3_EP_STALL;
2614
2615                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2616                 WARN_ON_ONCE(ret);
2617         }
2618 }
2619
2620 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2621 {
2622         int                     reg;
2623
2624         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2625         reg &= ~DWC3_DCTL_INITU1ENA;
2626         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2627
2628         reg &= ~DWC3_DCTL_INITU2ENA;
2629         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2630
2631         dwc3_disconnect_gadget(dwc);
2632
2633         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2634         dwc->setup_packet_pending = false;
2635         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2636
2637         dwc->connected = false;
2638 }
2639
2640 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2641 {
2642         u32                     reg;
2643
2644         dwc->connected = true;
2645
2646         /*
2647          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2648          * would cause a missing Disconnect Event if there's a
2649          * pending Setup Packet in the FIFO.
2650          *
2651          * There's no suggested workaround on the official Bug
2652          * report, which states that "unless the driver/application
2653          * is doing any special handling of a disconnect event,
2654          * there is no functional issue".
2655          *
2656          * Unfortunately, it turns out that we _do_ some special
2657          * handling of a disconnect event, namely complete all
2658          * pending transfers, notify gadget driver of the
2659          * disconnection, and so on.
2660          *
2661          * Our suggested workaround is to follow the Disconnect
2662          * Event steps here, instead, based on a setup_packet_pending
2663          * flag. Such flag gets set whenever we have a SETUP_PENDING
2664          * status for EP0 TRBs and gets cleared on XferComplete for the
2665          * same endpoint.
2666          *
2667          * Refers to:
2668          *
2669          * STAR#9000466709: RTL: Device : Disconnect event not
2670          * generated if setup packet pending in FIFO
2671          */
2672         if (dwc->revision < DWC3_REVISION_188A) {
2673                 if (dwc->setup_packet_pending)
2674                         dwc3_gadget_disconnect_interrupt(dwc);
2675         }
2676
2677         dwc3_reset_gadget(dwc);
2678
2679         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2680         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2681         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2682         dwc->test_mode = false;
2683         dwc3_clear_stall_all_ep(dwc);
2684
2685         /* Reset device address to zero */
2686         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2687         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2688         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2689 }
2690
2691 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2692 {
2693         struct dwc3_ep          *dep;
2694         int                     ret;
2695         u32                     reg;
2696         u8                      speed;
2697
2698         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2699         speed = reg & DWC3_DSTS_CONNECTSPD;
2700         dwc->speed = speed;
2701
2702         /*
2703          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2704          * each time on Connect Done.
2705          *
2706          * Currently we always use the reset value. If any platform
2707          * wants to set this to a different value, we need to add a
2708          * setting and update GCTL.RAMCLKSEL here.
2709          */
2710
2711         switch (speed) {
2712         case DWC3_DSTS_SUPERSPEED_PLUS:
2713                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2714                 dwc->gadget.ep0->maxpacket = 512;
2715                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2716                 break;
2717         case DWC3_DSTS_SUPERSPEED:
2718                 /*
2719                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2720                  * would cause a missing USB3 Reset event.
2721                  *
2722                  * In such situations, we should force a USB3 Reset
2723                  * event by calling our dwc3_gadget_reset_interrupt()
2724                  * routine.
2725                  *
2726                  * Refers to:
2727                  *
2728                  * STAR#9000483510: RTL: SS : USB3 reset event may
2729                  * not be generated always when the link enters poll
2730                  */
2731                 if (dwc->revision < DWC3_REVISION_190A)
2732                         dwc3_gadget_reset_interrupt(dwc);
2733
2734                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2735                 dwc->gadget.ep0->maxpacket = 512;
2736                 dwc->gadget.speed = USB_SPEED_SUPER;
2737                 break;
2738         case DWC3_DSTS_HIGHSPEED:
2739                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2740                 dwc->gadget.ep0->maxpacket = 64;
2741                 dwc->gadget.speed = USB_SPEED_HIGH;
2742                 break;
2743         case DWC3_DSTS_FULLSPEED:
2744                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2745                 dwc->gadget.ep0->maxpacket = 64;
2746                 dwc->gadget.speed = USB_SPEED_FULL;
2747                 break;
2748         case DWC3_DSTS_LOWSPEED:
2749                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2750                 dwc->gadget.ep0->maxpacket = 8;
2751                 dwc->gadget.speed = USB_SPEED_LOW;
2752                 break;
2753         }
2754
2755         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2756
2757         /* Enable USB2 LPM Capability */
2758
2759         if ((dwc->revision > DWC3_REVISION_194A) &&
2760             (speed != DWC3_DSTS_SUPERSPEED) &&
2761             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2762                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2763                 reg |= DWC3_DCFG_LPM_CAP;
2764                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2765
2766                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2767                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2768
2769                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2770
2771                 /*
2772                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2773                  * DCFG.LPMCap is set, core responses with an ACK and the
2774                  * BESL value in the LPM token is less than or equal to LPM
2775                  * NYET threshold.
2776                  */
2777                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2778                                 && dwc->has_lpm_erratum,
2779                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2780
2781                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2782                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2783
2784                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2785         } else {
2786                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2787                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2788                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2789         }
2790
2791         dep = dwc->eps[0];
2792         ret = __dwc3_gadget_ep_enable(dep, true, false);
2793         if (ret) {
2794                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2795                 return;
2796         }
2797
2798         dep = dwc->eps[1];
2799         ret = __dwc3_gadget_ep_enable(dep, true, false);
2800         if (ret) {
2801                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2802                 return;
2803         }
2804
2805         /*
2806          * Configure PHY via GUSB3PIPECTLn if required.
2807          *
2808          * Update GTXFIFOSIZn
2809          *
2810          * In both cases reset values should be sufficient.
2811          */
2812 }
2813
2814 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2815 {
2816         /*
2817          * TODO take core out of low power mode when that's
2818          * implemented.
2819          */
2820
2821         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2822                 spin_unlock(&dwc->lock);
2823                 dwc->gadget_driver->resume(&dwc->gadget);
2824                 spin_lock(&dwc->lock);
2825         }
2826 }
2827
2828 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2829                 unsigned int evtinfo)
2830 {
2831         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2832         unsigned int            pwropt;
2833
2834         /*
2835          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2836          * Hibernation mode enabled which would show up when device detects
2837          * host-initiated U3 exit.
2838          *
2839          * In that case, device will generate a Link State Change Interrupt
2840          * from U3 to RESUME which is only necessary if Hibernation is
2841          * configured in.
2842          *
2843          * There are no functional changes due to such spurious event and we
2844          * just need to ignore it.
2845          *
2846          * Refers to:
2847          *
2848          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2849          * operational mode
2850          */
2851         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2852         if ((dwc->revision < DWC3_REVISION_250A) &&
2853                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2854                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2855                                 (next == DWC3_LINK_STATE_RESUME)) {
2856                         return;
2857                 }
2858         }
2859
2860         /*
2861          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2862          * on the link partner, the USB session might do multiple entry/exit
2863          * of low power states before a transfer takes place.
2864          *
2865          * Due to this problem, we might experience lower throughput. The
2866          * suggested workaround is to disable DCTL[12:9] bits if we're
2867          * transitioning from U1/U2 to U0 and enable those bits again
2868          * after a transfer completes and there are no pending transfers
2869          * on any of the enabled endpoints.
2870          *
2871          * This is the first half of that workaround.
2872          *
2873          * Refers to:
2874          *
2875          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2876          * core send LGO_Ux entering U0
2877          */
2878         if (dwc->revision < DWC3_REVISION_183A) {
2879                 if (next == DWC3_LINK_STATE_U0) {
2880                         u32     u1u2;
2881                         u32     reg;
2882
2883                         switch (dwc->link_state) {
2884                         case DWC3_LINK_STATE_U1:
2885                         case DWC3_LINK_STATE_U2:
2886                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2887                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2888                                                 | DWC3_DCTL_ACCEPTU2ENA
2889                                                 | DWC3_DCTL_INITU1ENA
2890                                                 | DWC3_DCTL_ACCEPTU1ENA);
2891
2892                                 if (!dwc->u1u2)
2893                                         dwc->u1u2 = reg & u1u2;
2894
2895                                 reg &= ~u1u2;
2896
2897                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2898                                 break;
2899                         default:
2900                                 /* do nothing */
2901                                 break;
2902                         }
2903                 }
2904         }
2905
2906         switch (next) {
2907         case DWC3_LINK_STATE_U1:
2908                 if (dwc->speed == USB_SPEED_SUPER)
2909                         dwc3_suspend_gadget(dwc);
2910                 break;
2911         case DWC3_LINK_STATE_U2:
2912         case DWC3_LINK_STATE_U3:
2913                 dwc3_suspend_gadget(dwc);
2914                 break;
2915         case DWC3_LINK_STATE_RESUME:
2916                 dwc3_resume_gadget(dwc);
2917                 break;
2918         default:
2919                 /* do nothing */
2920                 break;
2921         }
2922
2923         dwc->link_state = next;
2924 }
2925
2926 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
2927                                           unsigned int evtinfo)
2928 {
2929         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2930
2931         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
2932                 dwc3_suspend_gadget(dwc);
2933
2934         dwc->link_state = next;
2935 }
2936
2937 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2938                 unsigned int evtinfo)
2939 {
2940         unsigned int is_ss = evtinfo & BIT(4);
2941
2942         /*
2943          * WORKAROUND: DWC3 revison 2.20a with hibernation support
2944          * have a known issue which can cause USB CV TD.9.23 to fail
2945          * randomly.
2946          *
2947          * Because of this issue, core could generate bogus hibernation
2948          * events which SW needs to ignore.
2949          *
2950          * Refers to:
2951          *
2952          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2953          * Device Fallback from SuperSpeed
2954          */
2955         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2956                 return;
2957
2958         /* enter hibernation here */
2959 }
2960
2961 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2962                 const struct dwc3_event_devt *event)
2963 {
2964         switch (event->type) {
2965         case DWC3_DEVICE_EVENT_DISCONNECT:
2966                 dwc3_gadget_disconnect_interrupt(dwc);
2967                 break;
2968         case DWC3_DEVICE_EVENT_RESET:
2969                 dwc3_gadget_reset_interrupt(dwc);
2970                 break;
2971         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2972                 dwc3_gadget_conndone_interrupt(dwc);
2973                 break;
2974         case DWC3_DEVICE_EVENT_WAKEUP:
2975                 dwc3_gadget_wakeup_interrupt(dwc);
2976                 break;
2977         case DWC3_DEVICE_EVENT_HIBER_REQ:
2978                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2979                                         "unexpected hibernation event\n"))
2980                         break;
2981
2982                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2983                 break;
2984         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2985                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2986                 break;
2987         case DWC3_DEVICE_EVENT_EOPF:
2988                 /* It changed to be suspend event for version 2.30a and above */
2989                 if (dwc->revision >= DWC3_REVISION_230A) {
2990                         /*
2991                          * Ignore suspend event until the gadget enters into
2992                          * USB_STATE_CONFIGURED state.
2993                          */
2994                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
2995                                 dwc3_gadget_suspend_interrupt(dwc,
2996                                                 event->event_info);
2997                 }
2998                 break;
2999         case DWC3_DEVICE_EVENT_SOF:
3000         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3001         case DWC3_DEVICE_EVENT_CMD_CMPL:
3002         case DWC3_DEVICE_EVENT_OVERFLOW:
3003                 break;
3004         default:
3005                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3006         }
3007 }
3008
3009 static void dwc3_process_event_entry(struct dwc3 *dwc,
3010                 const union dwc3_event *event)
3011 {
3012         trace_dwc3_event(event->raw, dwc);
3013
3014         if (!event->type.is_devspec)
3015                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3016         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3017                 dwc3_gadget_interrupt(dwc, &event->devt);
3018         else
3019                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3020 }
3021
3022 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3023 {
3024         struct dwc3 *dwc = evt->dwc;
3025         irqreturn_t ret = IRQ_NONE;
3026         int left;
3027         u32 reg;
3028
3029         left = evt->count;
3030
3031         if (!(evt->flags & DWC3_EVENT_PENDING))
3032                 return IRQ_NONE;
3033
3034         while (left > 0) {
3035                 union dwc3_event event;
3036
3037                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3038
3039                 dwc3_process_event_entry(dwc, &event);
3040
3041                 /*
3042                  * FIXME we wrap around correctly to the next entry as
3043                  * almost all entries are 4 bytes in size. There is one
3044                  * entry which has 12 bytes which is a regular entry
3045                  * followed by 8 bytes data. ATM I don't know how
3046                  * things are organized if we get next to the a
3047                  * boundary so I worry about that once we try to handle
3048                  * that.
3049                  */
3050                 evt->lpos = (evt->lpos + 4) % evt->length;
3051                 left -= 4;
3052         }
3053
3054         evt->count = 0;
3055         evt->flags &= ~DWC3_EVENT_PENDING;
3056         ret = IRQ_HANDLED;
3057
3058         /* Unmask interrupt */
3059         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3060         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3061         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3062
3063         if (dwc->imod_interval) {
3064                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3065                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3066         }
3067
3068         return ret;
3069 }
3070
3071 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3072 {
3073         struct dwc3_event_buffer *evt = _evt;
3074         struct dwc3 *dwc = evt->dwc;
3075         unsigned long flags;
3076         irqreturn_t ret = IRQ_NONE;
3077
3078         spin_lock_irqsave(&dwc->lock, flags);
3079         ret = dwc3_process_event_buf(evt);
3080         spin_unlock_irqrestore(&dwc->lock, flags);
3081
3082         return ret;
3083 }
3084
3085 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3086 {
3087         struct dwc3 *dwc = evt->dwc;
3088         u32 amount;
3089         u32 count;
3090         u32 reg;
3091
3092         if (pm_runtime_suspended(dwc->dev)) {
3093                 pm_runtime_get(dwc->dev);
3094                 disable_irq_nosync(dwc->irq_gadget);
3095                 dwc->pending_events = true;
3096                 return IRQ_HANDLED;
3097         }
3098
3099         /*
3100          * With PCIe legacy interrupt, test shows that top-half irq handler can
3101          * be called again after HW interrupt deassertion. Check if bottom-half
3102          * irq event handler completes before caching new event to prevent
3103          * losing events.
3104          */
3105         if (evt->flags & DWC3_EVENT_PENDING)
3106                 return IRQ_HANDLED;
3107
3108         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3109         count &= DWC3_GEVNTCOUNT_MASK;
3110         if (!count)
3111                 return IRQ_NONE;
3112
3113         evt->count = count;
3114         evt->flags |= DWC3_EVENT_PENDING;
3115
3116         /* Mask interrupt */
3117         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3118         reg |= DWC3_GEVNTSIZ_INTMASK;
3119         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3120
3121         amount = min(count, evt->length - evt->lpos);
3122         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3123
3124         if (amount < count)
3125                 memcpy(evt->cache, evt->buf, count - amount);
3126
3127         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3128
3129         return IRQ_WAKE_THREAD;
3130 }
3131
3132 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3133 {
3134         struct dwc3_event_buffer        *evt = _evt;
3135
3136         return dwc3_check_event_buf(evt);
3137 }
3138
3139 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3140 {
3141         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3142         int irq;
3143
3144         irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3145         if (irq > 0)
3146                 goto out;
3147
3148         if (irq == -EPROBE_DEFER)
3149                 goto out;
3150
3151         irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3152         if (irq > 0)
3153                 goto out;
3154
3155         if (irq == -EPROBE_DEFER)
3156                 goto out;
3157
3158         irq = platform_get_irq(dwc3_pdev, 0);
3159         if (irq > 0)
3160                 goto out;
3161
3162         if (irq != -EPROBE_DEFER)
3163                 dev_err(dwc->dev, "missing peripheral IRQ\n");
3164
3165         if (!irq)
3166                 irq = -EINVAL;
3167
3168 out:
3169         return irq;
3170 }
3171
3172 /**
3173  * dwc3_gadget_init - initializes gadget related registers
3174  * @dwc: pointer to our controller context structure
3175  *
3176  * Returns 0 on success otherwise negative errno.
3177  */
3178 int dwc3_gadget_init(struct dwc3 *dwc)
3179 {
3180         int ret;
3181         int irq;
3182
3183         irq = dwc3_gadget_get_irq(dwc);
3184         if (irq < 0) {
3185                 ret = irq;
3186                 goto err0;
3187         }
3188
3189         dwc->irq_gadget = irq;
3190
3191         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3192                                           sizeof(*dwc->ep0_trb) * 2,
3193                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3194         if (!dwc->ep0_trb) {
3195                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3196                 ret = -ENOMEM;
3197                 goto err0;
3198         }
3199
3200         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3201         if (!dwc->setup_buf) {
3202                 ret = -ENOMEM;
3203                 goto err1;
3204         }
3205
3206         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3207                         &dwc->bounce_addr, GFP_KERNEL);
3208         if (!dwc->bounce) {
3209                 ret = -ENOMEM;
3210                 goto err2;
3211         }
3212
3213         init_completion(&dwc->ep0_in_setup);
3214
3215         dwc->gadget.ops                 = &dwc3_gadget_ops;
3216         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3217         dwc->gadget.sg_supported        = true;
3218         dwc->gadget.name                = "dwc3-gadget";
3219         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
3220
3221         /*
3222          * FIXME We might be setting max_speed to <SUPER, however versions
3223          * <2.20a of dwc3 have an issue with metastability (documented
3224          * elsewhere in this driver) which tells us we can't set max speed to
3225          * anything lower than SUPER.
3226          *
3227          * Because gadget.max_speed is only used by composite.c and function
3228          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3229          * to happen so we avoid sending SuperSpeed Capability descriptor
3230          * together with our BOS descriptor as that could confuse host into
3231          * thinking we can handle super speed.
3232          *
3233          * Note that, in fact, we won't even support GetBOS requests when speed
3234          * is less than super speed because we don't have means, yet, to tell
3235          * composite.c that we are USB 2.0 + LPM ECN.
3236          */
3237         if (dwc->revision < DWC3_REVISION_220A &&
3238             !dwc->dis_metastability_quirk)
3239                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3240                                 dwc->revision);
3241
3242         dwc->gadget.max_speed           = dwc->maximum_speed;
3243
3244         /*
3245          * REVISIT: Here we should clear all pending IRQs to be
3246          * sure we're starting from a well known location.
3247          */
3248
3249         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3250         if (ret)
3251                 goto err3;
3252
3253         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3254         if (ret) {
3255                 dev_err(dwc->dev, "failed to register udc\n");
3256                 goto err4;
3257         }
3258
3259         return 0;
3260
3261 err4:
3262         dwc3_gadget_free_endpoints(dwc);
3263
3264 err3:
3265         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3266                         dwc->bounce_addr);
3267
3268 err2:
3269         kfree(dwc->setup_buf);
3270
3271 err1:
3272         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3273                         dwc->ep0_trb, dwc->ep0_trb_addr);
3274
3275 err0:
3276         return ret;
3277 }
3278
3279 /* -------------------------------------------------------------------------- */
3280
3281 void dwc3_gadget_exit(struct dwc3 *dwc)
3282 {
3283         usb_del_gadget_udc(&dwc->gadget);
3284         dwc3_gadget_free_endpoints(dwc);
3285         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3286                           dwc->bounce_addr);
3287         kfree(dwc->setup_buf);
3288         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3289                           dwc->ep0_trb, dwc->ep0_trb_addr);
3290 }
3291
3292 int dwc3_gadget_suspend(struct dwc3 *dwc)
3293 {
3294         if (!dwc->gadget_driver)
3295                 return 0;
3296
3297         dwc3_gadget_run_stop(dwc, false, false);
3298         dwc3_disconnect_gadget(dwc);
3299         __dwc3_gadget_stop(dwc);
3300
3301         return 0;
3302 }
3303
3304 int dwc3_gadget_resume(struct dwc3 *dwc)
3305 {
3306         int                     ret;
3307
3308         if (!dwc->gadget_driver)
3309                 return 0;
3310
3311         ret = __dwc3_gadget_start(dwc);
3312         if (ret < 0)
3313                 goto err0;
3314
3315         ret = dwc3_gadget_run_stop(dwc, true, false);
3316         if (ret < 0)
3317                 goto err1;
3318
3319         return 0;
3320
3321 err1:
3322         __dwc3_gadget_stop(dwc);
3323
3324 err0:
3325         return ret;
3326 }
3327
3328 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3329 {
3330         if (dwc->pending_events) {
3331                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3332                 dwc->pending_events = false;
3333                 enable_irq(dwc->irq_gadget);
3334         }
3335 }