drivers/spi/spi-qcom-qspi.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
   3
   4 #include <linux/clk.h>
   5 #include <linux/interconnect.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/module.h>
   9 #include <linux/of.h>
  10 #include <linux/of_platform.h>
  11 #include <linux/pm_runtime.h>
  12 #include <linux/pm_opp.h>
  13 #include <linux/spi/spi.h>
  14 #include <linux/spi/spi-mem.h>
  15
  16
  17 #define QSPI_NUM_CS             2
  18 #define QSPI_BYTES_PER_WORD     4
  19
  20 #define MSTR_CONFIG             0x0000
  21 #define FULL_CYCLE_MODE         BIT(3)
  22 #define FB_CLK_EN               BIT(4)
  23 #define PIN_HOLDN               BIT(6)
  24 #define PIN_WPN                 BIT(7)
  25 #define DMA_ENABLE              BIT(8)
  26 #define BIG_ENDIAN_MODE         BIT(9)
  27 #define SPI_MODE_MSK            0xc00
  28 #define SPI_MODE_SHFT           10
  29 #define CHIP_SELECT_NUM         BIT(12)
  30 #define SBL_EN                  BIT(13)
  31 #define LPA_BASE_MSK            0x3c000
  32 #define LPA_BASE_SHFT           14
  33 #define TX_DATA_DELAY_MSK       0xc0000
  34 #define TX_DATA_DELAY_SHFT      18
  35 #define TX_CLK_DELAY_MSK        0x300000
  36 #define TX_CLK_DELAY_SHFT       20
  37 #define TX_CS_N_DELAY_MSK       0xc00000
  38 #define TX_CS_N_DELAY_SHFT      22
  39 #define TX_DATA_OE_DELAY_MSK    0x3000000
  40 #define TX_DATA_OE_DELAY_SHFT   24
  41
  42 #define AHB_MASTER_CFG                          0x0004
  43 #define HMEM_TYPE_START_MID_TRANS_MSK           0x7
  44 #define HMEM_TYPE_START_MID_TRANS_SHFT          0
  45 #define HMEM_TYPE_LAST_TRANS_MSK                0x38
  46 #define HMEM_TYPE_LAST_TRANS_SHFT               3
  47 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK  0xc0
  48 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6
  49 #define HMEMTYPE_READ_TRANS_MSK                 0x700
  50 #define HMEMTYPE_READ_TRANS_SHFT                8
  51 #define HSHARED                                 BIT(11)
  52 #define HINNERSHARED                            BIT(12)
  53
  54 #define MSTR_INT_EN             0x000C
  55 #define MSTR_INT_STATUS         0x0010
  56 #define RESP_FIFO_UNDERRUN      BIT(0)
  57 #define RESP_FIFO_NOT_EMPTY     BIT(1)
  58 #define RESP_FIFO_RDY           BIT(2)
  59 #define HRESP_FROM_NOC_ERR      BIT(3)
  60 #define WR_FIFO_EMPTY           BIT(9)
  61 #define WR_FIFO_FULL            BIT(10)
  62 #define WR_FIFO_OVERRUN         BIT(11)
  63 #define TRANSACTION_DONE        BIT(16)
  64 #define QSPI_ERR_IRQS           (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
  65                                  WR_FIFO_OVERRUN)
  66 #define QSPI_ALL_IRQS           (QSPI_ERR_IRQS | RESP_FIFO_RDY | \
  67                                  WR_FIFO_EMPTY | WR_FIFO_FULL | \
  68                                  TRANSACTION_DONE)
  69
  70 #define PIO_XFER_CTRL           0x0014
  71 #define REQUEST_COUNT_MSK       0xffff
  72
  73 #define PIO_XFER_CFG            0x0018
  74 #define TRANSFER_DIRECTION      BIT(0)
  75 #define MULTI_IO_MODE_MSK       0xe
  76 #define MULTI_IO_MODE_SHFT      1
  77 #define TRANSFER_FRAGMENT       BIT(8)
  78 #define SDR_1BIT                1
  79 #define SDR_2BIT                2
  80 #define SDR_4BIT                3
  81 #define DDR_1BIT                5
  82 #define DDR_2BIT                6
  83 #define DDR_4BIT                7
  84 #define DMA_DESC_SINGLE_SPI     1
  85 #define DMA_DESC_DUAL_SPI       2
  86 #define DMA_DESC_QUAD_SPI       3
  87
  88 #define PIO_XFER_STATUS         0x001c
  89 #define WR_FIFO_BYTES_MSK       0xffff0000
  90 #define WR_FIFO_BYTES_SHFT      16
  91
  92 #define PIO_DATAOUT_1B          0x0020
  93 #define PIO_DATAOUT_4B          0x0024
  94
  95 #define RD_FIFO_CFG             0x0028
  96 #define CONTINUOUS_MODE         BIT(0)
  97
  98 #define RD_FIFO_STATUS  0x002c
  99 #define FIFO_EMPTY      BIT(11)
 100 #define WR_CNTS_MSK     0x7f0
 101 #define WR_CNTS_SHFT    4
 102 #define RDY_64BYTE      BIT(3)
 103 #define RDY_32BYTE      BIT(2)
 104 #define RDY_16BYTE      BIT(1)
 105 #define FIFO_RDY        BIT(0)
 106
 107 #define RD_FIFO_RESET           0x0030
 108 #define RESET_FIFO              BIT(0)
 109
 110 #define CUR_MEM_ADDR            0x0048
 111 #define HW_VERSION              0x004c
 112 #define RD_FIFO                 0x0050
 113 #define SAMPLING_CLK_CFG        0x0090
 114 #define SAMPLING_CLK_STATUS     0x0094
 115
 116
 117 enum qspi_dir {
 118         QSPI_READ,
 119         QSPI_WRITE,
 120 };
 121
 122 struct qspi_xfer {
 123         union {
 124                 const void *tx_buf;
 125                 void *rx_buf;
 126         };
 127         unsigned int rem_bytes;
 128         unsigned int buswidth;
 129         enum qspi_dir dir;
 130         bool is_last;
 131 };
 132
 133 enum qspi_clocks {
 134         QSPI_CLK_CORE,
 135         QSPI_CLK_IFACE,
 136         QSPI_NUM_CLKS
 137 };
 138
 139 struct qcom_qspi {
 140         void __iomem *base;
 141         struct device *dev;
 142         struct clk_bulk_data *clks;
 143         struct qspi_xfer xfer;
 144         struct icc_path *icc_path_cpu_to_qspi;
 145         struct opp_table *opp_table;
 146         unsigned long last_speed;
 147         /* Lock to protect data accessed by IRQs */
 148         spinlock_t lock;
 149 };
 150
 151 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
 152                                    unsigned int buswidth)
 153 {
 154         switch (buswidth) {
 155         case 1:
 156                 return SDR_1BIT << MULTI_IO_MODE_SHFT;
 157         case 2:
 158                 return SDR_2BIT << MULTI_IO_MODE_SHFT;
 159         case 4:
 160                 return SDR_4BIT << MULTI_IO_MODE_SHFT;
 161         default:
 162                 dev_warn_once(ctrl->dev,
 163                                 "Unexpected bus width: %u\n", buswidth);
 164                 return SDR_1BIT << MULTI_IO_MODE_SHFT;
 165         }
 166 }
 167
 168 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
 169 {
 170         u32 pio_xfer_cfg;
 171         const struct qspi_xfer *xfer;
 172
 173         xfer = &ctrl->xfer;
 174         pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
 175         pio_xfer_cfg &= ~TRANSFER_DIRECTION;
 176         pio_xfer_cfg |= xfer->dir;
 177         if (xfer->is_last)
 178                 pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
 179         else
 180                 pio_xfer_cfg |= TRANSFER_FRAGMENT;
 181         pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
 182         pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
 183
 184         writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
 185 }
 186
 187 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
 188 {
 189         u32 pio_xfer_ctrl;
 190
 191         pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
 192         pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
 193         pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
 194         writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
 195 }
 196
 197 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
 198 {
 199         u32 ints;
 200
 201         qcom_qspi_pio_xfer_cfg(ctrl);
 202
 203         /* Ack any previous interrupts that might be hanging around */
 204         writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
 205
 206         /* Setup new interrupts */
 207         if (ctrl->xfer.dir == QSPI_WRITE)
 208                 ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
 209         else
 210                 ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
 211         writel(ints, ctrl->base + MSTR_INT_EN);
 212
 213         /* Kick off the transfer */
 214         qcom_qspi_pio_xfer_ctrl(ctrl);
 215 }
 216
 217 static void qcom_qspi_handle_err(struct spi_master *master,
 218                                  struct spi_message *msg)
 219 {
 220         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 221         unsigned long flags;
 222
 223         spin_lock_irqsave(&ctrl->lock, flags);
 224         writel(0, ctrl->base + MSTR_INT_EN);
 225         ctrl->xfer.rem_bytes = 0;
 226         spin_unlock_irqrestore(&ctrl->lock, flags);
 227 }
 228
 229 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
 230 {
 231         int ret;
 232         unsigned int avg_bw_cpu;
 233
 234         if (speed_hz == ctrl->last_speed)
 235                 return 0;
 236
 237         /* In regular operation (SBL_EN=1) core must be 4x transfer clock */
 238         ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
 239         if (ret) {
 240                 dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
 241                 return ret;
 242         }
 243
 244         /*
 245          * Set BW quota for CPU as driver supports FIFO mode only.
 246          * We don't have explicit peak requirement so keep it equal to avg_bw.
 247          */
 248         avg_bw_cpu = Bps_to_icc(speed_hz);
 249         ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
 250         if (ret) {
 251                 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
 252                         __func__, ret);
 253                 return ret;
 254         }
 255
 256         ctrl->last_speed = speed_hz;
 257
 258         return 0;
 259 }
 260
 261 static int qcom_qspi_transfer_one(struct spi_master *master,
 262                                   struct spi_device *slv,
 263                                   struct spi_transfer *xfer)
 264 {
 265         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 266         int ret;
 267         unsigned long speed_hz;
 268         unsigned long flags;
 269
 270         speed_hz = slv->max_speed_hz;
 271         if (xfer->speed_hz)
 272                 speed_hz = xfer->speed_hz;
 273
 274         ret = qcom_qspi_set_speed(ctrl, speed_hz);
 275         if (ret)
 276                 return ret;
 277
 278         spin_lock_irqsave(&ctrl->lock, flags);
 279
 280         /* We are half duplex, so either rx or tx will be set */
 281         if (xfer->rx_buf) {
 282                 ctrl->xfer.dir = QSPI_READ;
 283                 ctrl->xfer.buswidth = xfer->rx_nbits;
 284                 ctrl->xfer.rx_buf = xfer->rx_buf;
 285         } else {
 286                 ctrl->xfer.dir = QSPI_WRITE;
 287                 ctrl->xfer.buswidth = xfer->tx_nbits;
 288                 ctrl->xfer.tx_buf = xfer->tx_buf;
 289         }
 290         ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
 291                                           &master->cur_msg->transfers);
 292         ctrl->xfer.rem_bytes = xfer->len;
 293         qcom_qspi_pio_xfer(ctrl);
 294
 295         spin_unlock_irqrestore(&ctrl->lock, flags);
 296
 297         /* We'll call spi_finalize_current_transfer() when done */
 298         return 1;
 299 }
 300
 301 static int qcom_qspi_prepare_message(struct spi_master *master,
 302                                      struct spi_message *message)
 303 {
 304         u32 mstr_cfg;
 305         struct qcom_qspi *ctrl;
 306         int tx_data_oe_delay = 1;
 307         int tx_data_delay = 1;
 308         unsigned long flags;
 309
 310         ctrl = spi_master_get_devdata(master);
 311         spin_lock_irqsave(&ctrl->lock, flags);
 312
 313         mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
 314         mstr_cfg &= ~CHIP_SELECT_NUM;
 315         if (message->spi->chip_select)
 316                 mstr_cfg |= CHIP_SELECT_NUM;
 317
 318         mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
 319         mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
 320         mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
 321         mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
 322         mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
 323         mstr_cfg &= ~DMA_ENABLE;
 324
 325         writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
 326         spin_unlock_irqrestore(&ctrl->lock, flags);
 327
 328         return 0;
 329 }
 330
 331 static irqreturn_t pio_read(struct qcom_qspi *ctrl)
 332 {
 333         u32 rd_fifo_status;
 334         u32 rd_fifo;
 335         unsigned int wr_cnts;
 336         unsigned int bytes_to_read;
 337         unsigned int words_to_read;
 338         u32 *word_buf;
 339         u8 *byte_buf;
 340         int i;
 341
 342         rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
 343
 344         if (!(rd_fifo_status & FIFO_RDY)) {
 345                 dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
 346                 return IRQ_NONE;
 347         }
 348
 349         wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
 350         wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
 351
 352         words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
 353         bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
 354
 355         if (words_to_read) {
 356                 word_buf = ctrl->xfer.rx_buf;
 357                 ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
 358                 ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
 359                 ctrl->xfer.rx_buf = word_buf + words_to_read;
 360         }
 361
 362         if (bytes_to_read) {
 363                 byte_buf = ctrl->xfer.rx_buf;
 364                 rd_fifo = readl(ctrl->base + RD_FIFO);
 365                 ctrl->xfer.rem_bytes -= bytes_to_read;
 366                 for (i = 0; i < bytes_to_read; i++)
 367                         *byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
 368                 ctrl->xfer.rx_buf = byte_buf;
 369         }
 370
 371         return IRQ_HANDLED;
 372 }
 373
 374 static irqreturn_t pio_write(struct qcom_qspi *ctrl)
 375 {
 376         const void *xfer_buf = ctrl->xfer.tx_buf;
 377         const int *word_buf;
 378         const char *byte_buf;
 379         unsigned int wr_fifo_bytes;
 380         unsigned int wr_fifo_words;
 381         unsigned int wr_size;
 382         unsigned int rem_words;
 383
 384         wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
 385         wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
 386
 387         if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
 388                 /* Process the last 1-3 bytes */
 389                 wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
 390                 ctrl->xfer.rem_bytes -= wr_size;
 391
 392                 byte_buf = xfer_buf;
 393                 while (wr_size--)
 394                         writel(*byte_buf++,
 395                                ctrl->base + PIO_DATAOUT_1B);
 396                 ctrl->xfer.tx_buf = byte_buf;
 397         } else {
 398                 /*
 399                  * Process all the whole words; to keep things simple we'll
 400                  * just wait for the next interrupt to handle the last 1-3
 401                  * bytes if we don't have an even number of words.
 402                  */
 403                 rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
 404                 wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
 405
 406                 wr_size = min(rem_words, wr_fifo_words);
 407                 ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
 408
 409                 word_buf = xfer_buf;
 410                 iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
 411                 ctrl->xfer.tx_buf = word_buf + wr_size;
 412
 413         }
 414
 415         return IRQ_HANDLED;
 416 }
 417
 418 static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
 419 {
 420         u32 int_status;
 421         struct qcom_qspi *ctrl = dev_id;
 422         irqreturn_t ret = IRQ_NONE;
 423
 424         spin_lock(&ctrl->lock);
 425
 426         int_status = readl(ctrl->base + MSTR_INT_STATUS);
 427         writel(int_status, ctrl->base + MSTR_INT_STATUS);
 428
 429         if (ctrl->xfer.dir == QSPI_WRITE) {
 430                 if (int_status & WR_FIFO_EMPTY)
 431                         ret = pio_write(ctrl);
 432         } else {
 433                 if (int_status & RESP_FIFO_RDY)
 434                         ret = pio_read(ctrl);
 435         }
 436
 437         if (int_status & QSPI_ERR_IRQS) {
 438                 if (int_status & RESP_FIFO_UNDERRUN)
 439                         dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
 440                 if (int_status & WR_FIFO_OVERRUN)
 441                         dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
 442                 if (int_status & HRESP_FROM_NOC_ERR)
 443                         dev_err(ctrl->dev, "IRQ error: NOC response error\n");
 444                 ret = IRQ_HANDLED;
 445         }
 446
 447         if (!ctrl->xfer.rem_bytes) {
 448                 writel(0, ctrl->base + MSTR_INT_EN);
 449                 spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
 450         }
 451
 452         spin_unlock(&ctrl->lock);
 453         return ret;
 454 }
 455
 456 static int qcom_qspi_probe(struct platform_device *pdev)
 457 {
 458         int ret;
 459         struct device *dev;
 460         struct spi_master *master;
 461         struct qcom_qspi *ctrl;
 462
 463         dev = &pdev->dev;
 464
 465         master = devm_spi_alloc_master(dev, sizeof(*ctrl));
 466         if (!master)
 467                 return -ENOMEM;
 468
 469         platform_set_drvdata(pdev, master);
 470
 471         ctrl = spi_master_get_devdata(master);
 472
 473         spin_lock_init(&ctrl->lock);
 474         ctrl->dev = dev;
 475         ctrl->base = devm_platform_ioremap_resource(pdev, 0);
 476         if (IS_ERR(ctrl->base))
 477                 return PTR_ERR(ctrl->base);
 478
 479         ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
 480                                   sizeof(*ctrl->clks), GFP_KERNEL);
 481         if (!ctrl->clks)
 482                 return -ENOMEM;
 483
 484         ctrl->clks[QSPI_CLK_CORE].id = "core";
 485         ctrl->clks[QSPI_CLK_IFACE].id = "iface";
 486         ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
 487         if (ret)
 488                 return ret;
 489
 490         ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
 491         if (IS_ERR(ctrl->icc_path_cpu_to_qspi))
 492                 return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
 493                                      "Failed to get cpu path\n");
 494
 495         /* Set BW vote for register access */
 496         ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
 497                                 Bps_to_icc(1000));
 498         if (ret) {
 499                 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
 500                                 __func__, ret);
 501                 return ret;
 502         }
 503
 504         ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 505         if (ret) {
 506                 dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 507                                 __func__, ret);
 508                 return ret;
 509         }
 510
 511         ret = platform_get_irq(pdev, 0);
 512         if (ret < 0)
 513                 return ret;
 514         ret = devm_request_irq(dev, ret, qcom_qspi_irq,
 515                         IRQF_TRIGGER_HIGH, dev_name(dev), ctrl);
 516         if (ret) {
 517                 dev_err(dev, "Failed to request irq %d\n", ret);
 518                 return ret;
 519         }
 520
 521         master->max_speed_hz = 300000000;
 522         master->num_chipselect = QSPI_NUM_CS;
 523         master->bus_num = -1;
 524         master->dev.of_node = pdev->dev.of_node;
 525         master->mode_bits = SPI_MODE_0 |
 526                             SPI_TX_DUAL | SPI_RX_DUAL |
 527                             SPI_TX_QUAD | SPI_RX_QUAD;
 528         master->flags = SPI_MASTER_HALF_DUPLEX;
 529         master->prepare_message = qcom_qspi_prepare_message;
 530         master->transfer_one = qcom_qspi_transfer_one;
 531         master->handle_err = qcom_qspi_handle_err;
 532         master->auto_runtime_pm = true;
 533
 534         ctrl->opp_table = dev_pm_opp_set_clkname(&pdev->dev, "core");
 535         if (IS_ERR(ctrl->opp_table))
 536                 return PTR_ERR(ctrl->opp_table);
 537         /* OPP table is optional */
 538         ret = dev_pm_opp_of_add_table(&pdev->dev);
 539         if (ret && ret != -ENODEV) {
 540                 dev_err(&pdev->dev, "invalid OPP table in device tree\n");
 541                 goto exit_probe_put_clkname;
 542         }
 543
 544         pm_runtime_use_autosuspend(dev);
 545         pm_runtime_set_autosuspend_delay(dev, 250);
 546         pm_runtime_enable(dev);
 547
 548         ret = spi_register_master(master);
 549         if (!ret)
 550                 return 0;
 551
 552         pm_runtime_disable(dev);
 553         dev_pm_opp_of_remove_table(&pdev->dev);
 554
 555 exit_probe_put_clkname:
 556         dev_pm_opp_put_clkname(ctrl->opp_table);
 557
 558         return ret;
 559 }
 560
 561 static int qcom_qspi_remove(struct platform_device *pdev)
 562 {
 563         struct spi_master *master = platform_get_drvdata(pdev);
 564         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 565
 566         /* Unregister _before_ disabling pm_runtime() so we stop transfers */
 567         spi_unregister_master(master);
 568
 569         pm_runtime_disable(&pdev->dev);
 570         dev_pm_opp_of_remove_table(&pdev->dev);
 571         dev_pm_opp_put_clkname(ctrl->opp_table);
 572
 573         return 0;
 574 }
 575
 576 static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
 577 {
 578         struct spi_master *master = dev_get_drvdata(dev);
 579         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 580         int ret;
 581
 582         /* Drop the performance state vote */
 583         dev_pm_opp_set_rate(dev, 0);
 584         clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
 585
 586         ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 587         if (ret) {
 588                 dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 589                         __func__, ret);
 590                 return ret;
 591         }
 592
 593         return 0;
 594 }
 595
 596 static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
 597 {
 598         struct spi_master *master = dev_get_drvdata(dev);
 599         struct qcom_qspi *ctrl = spi_master_get_devdata(master);
 600         int ret;
 601
 602         ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
 603         if (ret) {
 604                 dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
 605                         __func__, ret);
 606                 return ret;
 607         }
 608
 609         ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
 610         if (ret)
 611                 return ret;
 612
 613         return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
 614 }
 615
 616 static int __maybe_unused qcom_qspi_suspend(struct device *dev)
 617 {
 618         struct spi_master *master = dev_get_drvdata(dev);
 619         int ret;
 620
 621         ret = spi_master_suspend(master);
 622         if (ret)
 623                 return ret;
 624
 625         ret = pm_runtime_force_suspend(dev);
 626         if (ret)
 627                 spi_master_resume(master);
 628
 629         return ret;
 630 }
 631
 632 static int __maybe_unused qcom_qspi_resume(struct device *dev)
 633 {
 634         struct spi_master *master = dev_get_drvdata(dev);
 635         int ret;
 636
 637         ret = pm_runtime_force_resume(dev);
 638         if (ret)
 639                 return ret;
 640
 641         ret = spi_master_resume(master);
 642         if (ret)
 643                 pm_runtime_force_suspend(dev);
 644
 645         return ret;
 646 }
 647
 648 static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
 649         SET_RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
 650                            qcom_qspi_runtime_resume, NULL)
 651         SET_SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
 652 };
 653
 654 static const struct of_device_id qcom_qspi_dt_match[] = {
 655         { .compatible = "qcom,qspi-v1", },
 656         { }
 657 };
 658 MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
 659
 660 static struct platform_driver qcom_qspi_driver = {
 661         .driver = {
 662                 .name           = "qcom_qspi",
 663                 .pm             = &qcom_qspi_dev_pm_ops,
 664                 .of_match_table = qcom_qspi_dt_match,
 665         },
 666         .probe = qcom_qspi_probe,
 667         .remove = qcom_qspi_remove,
 668 };
 669 module_platform_driver(qcom_qspi_driver);
 670
 671 MODULE_DESCRIPTION("SPI driver for QSPI cores");
 672 MODULE_LICENSE("GPL v2");