arch/arm/mach-msm/dma.c

   1 /* linux/arch/arm/mach-msm/dma.c
   2  *
   3  * Copyright (C) 2007 Google, Inc.
   4  * Copyright (c) 2008-2011, Code Aurora Forum. All rights reserved.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #include <linux/clk.h>
  18 #include <linux/err.h>
  19 #include <linux/io.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/completion.h>
  22 #include <linux/module.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/slab.h>
  25 #include <mach/dma.h>
  26
  27 #define MSM_DMOV_CHANNEL_COUNT 16
  28 #define MSM_DMOV_MAX_ADMS 2
  29
  30 #define MODULE_NAME "msm_dmov"
  31
  32 enum {
  33         MSM_DMOV_PRINT_ERRORS = 1,
  34         MSM_DMOV_PRINT_IO = 2,
  35         MSM_DMOV_PRINT_FLOW = 4
  36 };
  37
  38 unsigned int msm_dmov_print_mask = MSM_DMOV_PRINT_ERRORS;
  39
  40 struct msm_dmov_conf {
  41         void __iomem *base;
  42         int channel_active;
  43         struct list_head ready_commands[MSM_DMOV_CHANNEL_COUNT];
  44         struct list_head active_commands[MSM_DMOV_CHANNEL_COUNT];
  45         spinlock_t lock;
  46         unsigned irq;
  47         struct clk *clk;
  48         struct clk *pclk;
  49 };
  50
  51 static struct msm_dmov_conf dmov_conf[MSM_DMOV_MAX_ADMS];
  52 static int nr_adms;
  53
  54 #if defined(CONFIG_ARCH_MSM7X30)
  55 #define DMOV_SD_AARM DMOV_SD2
  56 #define DMOV_SD_SIZE 0x400
  57 #elif defined(CONFIG_ARCH_MSM8X60)
  58 #define DMOV_SD_AARM DMOV_SD1
  59 #define DMOV_SD_SIZE 0x800
  60 #elif defined(CONFIG_ARCH_MSM8960)
  61 #define DMOV_SD_AARM DMOV_SD0
  62 #define DMOV_SD_SIZE 0x800
  63 #else
  64 #define DMOV_SD_AARM DMOV_SD3
  65 #define DMOV_SD_SIZE 0x400
  66 #endif
  67
  68 #define DMOV_ADDR(sd, off, ch) (((sd) * DMOV_SD_SIZE) + (off) + ((ch) << 2))
  69
  70 #define DMOV_SD0(off, ch) DMOV_ADDR(0, off, ch)
  71 #define DMOV_SD1(off, ch) DMOV_ADDR(1, off, ch)
  72 #define DMOV_SD2(off, ch) DMOV_ADDR(2, off, ch)
  73 #define DMOV_SD3(off, ch) DMOV_ADDR(3, off, ch)
  74
  75 #define DMOV_CMD_PTR(ch)      DMOV_SD_AARM(0x000, ch)
  76 #define DMOV_RSLT(ch)         DMOV_SD_AARM(0x040, ch)
  77 #define DMOV_FLUSH0(ch)       DMOV_SD_AARM(0x080, ch)
  78 #define DMOV_FLUSH1(ch)       DMOV_SD_AARM(0x0C0, ch)
  79 #define DMOV_FLUSH2(ch)       DMOV_SD_AARM(0x100, ch)
  80 #define DMOV_FLUSH3(ch)       DMOV_SD_AARM(0x140, ch)
  81 #define DMOV_FLUSH4(ch)       DMOV_SD_AARM(0x180, ch)
  82 #define DMOV_FLUSH5(ch)       DMOV_SD_AARM(0x1C0, ch)
  83 #define DMOV_STATUS(ch)       DMOV_SD_AARM(0x200, ch)
  84 #define DMOV_CONFIG(ch)       DMOV_SD_AARM(0x300, ch)
  85 #define DMOV_ISR              DMOV_SD_AARM(0x380, 0)
  86
  87 #define MSM_DMOV_DPRINTF(mask, format, args...) \
  88         do { \
  89                 if ((mask) & msm_dmov_print_mask) \
  90                         printk(KERN_ERR format, args); \
  91         } while (0)
  92 #define PRINT_ERROR(format, args...) \
  93         MSM_DMOV_DPRINTF(MSM_DMOV_PRINT_ERRORS, format, args);
  94 #define PRINT_IO(format, args...) \
  95         MSM_DMOV_DPRINTF(MSM_DMOV_PRINT_IO, format, args);
  96 #define PRINT_FLOW(format, args...) \
  97         MSM_DMOV_DPRINTF(MSM_DMOV_PRINT_FLOW, format, args);
  98
  99 static inline unsigned dmov_readl(unsigned addr, int adm)
 100 {
 101         return readl(dmov_conf[adm].base + addr);
 102 }
 103
 104 static inline void dmov_writel(unsigned val, unsigned addr, int adm)
 105 {
 106         writel(val, dmov_conf[adm].base + addr);
 107 }
 108
 109 #define DMOV_ID_TO_ADM(id)   ((id) / MSM_DMOV_CHANNEL_COUNT)
 110 #define DMOV_ID_TO_CHAN(id)   ((id) % MSM_DMOV_CHANNEL_COUNT)
 111 #define DMOV_CHAN_ADM_TO_ID(ch, adm) ((ch) + (adm) * MSM_DMOV_CHANNEL_COUNT)
 112
 113 int msm_dmov_stop_cmd(unsigned id, struct msm_dmov_cmd *cmd, int graceful)
 114 {
 115         int adm = DMOV_ID_TO_ADM(id);
 116         int ch = DMOV_ID_TO_CHAN(id);
 117
 118         if (!dmov_conf[adm].base)
 119                 return -ENODEV;
 120
 121         dmov_writel((graceful << 31), DMOV_FLUSH0(ch), adm);
 122
 123         return 0;
 124 }
 125 EXPORT_SYMBOL(msm_dmov_stop_cmd);
 126
 127 static int msm_dmov_clocks_on(int adm)
 128 {
 129         int ret = 0;
 130
 131         if (!IS_ERR(dmov_conf[adm].clk)) {
 132                 ret = clk_enable(dmov_conf[adm].clk);
 133                 if (ret)
 134                         return ret;
 135                 if (!IS_ERR(dmov_conf[adm].pclk)) {
 136                         ret = clk_enable(dmov_conf[adm].pclk);
 137                         if (ret)
 138                                 clk_disable(dmov_conf[adm].clk);
 139                 }
 140         }
 141         return ret;
 142 }
 143
 144 static void msm_dmov_clocks_off(int adm)
 145 {
 146         if (!IS_ERR(dmov_conf[adm].clk))
 147                 clk_disable(dmov_conf[adm].clk);
 148         if (!IS_ERR(dmov_conf[adm].pclk))
 149                 clk_disable(dmov_conf[adm].pclk);
 150 }
 151
 152 int msm_dmov_enqueue_cmd(unsigned id, struct msm_dmov_cmd *cmd)
 153 {
 154         unsigned long irq_flags;
 155         unsigned int status;
 156         int adm = DMOV_ID_TO_ADM(id);
 157         int ch = DMOV_ID_TO_CHAN(id);
 158
 159         if (!dmov_conf[adm].base)
 160                 return -ENODEV;
 161
 162         spin_lock_irqsave(&dmov_conf[adm].lock, irq_flags);
 163         if (!dmov_conf[adm].channel_active)
 164                 msm_dmov_clocks_on(adm);
 165         dsb();
 166         status = dmov_readl(DMOV_STATUS(ch), adm);
 167         if (list_empty(&dmov_conf[adm].ready_commands[ch]) &&
 168                 (status & DMOV_STATUS_CMD_PTR_RDY)) {
 169                 if (cmd->execute_func)
 170                         cmd->execute_func(cmd);
 171                 PRINT_IO("msm_dmov_enqueue_cmd(%d), start command, status %x\n",
 172                          id, status);
 173                 list_add_tail(&cmd->list, &dmov_conf[adm].active_commands[ch]);
 174                 if (!dmov_conf[adm].channel_active)
 175                         enable_irq(dmov_conf[adm].irq);
 176                 dmov_conf[adm].channel_active |= 1U << ch;
 177                 dmov_writel(cmd->cmdptr, DMOV_CMD_PTR(ch), adm);
 178         } else {
 179                 if (!dmov_conf[adm].channel_active)
 180                         msm_dmov_clocks_off(adm);
 181                 if (list_empty(&dmov_conf[adm].active_commands[ch]))
 182                         PRINT_ERROR("msm_dmov_enqueue_cmd(%d), error datamover "
 183                                     "stalled, status %x\n", id, status);
 184                 PRINT_IO("msm_dmov_enqueue_cmd(%d), enqueue command, status "
 185                          "%x\n", id, status);
 186                 list_add_tail(&cmd->list, &dmov_conf[adm].ready_commands[ch]);
 187         }
 188         spin_unlock_irqrestore(&dmov_conf[adm].lock, irq_flags);
 189
 190         return 0;
 191 }
 192 EXPORT_SYMBOL(msm_dmov_enqueue_cmd);
 193
 194 int msm_dmov_flush(unsigned int id)
 195 {
 196         unsigned long flags;
 197         int ch = DMOV_ID_TO_CHAN(id);
 198         int adm = DMOV_ID_TO_ADM(id);
 199
 200         if (!dmov_conf[adm].base)
 201                 return -ENODEV;
 202
 203         spin_lock_irqsave(&dmov_conf[adm].lock, flags);
 204         /* XXX not checking if flush cmd sent already */
 205         if (!list_empty(&dmov_conf[adm].active_commands[ch])) {
 206                 PRINT_IO("msm_dmov_flush(%d), send flush cmd\n", id);
 207                 dmov_writel(DMOV_FLUSH_GRACEFUL, DMOV_FLUSH0(ch), adm);
 208         }
 209         spin_unlock_irqrestore(&dmov_conf[adm].lock, flags);
 210
 211         return 0;
 212 }
 213 EXPORT_SYMBOL(msm_dmov_flush);
 214
 215 struct msm_dmov_exec_cmdptr_cmd {
 216         struct msm_dmov_cmd dmov_cmd;
 217         struct completion complete;
 218         unsigned id;
 219         unsigned int result;
 220         struct msm_dmov_errdata err;
 221 };
 222
 223 static void
 224 dmov_exec_cmdptr_complete_func(struct msm_dmov_cmd *_cmd,
 225                                unsigned int result,
 226                                struct msm_dmov_errdata *err)
 227 {
 228         struct msm_dmov_exec_cmdptr_cmd *cmd = container_of(_cmd, struct msm_dmov_exec_cmdptr_cmd, dmov_cmd);
 229         cmd->result = result;
 230         if (result != 0x80000002 && err)
 231                 memcpy(&cmd->err, err, sizeof(struct msm_dmov_errdata));
 232
 233         complete(&cmd->complete);
 234 }
 235
 236 int msm_dmov_exec_cmd(unsigned id, unsigned int cmdptr)
 237 {
 238         struct msm_dmov_exec_cmdptr_cmd cmd;
 239         int adm = DMOV_ID_TO_ADM(id);
 240
 241         if (!dmov_conf[adm].base)
 242                 return -ENODEV;
 243
 244         PRINT_FLOW("dmov_exec_cmdptr(%d, %x)\n", id, cmdptr);
 245
 246         cmd.dmov_cmd.cmdptr = cmdptr;
 247         cmd.dmov_cmd.complete_func = dmov_exec_cmdptr_complete_func;
 248         cmd.dmov_cmd.execute_func = NULL;
 249         cmd.id = id;
 250         init_completion(&cmd.complete);
 251
 252         msm_dmov_enqueue_cmd(id, &cmd.dmov_cmd);
 253         wait_for_completion(&cmd.complete);
 254
 255         if (cmd.result != 0x80000002) {
 256                 PRINT_ERROR("dmov_exec_cmdptr(%d): ERROR, result: %x\n", id, cmd.result);
 257                 PRINT_ERROR("dmov_exec_cmdptr(%d):  flush: %x %x %x %x\n",
 258                         id, cmd.err.flush[0], cmd.err.flush[1], cmd.err.flush[2], cmd.err.flush[3]);
 259                 return -EIO;
 260         }
 261         PRINT_FLOW("dmov_exec_cmdptr(%d, %x) done\n", id, cmdptr);
 262         return 0;
 263 }
 264 EXPORT_SYMBOL(msm_dmov_exec_cmd);
 265
 266 static void msm_dmov_fill_errdata(struct msm_dmov_errdata *errdata, int ch,
 267                                   int adm)
 268 {
 269         errdata->flush[0] = dmov_readl(DMOV_FLUSH0(ch), adm);
 270         errdata->flush[1] = dmov_readl(DMOV_FLUSH1(ch), adm);
 271         errdata->flush[2] = dmov_readl(DMOV_FLUSH2(ch), adm);
 272         errdata->flush[3] = dmov_readl(DMOV_FLUSH3(ch), adm);
 273         errdata->flush[4] = dmov_readl(DMOV_FLUSH4(ch), adm);
 274         errdata->flush[5] = dmov_readl(DMOV_FLUSH5(ch), adm);
 275 }
 276
 277 static int msm_dmov_irq_to_adm(unsigned irq)
 278 {
 279         int i;
 280         for (i = 0; i < nr_adms; i++)
 281                 if (dmov_conf[i].irq == irq)
 282                         return i;
 283         PRINT_ERROR("msm_dmov_irq_to_adm: can't match ADM to IRQ %d\n", irq);
 284         return -EINVAL;
 285 }
 286
 287 static irqreturn_t msm_datamover_irq_handler(int irq, void *dev_id)
 288 {
 289         unsigned int int_status, mask, id;
 290         unsigned long irq_flags;
 291         unsigned int ch;
 292         unsigned int ch_status;
 293         unsigned int ch_result;
 294         struct msm_dmov_cmd *cmd;
 295         int adm = msm_dmov_irq_to_adm(irq);
 296
 297         spin_lock_irqsave(&dmov_conf[adm].lock, irq_flags);
 298
 299         int_status = dmov_readl(DMOV_ISR, adm); /* read and clear interrupt */
 300         PRINT_FLOW("msm_datamover_irq_handler: DMOV_ISR %x\n", int_status);
 301
 302         while (int_status) {
 303                 mask = int_status & -int_status;
 304                 ch = fls(mask) - 1;
 305                 id = DMOV_CHAN_ADM_TO_ID(ch, adm);
 306                 PRINT_FLOW("msm_datamover_irq_handler %08x %08x id %d\n", int_status, mask, id);
 307                 int_status &= ~mask;
 308                 ch_status = dmov_readl(DMOV_STATUS(ch), adm);
 309                 if (!(ch_status & DMOV_STATUS_RSLT_VALID)) {
 310                         PRINT_FLOW("msm_datamover_irq_handler id %d, result not valid %x\n", id, ch_status);
 311                         continue;
 312                 }
 313                 do {
 314                         ch_result = dmov_readl(DMOV_RSLT(ch), adm);
 315                         if (list_empty(&dmov_conf[adm].active_commands[ch])) {
 316                                 PRINT_ERROR("msm_datamover_irq_handler id %d, got result "
 317                                         "with no active command, status %x, result %x\n",
 318                                         id, ch_status, ch_result);
 319                                 cmd = NULL;
 320                         } else
 321                                 cmd = list_entry(dmov_conf[adm].
 322                                         active_commands[ch].next, typeof(*cmd),
 323                                         list);
 324                         PRINT_FLOW("msm_datamover_irq_handler id %d, status %x,"
 325                                    " result %x\n", id, ch_status, ch_result);
 326                         if (ch_result & DMOV_RSLT_DONE) {
 327                                 PRINT_FLOW("msm_datamover_irq_handler id %d, status %x\n",
 328                                         id, ch_status);
 329                                 PRINT_IO("msm_datamover_irq_handler id %d, got result "
 330                                         "for %p, result %x\n", id, cmd, ch_result);
 331                                 if (cmd) {
 332                                         list_del(&cmd->list);
 333                                         dsb();
 334                                         cmd->complete_func(cmd, ch_result, NULL);
 335                                 }
 336                         }
 337                         if (ch_result & DMOV_RSLT_FLUSH) {
 338                                 struct msm_dmov_errdata errdata;
 339
 340                                 msm_dmov_fill_errdata(&errdata, ch, adm);
 341                                 PRINT_FLOW("msm_datamover_irq_handler id %d, status %x\n", id, ch_status);
 342                                 PRINT_FLOW("msm_datamover_irq_handler id %d, flush, result %x, flush0 %x\n", id, ch_result, errdata.flush[0]);
 343                                 if (cmd) {
 344                                         list_del(&cmd->list);
 345                                         dsb();
 346                                         cmd->complete_func(cmd, ch_result, &errdata);
 347                                 }
 348                         }
 349                         if (ch_result & DMOV_RSLT_ERROR) {
 350                                 struct msm_dmov_errdata errdata;
 351
 352                                 msm_dmov_fill_errdata(&errdata, ch, adm);
 353                                 PRINT_ERROR("msm_datamover_irq_handler id %d, status %x\n", id, ch_status);
 354                                 PRINT_ERROR("msm_datamover_irq_handler id %d, error, result %x, flush0 %x\n", id, ch_result, errdata.flush[0]);
 355                                 if (cmd) {
 356                                         list_del(&cmd->list);
 357                                         dsb();
 358                                         cmd->complete_func(cmd, ch_result, &errdata);
 359                                 }
 360                                 /* this does not seem to work, once we get an error */
 361                                 /* the datamover will no longer accept commands */
 362                                 dmov_writel(0, DMOV_FLUSH0(ch), adm);
 363                         }
 364                         ch_status = dmov_readl(DMOV_STATUS(ch), adm);
 365                         PRINT_FLOW("msm_datamover_irq_handler id %d, status %x\n", id, ch_status);
 366                         if ((ch_status & DMOV_STATUS_CMD_PTR_RDY) &&
 367                             !list_empty(&dmov_conf[adm].ready_commands[ch])) {
 368                                 cmd = list_entry(dmov_conf[adm].
 369                                         ready_commands[ch].next, typeof(*cmd),
 370                                         list);
 371                                 list_del(&cmd->list);
 372                                 list_add_tail(&cmd->list, &dmov_conf[adm].
 373                                               active_commands[ch]);
 374                                 if (cmd->execute_func)
 375                                         cmd->execute_func(cmd);
 376                                 PRINT_FLOW("msm_datamover_irq_handler id %d, start command\n", id);
 377                                 dmov_writel(cmd->cmdptr, DMOV_CMD_PTR(ch), adm);
 378                         }
 379                 } while (ch_status & DMOV_STATUS_RSLT_VALID);
 380                 if (list_empty(&dmov_conf[adm].active_commands[ch]) &&
 381                                 list_empty(&dmov_conf[adm].ready_commands[ch]))
 382                         dmov_conf[adm].channel_active &= ~(1U << ch);
 383                 PRINT_FLOW("msm_datamover_irq_handler id %d, status %x\n", id, ch_status);
 384         }
 385
 386         if (!dmov_conf[adm].channel_active) {
 387                 disable_irq_nosync(dmov_conf[adm].irq);
 388                 msm_dmov_clocks_off(adm);
 389         }
 390
 391         spin_unlock_irqrestore(&dmov_conf[adm].lock, irq_flags);
 392         return IRQ_HANDLED;
 393 }
 394
 395 static void __init msm_dmov_deinit_clocks(int adm)
 396 {
 397         if (!IS_ERR(dmov_conf[adm].clk))
 398                 clk_put(dmov_conf[adm].clk);
 399         if (!IS_ERR(dmov_conf[adm].pclk))
 400                 clk_put(dmov_conf[adm].pclk);
 401 }
 402
 403 #define PDEV_TO_ADM(pdev) \
 404 ({ \
 405         typeof(pdev) _pdev = pdev; \
 406         (_pdev->id == -1) ? 0 : _pdev->id; \
 407 })
 408
 409 static int __devinit msm_dmov_init_clocks(struct platform_device *pdev)
 410 {
 411         int ret = 0;
 412         int adm = PDEV_TO_ADM(pdev);
 413
 414         dmov_conf[adm].clk = clk_get(&pdev->dev, "adm_clk");
 415         if (IS_ERR(dmov_conf[adm].clk)) {
 416                 PRINT_ERROR("%s: Error getting adm_clk\n", __func__);
 417                 ret = PTR_ERR(dmov_conf[adm].clk);
 418         }
 419
 420         dmov_conf[adm].pclk = clk_get(&pdev->dev, "adm_pclk");
 421         /* pclk not present on all SoCs, don't return error on failure */
 422
 423         return ret;
 424 }
 425
 426 static int __devinit msm_dmov_conf_init(struct platform_device *pdev)
 427 {
 428         int i;
 429         int adm = PDEV_TO_ADM(pdev);
 430         struct resource *irqres =
 431                 platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 432         struct resource *memres =
 433                 platform_get_resource(pdev, IORESOURCE_MEM, 0);
 434
 435         if (!irqres || !memres || !irqres->start)
 436                 return -EINVAL;
 437
 438         dmov_conf[adm].irq = irqres->start;
 439
 440         dmov_conf[adm].base =
 441                 ioremap_nocache(memres->start, resource_size(memres));
 442         if (!dmov_conf[adm].base)
 443                 return -ENOMEM;
 444
 445         dmov_conf[adm].lock = __SPIN_LOCK_UNLOCKED(dmov_lock);
 446         for (i = 0; i < MSM_DMOV_CHANNEL_COUNT; i++) {
 447                 INIT_LIST_HEAD(&dmov_conf[adm].ready_commands[i]);
 448                 INIT_LIST_HEAD(&dmov_conf[adm].active_commands[i]);
 449         }
 450         return 0;
 451 }
 452
 453 static inline void __devinit msm_dmov_conf_free(int adm)
 454 {
 455         iounmap(dmov_conf[adm].base);
 456         dmov_conf[adm].base = NULL;
 457 }
 458
 459 static int __devinit msm_dmov_probe(struct platform_device *pdev)
 460 {
 461         int i;
 462         int ret;
 463         int adm = PDEV_TO_ADM(pdev);
 464
 465         ret = msm_dmov_conf_init(pdev);
 466         if (ret)
 467                 return ret;
 468
 469         for (i = 0; i < MSM_DMOV_CHANNEL_COUNT; i++)
 470                 dmov_writel(DMOV_CONFIG_IRQ_EN | DMOV_CONFIG_FORCE_TOP_PTR_RSLT
 471                           | DMOV_CONFIG_FORCE_FLUSH_RSLT, DMOV_CONFIG(i), adm);
 472
 473         ret = msm_dmov_init_clocks(pdev);
 474         if (ret)
 475                 goto out_conf;
 476
 477         ret = request_irq(dmov_conf[adm].irq, msm_datamover_irq_handler, 0,
 478                           "msmdatamover", NULL);
 479         if (ret)
 480                 goto out_clock;
 481         disable_irq(dmov_conf[adm].irq);
 482         nr_adms++;
 483
 484         return 0;
 485
 486 out_clock:
 487         msm_dmov_deinit_clocks(adm);
 488 out_conf:
 489         msm_dmov_conf_free(adm);
 490         return ret;
 491 }
 492
 493 static struct platform_driver msm_dmov_driver = {
 494         .probe = msm_dmov_probe,
 495         .driver = {
 496                 .name = MODULE_NAME,
 497                 .owner = THIS_MODULE,
 498         },
 499 };
 500
 501 static int __init msm_init_datamover(void)
 502 {
 503         return platform_driver_register(&msm_dmov_driver);
 504 }
 505
 506 arch_initcall(msm_init_datamover);