drivers/gpu/host1x/hw/cdma_hw.c

   1 /*
   2  * Tegra host1x Command DMA
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/slab.h>
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22
  23 #include "../cdma.h"
  24 #include "../channel.h"
  25 #include "../dev.h"
  26 #include "../debug.h"
  27
  28 /*
  29  * Put the restart at the end of pushbuffer memory
  30  */
  31 static void push_buffer_init(struct push_buffer *pb)
  32 {
  33         *(u32 *)(pb->mapped + pb->size_bytes) = host1x_opcode_restart(0);
  34 }
  35
  36 /*
  37  * Increment timedout buffer's syncpt via CPU.
  38  */
  39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  40                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  41 {
  42         struct host1x *host1x = cdma_to_host1x(cdma);
  43         struct push_buffer *pb = &cdma->push_buffer;
  44         u32 i;
  45
  46         for (i = 0; i < syncpt_incrs; i++)
  47                 host1x_syncpt_incr(cdma->timeout.syncpt);
  48
  49         /* after CPU incr, ensure shadow is up to date */
  50         host1x_syncpt_load(cdma->timeout.syncpt);
  51
  52         /* NOP all the PB slots */
  53         while (nr_slots--) {
  54                 u32 *p = (u32 *)(pb->mapped + getptr);
  55                 *(p++) = HOST1X_OPCODE_NOP;
  56                 *(p++) = HOST1X_OPCODE_NOP;
  57                 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
  58                         &pb->phys, getptr);
  59                 getptr = (getptr + 8) & (pb->size_bytes - 1);
  60         }
  61         wmb();
  62 }
  63
  64 /*
  65  * Start channel DMA
  66  */
  67 static void cdma_start(struct host1x_cdma *cdma)
  68 {
  69         struct host1x_channel *ch = cdma_to_channel(cdma);
  70
  71         if (cdma->running)
  72                 return;
  73
  74         cdma->last_pos = cdma->push_buffer.pos;
  75
  76         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  77                          HOST1X_CHANNEL_DMACTRL);
  78
  79         /* set base, put and end pointer */
  80         host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
  81         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  82         host1x_ch_writel(ch, cdma->push_buffer.phys +
  83                          cdma->push_buffer.size_bytes + 4,
  84                          HOST1X_CHANNEL_DMAEND);
  85
  86         /* reset GET */
  87         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  88                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  89                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  90                          HOST1X_CHANNEL_DMACTRL);
  91
  92         /* start the command DMA */
  93         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  94
  95         cdma->running = true;
  96 }
  97
  98 /*
  99  * Similar to cdma_start(), but rather than starting from an idle
 100  * state (where DMA GET is set to DMA PUT), on a timeout we restore
 101  * DMA GET from an explicit value (so DMA may again be pending).
 102  */
 103 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 104 {
 105         struct host1x *host1x = cdma_to_host1x(cdma);
 106         struct host1x_channel *ch = cdma_to_channel(cdma);
 107
 108         if (cdma->running)
 109                 return;
 110
 111         cdma->last_pos = cdma->push_buffer.pos;
 112
 113         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 114                          HOST1X_CHANNEL_DMACTRL);
 115
 116         /* set base, end pointer (all of memory) */
 117         host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
 118         host1x_ch_writel(ch, cdma->push_buffer.phys +
 119                          cdma->push_buffer.size_bytes,
 120                          HOST1X_CHANNEL_DMAEND);
 121
 122         /* set GET, by loading the value in PUT (then reset GET) */
 123         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 124         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 125                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 126                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 127                          HOST1X_CHANNEL_DMACTRL);
 128
 129         dev_dbg(host1x->dev,
 130                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 131                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 132                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 133                 cdma->last_pos);
 134
 135         /* deassert GET reset and set PUT */
 136         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 137                          HOST1X_CHANNEL_DMACTRL);
 138         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 139
 140         /* start the command DMA */
 141         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 142
 143         cdma->running = true;
 144 }
 145
 146 /*
 147  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 148  */
 149 static void cdma_flush(struct host1x_cdma *cdma)
 150 {
 151         struct host1x_channel *ch = cdma_to_channel(cdma);
 152
 153         if (cdma->push_buffer.pos != cdma->last_pos) {
 154                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 155                                  HOST1X_CHANNEL_DMAPUT);
 156                 cdma->last_pos = cdma->push_buffer.pos;
 157         }
 158 }
 159
 160 static void cdma_stop(struct host1x_cdma *cdma)
 161 {
 162         struct host1x_channel *ch = cdma_to_channel(cdma);
 163
 164         mutex_lock(&cdma->lock);
 165         if (cdma->running) {
 166                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 167                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 168                                  HOST1X_CHANNEL_DMACTRL);
 169                 cdma->running = false;
 170         }
 171         mutex_unlock(&cdma->lock);
 172 }
 173
 174 /*
 175  * Stops both channel's command processor and CDMA immediately.
 176  * Also, tears down the channel and resets corresponding module.
 177  */
 178 static void cdma_freeze(struct host1x_cdma *cdma)
 179 {
 180         struct host1x *host = cdma_to_host1x(cdma);
 181         struct host1x_channel *ch = cdma_to_channel(cdma);
 182         u32 cmdproc_stop;
 183
 184         if (cdma->torndown && !cdma->running) {
 185                 dev_warn(host->dev, "Already torn down\n");
 186                 return;
 187         }
 188
 189         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 190
 191         cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 192         cmdproc_stop |= BIT(ch->id);
 193         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 194
 195         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 196                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 197                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 198                 cdma->last_pos);
 199
 200         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 201                          HOST1X_CHANNEL_DMACTRL);
 202
 203         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 204
 205         cdma->running = false;
 206         cdma->torndown = true;
 207 }
 208
 209 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 210 {
 211         struct host1x *host1x = cdma_to_host1x(cdma);
 212         struct host1x_channel *ch = cdma_to_channel(cdma);
 213         u32 cmdproc_stop;
 214
 215         dev_dbg(host1x->dev,
 216                 "resuming channel (id %d, DMAGET restart = 0x%x)\n",
 217                 ch->id, getptr);
 218
 219         cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 220         cmdproc_stop &= ~(BIT(ch->id));
 221         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 222
 223         cdma->torndown = false;
 224         cdma_timeout_restart(cdma, getptr);
 225 }
 226
 227 /*
 228  * If this timeout fires, it indicates the current sync_queue entry has
 229  * exceeded its TTL and the userctx should be timed out and remaining
 230  * submits already issued cleaned up (future submits return an error).
 231  */
 232 static void cdma_timeout_handler(struct work_struct *work)
 233 {
 234         struct host1x_cdma *cdma;
 235         struct host1x *host1x;
 236         struct host1x_channel *ch;
 237
 238         u32 syncpt_val;
 239
 240         u32 prev_cmdproc, cmdproc_stop;
 241
 242         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 243                             timeout.wq);
 244         host1x = cdma_to_host1x(cdma);
 245         ch = cdma_to_channel(cdma);
 246
 247         host1x_debug_dump(cdma_to_host1x(cdma));
 248
 249         mutex_lock(&cdma->lock);
 250
 251         if (!cdma->timeout.client) {
 252                 dev_dbg(host1x->dev,
 253                         "cdma_timeout: expired, but has no clientid\n");
 254                 mutex_unlock(&cdma->lock);
 255                 return;
 256         }
 257
 258         /* stop processing to get a clean snapshot */
 259         prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 260         cmdproc_stop = prev_cmdproc | BIT(ch->id);
 261         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 262
 263         dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
 264                 prev_cmdproc, cmdproc_stop);
 265
 266         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 267
 268         /* has buffer actually completed? */
 269         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 270                 dev_dbg(host1x->dev,
 271                         "cdma_timeout: expired, but buffer had completed\n");
 272                 /* restore */
 273                 cmdproc_stop = prev_cmdproc & ~(BIT(ch->id));
 274                 host1x_sync_writel(host1x, cmdproc_stop,
 275                                    HOST1X_SYNC_CMDPROC_STOP);
 276                 mutex_unlock(&cdma->lock);
 277                 return;
 278         }
 279
 280         dev_warn(host1x->dev, "%s: timeout: %d (%s), HW thresh %d, done %d\n",
 281                 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 282                 syncpt_val, cdma->timeout.syncpt_val);
 283
 284         /* stop HW, resetting channel/module */
 285         host1x_hw_cdma_freeze(host1x, cdma);
 286
 287         host1x_cdma_update_sync_queue(cdma, ch->dev);
 288         mutex_unlock(&cdma->lock);
 289 }
 290
 291 /*
 292  * Init timeout resources
 293  */
 294 static int cdma_timeout_init(struct host1x_cdma *cdma, u32 syncpt_id)
 295 {
 296         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 297         cdma->timeout.initialized = true;
 298
 299         return 0;
 300 }
 301
 302 /*
 303  * Clean up timeout resources
 304  */
 305 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 306 {
 307         if (cdma->timeout.initialized)
 308                 cancel_delayed_work(&cdma->timeout.wq);
 309         cdma->timeout.initialized = false;
 310 }
 311
 312 static const struct host1x_cdma_ops host1x_cdma_ops = {
 313         .start = cdma_start,
 314         .stop = cdma_stop,
 315         .flush = cdma_flush,
 316
 317         .timeout_init = cdma_timeout_init,
 318         .timeout_destroy = cdma_timeout_destroy,
 319         .freeze = cdma_freeze,
 320         .resume = cdma_resume,
 321         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 322 };
 323
 324 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 325         .init = push_buffer_init,
 326 };