drivers/gpu/host1x/hw/cdma_hw.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Tegra host1x Command DMA
   4  *
   5  * Copyright (c) 2010-2013, NVIDIA Corporation.
   6  */
   7
   8 #include <linux/slab.h>
   9 #include <linux/scatterlist.h>
  10 #include <linux/dma-mapping.h>
  11
  12 #include "../cdma.h"
  13 #include "../channel.h"
  14 #include "../dev.h"
  15 #include "../debug.h"
  16
  17 /*
  18  * Put the restart at the end of pushbuffer memory
  19  */
  20 static void push_buffer_init(struct push_buffer *pb)
  21 {
  22         *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
  23 }
  24
  25 /*
  26  * Increment timedout buffer's syncpt via CPU.
  27  */
  28 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  29                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  30 {
  31         unsigned int i;
  32
  33         for (i = 0; i < syncpt_incrs; i++)
  34                 host1x_syncpt_incr(cdma->timeout.syncpt);
  35
  36         /* after CPU incr, ensure shadow is up to date */
  37         host1x_syncpt_load(cdma->timeout.syncpt);
  38 }
  39
  40 /*
  41  * Start channel DMA
  42  */
  43 static void cdma_start(struct host1x_cdma *cdma)
  44 {
  45         struct host1x_channel *ch = cdma_to_channel(cdma);
  46         u64 start, end;
  47
  48         if (cdma->running)
  49                 return;
  50
  51         cdma->last_pos = cdma->push_buffer.pos;
  52         start = cdma->push_buffer.dma;
  53         end = cdma->push_buffer.size + 4;
  54
  55         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  56                          HOST1X_CHANNEL_DMACTRL);
  57
  58         /* set base, put and end pointer */
  59         host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
  60 #if HOST1X_HW >= 6
  61         host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
  62 #endif
  63         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  64 #if HOST1X_HW >= 6
  65         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
  66 #endif
  67         host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
  68 #if HOST1X_HW >= 6
  69         host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
  70 #endif
  71
  72         /* reset GET */
  73         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  74                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  75                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  76                          HOST1X_CHANNEL_DMACTRL);
  77
  78         /* start the command DMA */
  79         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  80
  81         cdma->running = true;
  82 }
  83
  84 /*
  85  * Similar to cdma_start(), but rather than starting from an idle
  86  * state (where DMA GET is set to DMA PUT), on a timeout we restore
  87  * DMA GET from an explicit value (so DMA may again be pending).
  88  */
  89 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
  90 {
  91         struct host1x *host1x = cdma_to_host1x(cdma);
  92         struct host1x_channel *ch = cdma_to_channel(cdma);
  93         u64 start, end;
  94
  95         if (cdma->running)
  96                 return;
  97
  98         cdma->last_pos = cdma->push_buffer.pos;
  99
 100         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 101                          HOST1X_CHANNEL_DMACTRL);
 102
 103         start = cdma->push_buffer.dma;
 104         end = cdma->push_buffer.size + 4;
 105
 106         /* set base, end pointer (all of memory) */
 107         host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
 108 #if HOST1X_HW >= 6
 109         host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
 110 #endif
 111         host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
 112 #if HOST1X_HW >= 6
 113         host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
 114 #endif
 115
 116         /* set GET, by loading the value in PUT (then reset GET) */
 117         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 118         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 119                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 120                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 121                          HOST1X_CHANNEL_DMACTRL);
 122
 123         dev_dbg(host1x->dev,
 124                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 125                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 126                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 127                 cdma->last_pos);
 128
 129         /* deassert GET reset and set PUT */
 130         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 131                          HOST1X_CHANNEL_DMACTRL);
 132         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 133
 134         /* start the command DMA */
 135         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 136
 137         cdma->running = true;
 138 }
 139
 140 /*
 141  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 142  */
 143 static void cdma_flush(struct host1x_cdma *cdma)
 144 {
 145         struct host1x_channel *ch = cdma_to_channel(cdma);
 146
 147         if (cdma->push_buffer.pos != cdma->last_pos) {
 148                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 149                                  HOST1X_CHANNEL_DMAPUT);
 150                 cdma->last_pos = cdma->push_buffer.pos;
 151         }
 152 }
 153
 154 static void cdma_stop(struct host1x_cdma *cdma)
 155 {
 156         struct host1x_channel *ch = cdma_to_channel(cdma);
 157
 158         mutex_lock(&cdma->lock);
 159
 160         if (cdma->running) {
 161                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 162                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 163                                  HOST1X_CHANNEL_DMACTRL);
 164                 cdma->running = false;
 165         }
 166
 167         mutex_unlock(&cdma->lock);
 168 }
 169
 170 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
 171                                  bool stop)
 172 {
 173 #if HOST1X_HW >= 6
 174         host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
 175 #else
 176         u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 177         if (stop)
 178                 cmdproc_stop |= BIT(ch->id);
 179         else
 180                 cmdproc_stop &= ~BIT(ch->id);
 181         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 182 #endif
 183 }
 184
 185 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
 186 {
 187 #if HOST1X_HW >= 6
 188         host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
 189 #else
 190         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 191 #endif
 192 }
 193
 194 /*
 195  * Stops both channel's command processor and CDMA immediately.
 196  * Also, tears down the channel and resets corresponding module.
 197  */
 198 static void cdma_freeze(struct host1x_cdma *cdma)
 199 {
 200         struct host1x *host = cdma_to_host1x(cdma);
 201         struct host1x_channel *ch = cdma_to_channel(cdma);
 202
 203         if (cdma->torndown && !cdma->running) {
 204                 dev_warn(host->dev, "Already torn down\n");
 205                 return;
 206         }
 207
 208         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 209
 210         cdma_hw_cmdproc_stop(host, ch, true);
 211
 212         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 213                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 214                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 215                 cdma->last_pos);
 216
 217         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 218                          HOST1X_CHANNEL_DMACTRL);
 219
 220         cdma_hw_teardown(host, ch);
 221
 222         cdma->running = false;
 223         cdma->torndown = true;
 224 }
 225
 226 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 227 {
 228         struct host1x *host1x = cdma_to_host1x(cdma);
 229         struct host1x_channel *ch = cdma_to_channel(cdma);
 230
 231         dev_dbg(host1x->dev,
 232                 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 233                 ch->id, getptr);
 234
 235         cdma_hw_cmdproc_stop(host1x, ch, false);
 236
 237         cdma->torndown = false;
 238         cdma_timeout_restart(cdma, getptr);
 239 }
 240
 241 /*
 242  * If this timeout fires, it indicates the current sync_queue entry has
 243  * exceeded its TTL and the userctx should be timed out and remaining
 244  * submits already issued cleaned up (future submits return an error).
 245  */
 246 static void cdma_timeout_handler(struct work_struct *work)
 247 {
 248         u32 syncpt_val;
 249         struct host1x_cdma *cdma;
 250         struct host1x *host1x;
 251         struct host1x_channel *ch;
 252
 253         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 254                             timeout.wq);
 255         host1x = cdma_to_host1x(cdma);
 256         ch = cdma_to_channel(cdma);
 257
 258         host1x_debug_dump(cdma_to_host1x(cdma));
 259
 260         mutex_lock(&cdma->lock);
 261
 262         if (!cdma->timeout.client) {
 263                 dev_dbg(host1x->dev,
 264                         "cdma_timeout: expired, but has no clientid\n");
 265                 mutex_unlock(&cdma->lock);
 266                 return;
 267         }
 268
 269         /* stop processing to get a clean snapshot */
 270         cdma_hw_cmdproc_stop(host1x, ch, true);
 271
 272         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 273
 274         /* has buffer actually completed? */
 275         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 276                 dev_dbg(host1x->dev,
 277                         "cdma_timeout: expired, but buffer had completed\n");
 278                 /* restore */
 279                 cdma_hw_cmdproc_stop(host1x, ch, false);
 280                 mutex_unlock(&cdma->lock);
 281                 return;
 282         }
 283
 284         dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 285                  __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 286                  syncpt_val, cdma->timeout.syncpt_val);
 287
 288         /* stop HW, resetting channel/module */
 289         host1x_hw_cdma_freeze(host1x, cdma);
 290
 291         host1x_cdma_update_sync_queue(cdma, ch->dev);
 292         mutex_unlock(&cdma->lock);
 293 }
 294
 295 /*
 296  * Init timeout resources
 297  */
 298 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
 299 {
 300         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 301         cdma->timeout.initialized = true;
 302
 303         return 0;
 304 }
 305
 306 /*
 307  * Clean up timeout resources
 308  */
 309 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 310 {
 311         if (cdma->timeout.initialized)
 312                 cancel_delayed_work(&cdma->timeout.wq);
 313
 314         cdma->timeout.initialized = false;
 315 }
 316
 317 static const struct host1x_cdma_ops host1x_cdma_ops = {
 318         .start = cdma_start,
 319         .stop = cdma_stop,
 320         .flush = cdma_flush,
 321
 322         .timeout_init = cdma_timeout_init,
 323         .timeout_destroy = cdma_timeout_destroy,
 324         .freeze = cdma_freeze,
 325         .resume = cdma_resume,
 326         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 327 };
 328
 329 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 330         .init = push_buffer_init,
 331 };