drivers/gpu/drm/msm/adreno/a5xx_preempt.c

   1 /* Copyright (c) 2017 The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  */
  13
  14 #include "msm_gem.h"
  15 #include "a5xx_gpu.h"
  16
  17 /*
  18  * Try to transition the preemption state from old to new. Return
  19  * true on success or false if the original state wasn't 'old'
  20  */
  21 static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
  22                 enum preempt_state old, enum preempt_state new)
  23 {
  24         enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
  25                 old, new);
  26
  27         return (cur == old);
  28 }
  29
  30 /*
  31  * Force the preemption state to the specified state.  This is used in cases
  32  * where the current state is known and won't change
  33  */
  34 static inline void set_preempt_state(struct a5xx_gpu *gpu,
  35                 enum preempt_state new)
  36 {
  37         /*
  38          * preempt_state may be read by other cores trying to trigger a
  39          * preemption or in the interrupt handler so barriers are needed
  40          * before...
  41          */
  42         smp_mb__before_atomic();
  43         atomic_set(&gpu->preempt_state, new);
  44         /* ... and after*/
  45         smp_mb__after_atomic();
  46 }
  47
  48 /* Write the most recent wptr for the given ring into the hardware */
  49 static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
  50 {
  51         unsigned long flags;
  52         uint32_t wptr;
  53
  54         if (!ring)
  55                 return;
  56
  57         spin_lock_irqsave(&ring->lock, flags);
  58         wptr = get_wptr(ring);
  59         spin_unlock_irqrestore(&ring->lock, flags);
  60
  61         gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
  62 }
  63
  64 /* Return the highest priority ringbuffer with something in it */
  65 static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
  66 {
  67         unsigned long flags;
  68         int i;
  69
  70         for (i = 0; i < gpu->nr_rings; i++) {
  71                 bool empty;
  72                 struct msm_ringbuffer *ring = gpu->rb[i];
  73
  74                 spin_lock_irqsave(&ring->lock, flags);
  75                 empty = (get_wptr(ring) == ring->memptrs->rptr);
  76                 spin_unlock_irqrestore(&ring->lock, flags);
  77
  78                 if (!empty)
  79                         return ring;
  80         }
  81
  82         return NULL;
  83 }
  84
  85 static void a5xx_preempt_timer(struct timer_list *t)
  86 {
  87         struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
  88         struct msm_gpu *gpu = &a5xx_gpu->base.base;
  89         struct drm_device *dev = gpu->dev;
  90         struct msm_drm_private *priv = dev->dev_private;
  91
  92         if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
  93                 return;
  94
  95         dev_err(dev->dev, "%s: preemption timed out\n", gpu->name);
  96         queue_work(priv->wq, &gpu->recover_work);
  97 }
  98
  99 /* Try to trigger a preemption switch */
 100 void a5xx_preempt_trigger(struct msm_gpu *gpu)
 101 {
 102         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 103         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 104         unsigned long flags;
 105         struct msm_ringbuffer *ring;
 106
 107         if (gpu->nr_rings == 1)
 108                 return;
 109
 110         /*
 111          * Try to start preemption by moving from NONE to START. If
 112          * unsuccessful, a preemption is already in flight
 113          */
 114         if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
 115                 return;
 116
 117         /* Get the next ring to preempt to */
 118         ring = get_next_ring(gpu);
 119
 120         /*
 121          * If no ring is populated or the highest priority ring is the current
 122          * one do nothing except to update the wptr to the latest and greatest
 123          */
 124         if (!ring || (a5xx_gpu->cur_ring == ring)) {
 125                 /*
 126                  * Its possible that while a preemption request is in progress
 127                  * from an irq context, a user context trying to submit might
 128                  * fail to update the write pointer, because it determines
 129                  * that the preempt state is not PREEMPT_NONE.
 130                  *
 131                  * Close the race by introducing an intermediate
 132                  * state PREEMPT_ABORT to let the submit path
 133                  * know that the ringbuffer is not going to change
 134                  * and can safely update the write pointer.
 135                  */
 136
 137                 set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
 138                 update_wptr(gpu, a5xx_gpu->cur_ring);
 139                 set_preempt_state(a5xx_gpu, PREEMPT_NONE);
 140                 return;
 141         }
 142
 143         /* Make sure the wptr doesn't update while we're in motion */
 144         spin_lock_irqsave(&ring->lock, flags);
 145         a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
 146         spin_unlock_irqrestore(&ring->lock, flags);
 147
 148         /* Set the address of the incoming preemption record */
 149         gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
 150                 REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
 151                 a5xx_gpu->preempt_iova[ring->id]);
 152
 153         a5xx_gpu->next_ring = ring;
 154
 155         /* Start a timer to catch a stuck preemption */
 156         mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
 157
 158         /* Set the preemption state to triggered */
 159         set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
 160
 161         /* Make sure everything is written before hitting the button */
 162         wmb();
 163
 164         /* And actually start the preemption */
 165         gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
 166 }
 167
 168 void a5xx_preempt_irq(struct msm_gpu *gpu)
 169 {
 170         uint32_t status;
 171         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 172         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 173         struct drm_device *dev = gpu->dev;
 174         struct msm_drm_private *priv = dev->dev_private;
 175
 176         if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
 177                 return;
 178
 179         /* Delete the preemption watchdog timer */
 180         del_timer(&a5xx_gpu->preempt_timer);
 181
 182         /*
 183          * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
 184          * firing the interrupt, but there is a non zero chance of a hardware
 185          * condition or a software race that could set it again before we have a
 186          * chance to finish. If that happens, log and go for recovery
 187          */
 188         status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
 189         if (unlikely(status)) {
 190                 set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
 191                 dev_err(dev->dev, "%s: Preemption failed to complete\n",
 192                         gpu->name);
 193                 queue_work(priv->wq, &gpu->recover_work);
 194                 return;
 195         }
 196
 197         a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
 198         a5xx_gpu->next_ring = NULL;
 199
 200         update_wptr(gpu, a5xx_gpu->cur_ring);
 201
 202         set_preempt_state(a5xx_gpu, PREEMPT_NONE);
 203 }
 204
 205 void a5xx_preempt_hw_init(struct msm_gpu *gpu)
 206 {
 207         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 208         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 209         int i;
 210
 211         for (i = 0; i < gpu->nr_rings; i++) {
 212                 a5xx_gpu->preempt[i]->wptr = 0;
 213                 a5xx_gpu->preempt[i]->rptr = 0;
 214                 a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
 215         }
 216
 217         /* Write a 0 to signal that we aren't switching pagetables */
 218         gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
 219                 REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
 220
 221         /* Reset the preemption state */
 222         set_preempt_state(a5xx_gpu, PREEMPT_NONE);
 223
 224         /* Always come up on rb 0 */
 225         a5xx_gpu->cur_ring = gpu->rb[0];
 226 }
 227
 228 static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
 229                 struct msm_ringbuffer *ring)
 230 {
 231         struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
 232         struct msm_gpu *gpu = &adreno_gpu->base;
 233         struct a5xx_preempt_record *ptr;
 234         struct drm_gem_object *bo = NULL;
 235         u64 iova = 0;
 236
 237         ptr = msm_gem_kernel_new(gpu->dev,
 238                 A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
 239                 MSM_BO_UNCACHED, gpu->aspace, &bo, &iova);
 240
 241         if (IS_ERR(ptr))
 242                 return PTR_ERR(ptr);
 243
 244         a5xx_gpu->preempt_bo[ring->id] = bo;
 245         a5xx_gpu->preempt_iova[ring->id] = iova;
 246         a5xx_gpu->preempt[ring->id] = ptr;
 247
 248         /* Set up the defaults on the preemption record */
 249
 250         ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
 251         ptr->info = 0;
 252         ptr->data = 0;
 253         ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT;
 254         ptr->rptr_addr = rbmemptr(ring, rptr);
 255         ptr->counter = iova + A5XX_PREEMPT_RECORD_SIZE;
 256
 257         return 0;
 258 }
 259
 260 void a5xx_preempt_fini(struct msm_gpu *gpu)
 261 {
 262         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 263         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 264         int i;
 265
 266         for (i = 0; i < gpu->nr_rings; i++) {
 267                 if (!a5xx_gpu->preempt_bo[i])
 268                         continue;
 269
 270                 msm_gem_put_vaddr(a5xx_gpu->preempt_bo[i]);
 271
 272                 if (a5xx_gpu->preempt_iova[i])
 273                         msm_gem_put_iova(a5xx_gpu->preempt_bo[i], gpu->aspace);
 274
 275                 drm_gem_object_unreference(a5xx_gpu->preempt_bo[i]);
 276                 a5xx_gpu->preempt_bo[i] = NULL;
 277         }
 278 }
 279
 280 void a5xx_preempt_init(struct msm_gpu *gpu)
 281 {
 282         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 283         struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 284         int i;
 285
 286         /* No preemption if we only have one ring */
 287         if (gpu->nr_rings <= 1)
 288                 return;
 289
 290         for (i = 0; i < gpu->nr_rings; i++) {
 291                 if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
 292                         /*
 293                          * On any failure our adventure is over. Clean up and
 294                          * set nr_rings to 1 to force preemption off
 295                          */
 296                         a5xx_preempt_fini(gpu);
 297                         gpu->nr_rings = 1;
 298
 299                         return;
 300                 }
 301         }
 302
 303         timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
 304 }