drivers/gpu/drm/i915/gem/i915_gem_wait.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2016 Intel Corporation
   5  */
   6
   7 #include <linux/dma-fence-array.h>
   8 #include <linux/jiffies.h>
   9
  10 #include "gt/intel_engine.h"
  11
  12 #include "i915_gem_ioctls.h"
  13 #include "i915_gem_object.h"
  14
  15 static long
  16 i915_gem_object_wait_fence(struct dma_fence *fence,
  17                            unsigned int flags,
  18                            long timeout)
  19 {
  20         BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
  21
  22         if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
  23                 return timeout;
  24
  25         if (dma_fence_is_i915(fence))
  26                 return i915_request_wait(to_request(fence), flags, timeout);
  27
  28         return dma_fence_wait_timeout(fence,
  29                                       flags & I915_WAIT_INTERRUPTIBLE,
  30                                       timeout);
  31 }
  32
  33 static long
  34 i915_gem_object_wait_reservation(struct dma_resv *resv,
  35                                  unsigned int flags,
  36                                  long timeout)
  37 {
  38         struct dma_fence *excl;
  39         bool prune_fences = false;
  40
  41         if (flags & I915_WAIT_ALL) {
  42                 struct dma_fence **shared;
  43                 unsigned int count, i;
  44                 int ret;
  45
  46                 ret = dma_resv_get_fences_rcu(resv,
  47                                                         &excl, &count, &shared);
  48                 if (ret)
  49                         return ret;
  50
  51                 for (i = 0; i < count; i++) {
  52                         timeout = i915_gem_object_wait_fence(shared[i],
  53                                                              flags, timeout);
  54                         if (timeout < 0)
  55                                 break;
  56
  57                         dma_fence_put(shared[i]);
  58                 }
  59
  60                 for (; i < count; i++)
  61                         dma_fence_put(shared[i]);
  62                 kfree(shared);
  63
  64                 /*
  65                  * If both shared fences and an exclusive fence exist,
  66                  * then by construction the shared fences must be later
  67                  * than the exclusive fence. If we successfully wait for
  68                  * all the shared fences, we know that the exclusive fence
  69                  * must all be signaled. If all the shared fences are
  70                  * signaled, we can prune the array and recover the
  71                  * floating references on the fences/requests.
  72                  */
  73                 prune_fences = count && timeout >= 0;
  74         } else {
  75                 excl = dma_resv_get_excl_rcu(resv);
  76         }
  77
  78         if (excl && timeout >= 0)
  79                 timeout = i915_gem_object_wait_fence(excl, flags, timeout);
  80
  81         dma_fence_put(excl);
  82
  83         /*
  84          * Opportunistically prune the fences iff we know they have *all* been
  85          * signaled.
  86          */
  87         if (prune_fences && dma_resv_trylock(resv)) {
  88                 if (dma_resv_test_signaled_rcu(resv, true))
  89                         dma_resv_add_excl_fence(resv, NULL);
  90                 dma_resv_unlock(resv);
  91         }
  92
  93         return timeout;
  94 }
  95
  96 static void __fence_set_priority(struct dma_fence *fence,
  97                                  const struct i915_sched_attr *attr)
  98 {
  99         struct i915_request *rq;
 100         struct intel_engine_cs *engine;
 101
 102         if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
 103                 return;
 104
 105         rq = to_request(fence);
 106         engine = rq->engine;
 107
 108         local_bh_disable();
 109         rcu_read_lock(); /* RCU serialisation for set-wedged protection */
 110         if (engine->schedule)
 111                 engine->schedule(rq, attr);
 112         rcu_read_unlock();
 113         local_bh_enable(); /* kick the tasklets if queues were reprioritised */
 114 }
 115
 116 static void fence_set_priority(struct dma_fence *fence,
 117                                const struct i915_sched_attr *attr)
 118 {
 119         /* Recurse once into a fence-array */
 120         if (dma_fence_is_array(fence)) {
 121                 struct dma_fence_array *array = to_dma_fence_array(fence);
 122                 int i;
 123
 124                 for (i = 0; i < array->num_fences; i++)
 125                         __fence_set_priority(array->fences[i], attr);
 126         } else {
 127                 __fence_set_priority(fence, attr);
 128         }
 129 }
 130
 131 int
 132 i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
 133                               unsigned int flags,
 134                               const struct i915_sched_attr *attr)
 135 {
 136         struct dma_fence *excl;
 137
 138         if (flags & I915_WAIT_ALL) {
 139                 struct dma_fence **shared;
 140                 unsigned int count, i;
 141                 int ret;
 142
 143                 ret = dma_resv_get_fences_rcu(obj->base.resv,
 144                                                         &excl, &count, &shared);
 145                 if (ret)
 146                         return ret;
 147
 148                 for (i = 0; i < count; i++) {
 149                         fence_set_priority(shared[i], attr);
 150                         dma_fence_put(shared[i]);
 151                 }
 152
 153                 kfree(shared);
 154         } else {
 155                 excl = dma_resv_get_excl_rcu(obj->base.resv);
 156         }
 157
 158         if (excl) {
 159                 fence_set_priority(excl, attr);
 160                 dma_fence_put(excl);
 161         }
 162         return 0;
 163 }
 164
 165 /**
 166  * Waits for rendering to the object to be completed
 167  * @obj: i915 gem object
 168  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 169  * @timeout: how long to wait
 170  */
 171 int
 172 i915_gem_object_wait(struct drm_i915_gem_object *obj,
 173                      unsigned int flags,
 174                      long timeout)
 175 {
 176         might_sleep();
 177         GEM_BUG_ON(timeout < 0);
 178
 179         timeout = i915_gem_object_wait_reservation(obj->base.resv,
 180                                                    flags, timeout);
 181         return timeout < 0 ? timeout : 0;
 182 }
 183
 184 static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
 185 {
 186         /* nsecs_to_jiffies64() does not guard against overflow */
 187         if (NSEC_PER_SEC % HZ &&
 188             div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
 189                 return MAX_JIFFY_OFFSET;
 190
 191         return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
 192 }
 193
 194 static unsigned long to_wait_timeout(s64 timeout_ns)
 195 {
 196         if (timeout_ns < 0)
 197                 return MAX_SCHEDULE_TIMEOUT;
 198
 199         if (timeout_ns == 0)
 200                 return 0;
 201
 202         return nsecs_to_jiffies_timeout(timeout_ns);
 203 }
 204
 205 /**
 206  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 207  * @dev: drm device pointer
 208  * @data: ioctl data blob
 209  * @file: drm file pointer
 210  *
 211  * Returns 0 if successful, else an error is returned with the remaining time in
 212  * the timeout parameter.
 213  *  -ETIME: object is still busy after timeout
 214  *  -ERESTARTSYS: signal interrupted the wait
 215  *  -ENONENT: object doesn't exist
 216  * Also possible, but rare:
 217  *  -EAGAIN: incomplete, restart syscall
 218  *  -ENOMEM: damn
 219  *  -ENODEV: Internal IRQ fail
 220  *  -E?: The add request failed
 221  *
 222  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 223  * non-zero timeout parameter the wait ioctl will wait for the given number of
 224  * nanoseconds on an object becoming unbusy. Since the wait itself does so
 225  * without holding struct_mutex the object may become re-busied before this
 226  * function completes. A similar but shorter * race condition exists in the busy
 227  * ioctl
 228  */
 229 int
 230 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 231 {
 232         struct drm_i915_gem_wait *args = data;
 233         struct drm_i915_gem_object *obj;
 234         ktime_t start;
 235         long ret;
 236
 237         if (args->flags != 0)
 238                 return -EINVAL;
 239
 240         obj = i915_gem_object_lookup(file, args->bo_handle);
 241         if (!obj)
 242                 return -ENOENT;
 243
 244         start = ktime_get();
 245
 246         ret = i915_gem_object_wait(obj,
 247                                    I915_WAIT_INTERRUPTIBLE |
 248                                    I915_WAIT_PRIORITY |
 249                                    I915_WAIT_ALL,
 250                                    to_wait_timeout(args->timeout_ns));
 251
 252         if (args->timeout_ns > 0) {
 253                 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
 254                 if (args->timeout_ns < 0)
 255                         args->timeout_ns = 0;
 256
 257                 /*
 258                  * Apparently ktime isn't accurate enough and occasionally has a
 259                  * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
 260                  * things up to make the test happy. We allow up to 1 jiffy.
 261                  *
 262                  * This is a regression from the timespec->ktime conversion.
 263                  */
 264                 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
 265                         args->timeout_ns = 0;
 266
 267                 /* Asked to wait beyond the jiffie/scheduler precision? */
 268                 if (ret == -ETIME && args->timeout_ns)
 269                         ret = -EAGAIN;
 270         }
 271
 272         i915_gem_object_put(obj);
 273         return ret;
 274 }