drivers/dma-buf/dma-fence-unwrap.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * dma-fence-util: misc functions for dma_fence objects
   4  *
   5  * Copyright (C) 2022 Advanced Micro Devices, Inc.
   6  * Authors:
   7  *      Christian König <christian.koenig@amd.com>
   8  */
   9
  10 #include <linux/dma-fence.h>
  11 #include <linux/dma-fence-array.h>
  12 #include <linux/dma-fence-chain.h>
  13 #include <linux/dma-fence-unwrap.h>
  14 #include <linux/slab.h>
  15 #include <linux/sort.h>
  16
  17 /* Internal helper to start new array iteration, don't use directly */
  18 static struct dma_fence *
  19 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
  20 {
  21         cursor->array = dma_fence_chain_contained(cursor->chain);
  22         cursor->index = 0;
  23         return dma_fence_array_first(cursor->array);
  24 }
  25
  26 /**
  27  * dma_fence_unwrap_first - return the first fence from fence containers
  28  * @head: the entrypoint into the containers
  29  * @cursor: current position inside the containers
  30  *
  31  * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
  32  * first fence.
  33  */
  34 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
  35                                          struct dma_fence_unwrap *cursor)
  36 {
  37         cursor->chain = dma_fence_get(head);
  38         return __dma_fence_unwrap_array(cursor);
  39 }
  40 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);
  41
  42 /**
  43  * dma_fence_unwrap_next - return the next fence from a fence containers
  44  * @cursor: current position inside the containers
  45  *
  46  * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
  47  * the next fence from them.
  48  */
  49 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
  50 {
  51         struct dma_fence *tmp;
  52
  53         ++cursor->index;
  54         tmp = dma_fence_array_next(cursor->array, cursor->index);
  55         if (tmp)
  56                 return tmp;
  57
  58         cursor->chain = dma_fence_chain_walk(cursor->chain);
  59         return __dma_fence_unwrap_array(cursor);
  60 }
  61 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);
  62
  63
  64 static int fence_cmp(const void *_a, const void *_b)
  65 {
  66         struct dma_fence *a = *(struct dma_fence **)_a;
  67         struct dma_fence *b = *(struct dma_fence **)_b;
  68
  69         if (a->context < b->context)
  70                 return -1;
  71         else if (a->context > b->context)
  72                 return 1;
  73
  74         if (dma_fence_is_later(b, a))
  75                 return 1;
  76         else if (dma_fence_is_later(a, b))
  77                 return -1;
  78
  79         return 0;
  80 }
  81
  82 /* Implementation for the dma_fence_merge() marco, don't use directly */
  83 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
  84                                            struct dma_fence **fences,
  85                                            struct dma_fence_unwrap *iter)
  86 {
  87         struct dma_fence *tmp, *unsignaled = NULL, **array;
  88         struct dma_fence_array *result;
  89         ktime_t timestamp;
  90         int i, j, count;
  91
  92         count = 0;
  93         timestamp = ns_to_ktime(0);
  94         for (i = 0; i < num_fences; ++i) {
  95                 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
  96                         if (!dma_fence_is_signaled(tmp)) {
  97                                 dma_fence_put(unsignaled);
  98                                 unsignaled = dma_fence_get(tmp);
  99                                 ++count;
 100                         } else {
 101                                 ktime_t t = dma_fence_timestamp(tmp);
 102
 103                                 if (ktime_after(t, timestamp))
 104                                         timestamp = t;
 105                         }
 106                 }
 107         }
 108
 109         /*
 110          * If we couldn't find a pending fence just return a private signaled
 111          * fence with the timestamp of the last signaled one.
 112          *
 113          * Or if there was a single unsignaled fence left we can return it
 114          * directly and early since that is a major path on many workloads.
 115          */
 116         if (count == 0)
 117                 return dma_fence_allocate_private_stub(timestamp);
 118         else if (count == 1)
 119                 return unsignaled;
 120
 121         dma_fence_put(unsignaled);
 122
 123         array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
 124         if (!array)
 125                 return NULL;
 126
 127         count = 0;
 128         for (i = 0; i < num_fences; ++i) {
 129                 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
 130                         if (!dma_fence_is_signaled(tmp)) {
 131                                 array[count++] = dma_fence_get(tmp);
 132                         } else {
 133                                 ktime_t t = dma_fence_timestamp(tmp);
 134
 135                                 if (ktime_after(t, timestamp))
 136                                         timestamp = t;
 137                         }
 138                 }
 139         }
 140
 141         if (count == 0 || count == 1)
 142                 goto return_fastpath;
 143
 144         sort(array, count, sizeof(*array), fence_cmp, NULL);
 145
 146         /*
 147          * Only keep the most recent fence for each context.
 148          */
 149         j = 0;
 150         for (i = 1; i < count; i++) {
 151                 if (array[i]->context == array[j]->context)
 152                         dma_fence_put(array[i]);
 153                 else
 154                         array[++j] = array[i];
 155         }
 156         count = ++j;
 157
 158         if (count > 1) {
 159                 result = dma_fence_array_create(count, array,
 160                                                 dma_fence_context_alloc(1),
 161                                                 1, false);
 162                 if (!result) {
 163                         for (i = 0; i < count; i++)
 164                                 dma_fence_put(array[i]);
 165                         tmp = NULL;
 166                         goto return_tmp;
 167                 }
 168                 return &result->base;
 169         }
 170
 171 return_fastpath:
 172         if (count == 0)
 173                 tmp = dma_fence_allocate_private_stub(timestamp);
 174         else
 175                 tmp = array[0];
 176
 177 return_tmp:
 178         kfree(array);
 179         return tmp;
 180 }
 181 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);