drivers/gpu/drm/i915/gem/i915_gem_client_blt.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "gt/intel_context.h"
   8 #include "gt/intel_engine_pm.h"
   9 #include "gt/intel_engine_pool.h"
  10 #include "i915_gem_client_blt.h"
  11 #include "i915_gem_object_blt.h"
  12
  13 struct i915_sleeve {
  14         struct i915_vma *vma;
  15         struct drm_i915_gem_object *obj;
  16         struct sg_table *pages;
  17         struct i915_page_sizes page_sizes;
  18 };
  19
  20 static int vma_set_pages(struct i915_vma *vma)
  21 {
  22         struct i915_sleeve *sleeve = vma->private;
  23
  24         vma->pages = sleeve->pages;
  25         vma->page_sizes = sleeve->page_sizes;
  26
  27         return 0;
  28 }
  29
  30 static void vma_clear_pages(struct i915_vma *vma)
  31 {
  32         GEM_BUG_ON(!vma->pages);
  33         vma->pages = NULL;
  34 }
  35
  36 static int vma_bind(struct i915_vma *vma,
  37                     enum i915_cache_level cache_level,
  38                     u32 flags)
  39 {
  40         return vma->vm->vma_ops.bind_vma(vma, cache_level, flags);
  41 }
  42
  43 static void vma_unbind(struct i915_vma *vma)
  44 {
  45         vma->vm->vma_ops.unbind_vma(vma);
  46 }
  47
  48 static const struct i915_vma_ops proxy_vma_ops = {
  49         .set_pages = vma_set_pages,
  50         .clear_pages = vma_clear_pages,
  51         .bind_vma = vma_bind,
  52         .unbind_vma = vma_unbind,
  53 };
  54
  55 static struct i915_sleeve *create_sleeve(struct i915_address_space *vm,
  56                                          struct drm_i915_gem_object *obj,
  57                                          struct sg_table *pages,
  58                                          struct i915_page_sizes *page_sizes)
  59 {
  60         struct i915_sleeve *sleeve;
  61         struct i915_vma *vma;
  62         int err;
  63
  64         sleeve = kzalloc(sizeof(*sleeve), GFP_KERNEL);
  65         if (!sleeve)
  66                 return ERR_PTR(-ENOMEM);
  67
  68         vma = i915_vma_instance(obj, vm, NULL);
  69         if (IS_ERR(vma)) {
  70                 err = PTR_ERR(vma);
  71                 goto err_free;
  72         }
  73
  74         vma->private = sleeve;
  75         vma->ops = &proxy_vma_ops;
  76
  77         sleeve->vma = vma;
  78         sleeve->pages = pages;
  79         sleeve->page_sizes = *page_sizes;
  80
  81         return sleeve;
  82
  83 err_free:
  84         kfree(sleeve);
  85         return ERR_PTR(err);
  86 }
  87
  88 static void destroy_sleeve(struct i915_sleeve *sleeve)
  89 {
  90         kfree(sleeve);
  91 }
  92
  93 struct clear_pages_work {
  94         struct dma_fence dma;
  95         struct dma_fence_cb cb;
  96         struct i915_sw_fence wait;
  97         struct work_struct work;
  98         struct irq_work irq_work;
  99         struct i915_sleeve *sleeve;
 100         struct intel_context *ce;
 101         u32 value;
 102 };
 103
 104 static const char *clear_pages_work_driver_name(struct dma_fence *fence)
 105 {
 106         return DRIVER_NAME;
 107 }
 108
 109 static const char *clear_pages_work_timeline_name(struct dma_fence *fence)
 110 {
 111         return "clear";
 112 }
 113
 114 static void clear_pages_work_release(struct dma_fence *fence)
 115 {
 116         struct clear_pages_work *w = container_of(fence, typeof(*w), dma);
 117
 118         destroy_sleeve(w->sleeve);
 119
 120         i915_sw_fence_fini(&w->wait);
 121
 122         BUILD_BUG_ON(offsetof(typeof(*w), dma));
 123         dma_fence_free(&w->dma);
 124 }
 125
 126 static const struct dma_fence_ops clear_pages_work_ops = {
 127         .get_driver_name = clear_pages_work_driver_name,
 128         .get_timeline_name = clear_pages_work_timeline_name,
 129         .release = clear_pages_work_release,
 130 };
 131
 132 static void clear_pages_signal_irq_worker(struct irq_work *work)
 133 {
 134         struct clear_pages_work *w = container_of(work, typeof(*w), irq_work);
 135
 136         dma_fence_signal(&w->dma);
 137         dma_fence_put(&w->dma);
 138 }
 139
 140 static void clear_pages_dma_fence_cb(struct dma_fence *fence,
 141                                      struct dma_fence_cb *cb)
 142 {
 143         struct clear_pages_work *w = container_of(cb, typeof(*w), cb);
 144
 145         if (fence->error)
 146                 dma_fence_set_error(&w->dma, fence->error);
 147
 148         /*
 149          * Push the signalling of the fence into yet another worker to avoid
 150          * the nightmare locking around the fence spinlock.
 151          */
 152         irq_work_queue(&w->irq_work);
 153 }
 154
 155 static void clear_pages_worker(struct work_struct *work)
 156 {
 157         struct clear_pages_work *w = container_of(work, typeof(*w), work);
 158         struct drm_i915_gem_object *obj = w->sleeve->vma->obj;
 159         struct i915_vma *vma = w->sleeve->vma;
 160         struct i915_request *rq;
 161         struct i915_vma *batch;
 162         int err = w->dma.error;
 163
 164         if (unlikely(err))
 165                 goto out_signal;
 166
 167         if (obj->cache_dirty) {
 168                 if (i915_gem_object_has_struct_page(obj))
 169                         drm_clflush_sg(w->sleeve->pages);
 170                 obj->cache_dirty = false;
 171         }
 172         obj->read_domains = I915_GEM_GPU_DOMAINS;
 173         obj->write_domain = 0;
 174
 175         err = i915_vma_pin(vma, 0, 0, PIN_USER);
 176         if (unlikely(err))
 177                 goto out_signal;
 178
 179         batch = intel_emit_vma_fill_blt(w->ce, vma, w->value);
 180         if (IS_ERR(batch)) {
 181                 err = PTR_ERR(batch);
 182                 goto out_unpin;
 183         }
 184
 185         rq = intel_context_create_request(w->ce);
 186         if (IS_ERR(rq)) {
 187                 err = PTR_ERR(rq);
 188                 goto out_batch;
 189         }
 190
 191         /* There's no way the fence has signalled */
 192         if (dma_fence_add_callback(&rq->fence, &w->cb,
 193                                    clear_pages_dma_fence_cb))
 194                 GEM_BUG_ON(1);
 195
 196         err = intel_emit_vma_mark_active(batch, rq);
 197         if (unlikely(err))
 198                 goto out_request;
 199
 200         if (w->ce->engine->emit_init_breadcrumb) {
 201                 err = w->ce->engine->emit_init_breadcrumb(rq);
 202                 if (unlikely(err))
 203                         goto out_request;
 204         }
 205
 206         /*
 207          * w->dma is already exported via (vma|obj)->resv we need only
 208          * keep track of the GPU activity within this vma/request, and
 209          * propagate the signal from the request to w->dma.
 210          */
 211         err = __i915_vma_move_to_active(vma, rq);
 212         if (err)
 213                 goto out_request;
 214
 215         err = w->ce->engine->emit_bb_start(rq,
 216                                            batch->node.start, batch->node.size,
 217                                            0);
 218 out_request:
 219         if (unlikely(err)) {
 220                 i915_request_skip(rq, err);
 221                 err = 0;
 222         }
 223
 224         i915_request_add(rq);
 225 out_batch:
 226         intel_emit_vma_release(w->ce, batch);
 227 out_unpin:
 228         i915_vma_unpin(vma);
 229 out_signal:
 230         if (unlikely(err)) {
 231                 dma_fence_set_error(&w->dma, err);
 232                 dma_fence_signal(&w->dma);
 233                 dma_fence_put(&w->dma);
 234         }
 235 }
 236
 237 static int __i915_sw_fence_call
 238 clear_pages_work_notify(struct i915_sw_fence *fence,
 239                         enum i915_sw_fence_notify state)
 240 {
 241         struct clear_pages_work *w = container_of(fence, typeof(*w), wait);
 242
 243         switch (state) {
 244         case FENCE_COMPLETE:
 245                 schedule_work(&w->work);
 246                 break;
 247
 248         case FENCE_FREE:
 249                 dma_fence_put(&w->dma);
 250                 break;
 251         }
 252
 253         return NOTIFY_DONE;
 254 }
 255
 256 static DEFINE_SPINLOCK(fence_lock);
 257
 258 /* XXX: better name please */
 259 int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj,
 260                                      struct intel_context *ce,
 261                                      struct sg_table *pages,
 262                                      struct i915_page_sizes *page_sizes,
 263                                      u32 value)
 264 {
 265         struct clear_pages_work *work;
 266         struct i915_sleeve *sleeve;
 267         int err;
 268
 269         sleeve = create_sleeve(ce->vm, obj, pages, page_sizes);
 270         if (IS_ERR(sleeve))
 271                 return PTR_ERR(sleeve);
 272
 273         work = kmalloc(sizeof(*work), GFP_KERNEL);
 274         if (!work) {
 275                 destroy_sleeve(sleeve);
 276                 return -ENOMEM;
 277         }
 278
 279         work->value = value;
 280         work->sleeve = sleeve;
 281         work->ce = ce;
 282
 283         INIT_WORK(&work->work, clear_pages_worker);
 284
 285         init_irq_work(&work->irq_work, clear_pages_signal_irq_worker);
 286
 287         dma_fence_init(&work->dma, &clear_pages_work_ops, &fence_lock, 0, 0);
 288         i915_sw_fence_init(&work->wait, clear_pages_work_notify);
 289
 290         i915_gem_object_lock(obj);
 291         err = i915_sw_fence_await_reservation(&work->wait,
 292                                               obj->base.resv, NULL,
 293                                               true, I915_FENCE_TIMEOUT,
 294                                               I915_FENCE_GFP);
 295         if (err < 0) {
 296                 dma_fence_set_error(&work->dma, err);
 297         } else {
 298                 dma_resv_add_excl_fence(obj->base.resv, &work->dma);
 299                 err = 0;
 300         }
 301         i915_gem_object_unlock(obj);
 302
 303         dma_fence_get(&work->dma);
 304         i915_sw_fence_commit(&work->wait);
 305
 306         return err;
 307 }
 308
 309 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 310 #include "selftests/i915_gem_client_blt.c"
 311 #endif