drivers/accel/ivpu/ivpu_job.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2024 Intel Corporation
   4  */
   5
   6 #include <drm/drm_file.h>
   7
   8 #include <linux/bitfield.h>
   9 #include <linux/highmem.h>
  10 #include <linux/pci.h>
  11 #include <linux/module.h>
  12 #include <uapi/drm/ivpu_accel.h>
  13
  14 #include "ivpu_drv.h"
  15 #include "ivpu_fw.h"
  16 #include "ivpu_hw.h"
  17 #include "ivpu_ipc.h"
  18 #include "ivpu_job.h"
  19 #include "ivpu_jsm_msg.h"
  20 #include "ivpu_mmu.h"
  21 #include "ivpu_pm.h"
  22 #include "ivpu_trace.h"
  23 #include "vpu_boot_api.h"
  24
  25 #define CMD_BUF_IDX          0
  26 #define JOB_MAX_BUFFER_COUNT 65535
  27
  28 static void ivpu_cmdq_ring_db(struct ivpu_device *vdev, struct ivpu_cmdq *cmdq)
  29 {
  30         ivpu_hw_db_set(vdev, cmdq->db_id);
  31 }
  32
  33 static int ivpu_preemption_buffers_create(struct ivpu_device *vdev,
  34                                           struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  35 {
  36         u64 primary_size = ALIGN(vdev->fw->primary_preempt_buf_size, PAGE_SIZE);
  37         u64 secondary_size = ALIGN(vdev->fw->secondary_preempt_buf_size, PAGE_SIZE);
  38
  39         if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW ||
  40             ivpu_test_mode & IVPU_TEST_MODE_MIP_DISABLE)
  41                 return 0;
  42
  43         cmdq->primary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.user,
  44                                                    primary_size, DRM_IVPU_BO_WC);
  45         if (!cmdq->primary_preempt_buf) {
  46                 ivpu_err(vdev, "Failed to create primary preemption buffer\n");
  47                 return -ENOMEM;
  48         }
  49
  50         cmdq->secondary_preempt_buf = ivpu_bo_create(vdev, &file_priv->ctx, &vdev->hw->ranges.dma,
  51                                                      secondary_size, DRM_IVPU_BO_WC);
  52         if (!cmdq->secondary_preempt_buf) {
  53                 ivpu_err(vdev, "Failed to create secondary preemption buffer\n");
  54                 goto err_free_primary;
  55         }
  56
  57         return 0;
  58
  59 err_free_primary:
  60         ivpu_bo_free(cmdq->primary_preempt_buf);
  61         cmdq->primary_preempt_buf = NULL;
  62         return -ENOMEM;
  63 }
  64
  65 static void ivpu_preemption_buffers_free(struct ivpu_device *vdev,
  66                                          struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
  67 {
  68         if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW)
  69                 return;
  70
  71         if (cmdq->primary_preempt_buf)
  72                 ivpu_bo_free(cmdq->primary_preempt_buf);
  73         if (cmdq->secondary_preempt_buf)
  74                 ivpu_bo_free(cmdq->secondary_preempt_buf);
  75 }
  76
  77 static struct ivpu_cmdq *ivpu_cmdq_alloc(struct ivpu_file_priv *file_priv)
  78 {
  79         struct ivpu_device *vdev = file_priv->vdev;
  80         struct ivpu_cmdq *cmdq;
  81         int ret;
  82
  83         cmdq = kzalloc(sizeof(*cmdq), GFP_KERNEL);
  84         if (!cmdq)
  85                 return NULL;
  86
  87         cmdq->mem = ivpu_bo_create_global(vdev, SZ_4K, DRM_IVPU_BO_WC | DRM_IVPU_BO_MAPPABLE);
  88         if (!cmdq->mem)
  89                 goto err_free_cmdq;
  90
  91         ret = ivpu_preemption_buffers_create(vdev, file_priv, cmdq);
  92         if (ret)
  93                 ivpu_warn(vdev, "Failed to allocate preemption buffers, preemption limited\n");
  94
  95         return cmdq;
  96
  97 err_free_cmdq:
  98         kfree(cmdq);
  99         return NULL;
 100 }
 101
 102 static void ivpu_cmdq_free(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 103 {
 104         ivpu_preemption_buffers_free(file_priv->vdev, file_priv, cmdq);
 105         ivpu_bo_free(cmdq->mem);
 106         kfree(cmdq);
 107 }
 108
 109 static struct ivpu_cmdq *ivpu_cmdq_create(struct ivpu_file_priv *file_priv, u8 priority,
 110                                           bool is_legacy)
 111 {
 112         struct ivpu_device *vdev = file_priv->vdev;
 113         struct ivpu_cmdq *cmdq = NULL;
 114         int ret;
 115
 116         lockdep_assert_held(&file_priv->lock);
 117
 118         cmdq = ivpu_cmdq_alloc(file_priv);
 119         if (!cmdq) {
 120                 ivpu_err(vdev, "Failed to allocate command queue\n");
 121                 return NULL;
 122         }
 123
 124         cmdq->priority = priority;
 125         cmdq->is_legacy = is_legacy;
 126
 127         ret = xa_alloc_cyclic(&file_priv->cmdq_xa, &cmdq->id, cmdq, file_priv->cmdq_limit,
 128                               &file_priv->cmdq_id_next, GFP_KERNEL);
 129         if (ret < 0) {
 130                 ivpu_err(vdev, "Failed to allocate command queue ID: %d\n", ret);
 131                 goto err_free_cmdq;
 132         }
 133
 134         ivpu_dbg(vdev, JOB, "Command queue %d created, ctx %d\n", cmdq->id, file_priv->ctx.id);
 135         return cmdq;
 136
 137 err_free_cmdq:
 138         ivpu_cmdq_free(file_priv, cmdq);
 139         return NULL;
 140 }
 141
 142 static int ivpu_hws_cmdq_init(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq, u16 engine,
 143                               u8 priority)
 144 {
 145         struct ivpu_device *vdev = file_priv->vdev;
 146         int ret;
 147
 148         ret = ivpu_jsm_hws_create_cmdq(vdev, file_priv->ctx.id, file_priv->ctx.id, cmdq->id,
 149                                        task_pid_nr(current), engine,
 150                                        cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 151         if (ret)
 152                 return ret;
 153
 154         ret = ivpu_jsm_hws_set_context_sched_properties(vdev, file_priv->ctx.id, cmdq->id,
 155                                                         priority);
 156         if (ret)
 157                 return ret;
 158
 159         return 0;
 160 }
 161
 162 static int ivpu_register_db(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 163 {
 164         struct ivpu_device *vdev = file_priv->vdev;
 165         int ret;
 166
 167         ret = xa_alloc_cyclic(&vdev->db_xa, &cmdq->db_id, NULL, vdev->db_limit, &vdev->db_next,
 168                               GFP_KERNEL);
 169         if (ret < 0) {
 170                 ivpu_err(vdev, "Failed to allocate doorbell ID: %d\n", ret);
 171                 return ret;
 172         }
 173
 174         if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW)
 175                 ret = ivpu_jsm_hws_register_db(vdev, file_priv->ctx.id, cmdq->id, cmdq->db_id,
 176                                                cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 177         else
 178                 ret = ivpu_jsm_register_db(vdev, file_priv->ctx.id, cmdq->db_id,
 179                                            cmdq->mem->vpu_addr, ivpu_bo_size(cmdq->mem));
 180
 181         if (!ret)
 182                 ivpu_dbg(vdev, JOB, "DB %d registered to cmdq %d ctx %d priority %d\n",
 183                          cmdq->db_id, cmdq->id, file_priv->ctx.id, cmdq->priority);
 184         else
 185                 xa_erase(&vdev->db_xa, cmdq->db_id);
 186
 187         return ret;
 188 }
 189
 190 static void ivpu_cmdq_jobq_init(struct ivpu_device *vdev, struct vpu_job_queue *jobq)
 191 {
 192         jobq->header.engine_idx = VPU_ENGINE_COMPUTE;
 193         jobq->header.head = 0;
 194         jobq->header.tail = 0;
 195
 196         if (ivpu_test_mode & IVPU_TEST_MODE_TURBO) {
 197                 ivpu_dbg(vdev, JOB, "Turbo mode enabled");
 198                 jobq->header.flags = VPU_JOB_QUEUE_FLAGS_TURBO_MODE;
 199         }
 200
 201         wmb(); /* Flush WC buffer for jobq->header */
 202 }
 203
 204 static inline u32 ivpu_cmdq_get_entry_count(struct ivpu_cmdq *cmdq)
 205 {
 206         size_t size = ivpu_bo_size(cmdq->mem) - sizeof(struct vpu_job_queue_header);
 207
 208         return size / sizeof(struct vpu_job_queue_entry);
 209 }
 210
 211 static int ivpu_cmdq_register(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 212 {
 213         struct ivpu_device *vdev = file_priv->vdev;
 214         int ret;
 215
 216         lockdep_assert_held(&file_priv->lock);
 217
 218         if (cmdq->db_id)
 219                 return 0;
 220
 221         cmdq->entry_count = ivpu_cmdq_get_entry_count(cmdq);
 222         cmdq->jobq = (struct vpu_job_queue *)ivpu_bo_vaddr(cmdq->mem);
 223
 224         ivpu_cmdq_jobq_init(vdev, cmdq->jobq);
 225
 226         if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) {
 227                 ret = ivpu_hws_cmdq_init(file_priv, cmdq, VPU_ENGINE_COMPUTE, cmdq->priority);
 228                 if (ret)
 229                         return ret;
 230         }
 231
 232         ret = ivpu_register_db(file_priv, cmdq);
 233         if (ret)
 234                 return ret;
 235
 236         return 0;
 237 }
 238
 239 static int ivpu_cmdq_unregister(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 240 {
 241         struct ivpu_device *vdev = file_priv->vdev;
 242         int ret;
 243
 244         lockdep_assert_held(&file_priv->lock);
 245
 246         if (!cmdq->db_id)
 247                 return 0;
 248
 249         if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) {
 250                 ret = ivpu_jsm_hws_destroy_cmdq(vdev, file_priv->ctx.id, cmdq->id);
 251                 if (!ret)
 252                         ivpu_dbg(vdev, JOB, "Command queue %d destroyed, ctx %d\n",
 253                                  cmdq->id, file_priv->ctx.id);
 254         }
 255
 256         ret = ivpu_jsm_unregister_db(vdev, cmdq->db_id);
 257         if (!ret)
 258                 ivpu_dbg(vdev, JOB, "DB %d unregistered\n", cmdq->db_id);
 259
 260         xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
 261         cmdq->db_id = 0;
 262
 263         return 0;
 264 }
 265
 266 static inline u8 ivpu_job_to_jsm_priority(u8 priority)
 267 {
 268         if (priority == DRM_IVPU_JOB_PRIORITY_DEFAULT)
 269                 return VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL;
 270
 271         return priority - 1;
 272 }
 273
 274 static void ivpu_cmdq_destroy(struct ivpu_file_priv *file_priv, struct ivpu_cmdq *cmdq)
 275 {
 276         ivpu_cmdq_unregister(file_priv, cmdq);
 277         xa_erase(&file_priv->cmdq_xa, cmdq->id);
 278         ivpu_cmdq_free(file_priv, cmdq);
 279 }
 280
 281 static struct ivpu_cmdq *ivpu_cmdq_acquire_legacy(struct ivpu_file_priv *file_priv, u8 priority)
 282 {
 283         struct ivpu_cmdq *cmdq;
 284         unsigned long id;
 285
 286         lockdep_assert_held(&file_priv->lock);
 287
 288         xa_for_each(&file_priv->cmdq_xa, id, cmdq)
 289                 if (cmdq->is_legacy && cmdq->priority == priority)
 290                         break;
 291
 292         if (!cmdq) {
 293                 cmdq = ivpu_cmdq_create(file_priv, priority, true);
 294                 if (!cmdq)
 295                         return NULL;
 296         }
 297
 298         return cmdq;
 299 }
 300
 301 static struct ivpu_cmdq *ivpu_cmdq_acquire(struct ivpu_file_priv *file_priv, u32 cmdq_id)
 302 {
 303         struct ivpu_device *vdev = file_priv->vdev;
 304         struct ivpu_cmdq *cmdq;
 305
 306         lockdep_assert_held(&file_priv->lock);
 307
 308         cmdq = xa_load(&file_priv->cmdq_xa, cmdq_id);
 309         if (!cmdq) {
 310                 ivpu_warn_ratelimited(vdev, "Failed to find command queue with ID: %u\n", cmdq_id);
 311                 return NULL;
 312         }
 313
 314         return cmdq;
 315 }
 316
 317 void ivpu_cmdq_release_all_locked(struct ivpu_file_priv *file_priv)
 318 {
 319         struct ivpu_cmdq *cmdq;
 320         unsigned long cmdq_id;
 321
 322         lockdep_assert_held(&file_priv->lock);
 323
 324         xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)
 325                 ivpu_cmdq_destroy(file_priv, cmdq);
 326 }
 327
 328 /*
 329  * Mark the doorbell as unregistered
 330  * This function needs to be called when the VPU hardware is restarted
 331  * and FW loses job queue state. The next time job queue is used it
 332  * will be registered again.
 333  */
 334 static void ivpu_cmdq_reset(struct ivpu_file_priv *file_priv)
 335 {
 336         struct ivpu_cmdq *cmdq;
 337         unsigned long cmdq_id;
 338
 339         mutex_lock(&file_priv->lock);
 340
 341         xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq) {
 342                 xa_erase(&file_priv->vdev->db_xa, cmdq->db_id);
 343                 cmdq->db_id = 0;
 344         }
 345
 346         mutex_unlock(&file_priv->lock);
 347 }
 348
 349 void ivpu_cmdq_reset_all_contexts(struct ivpu_device *vdev)
 350 {
 351         struct ivpu_file_priv *file_priv;
 352         unsigned long ctx_id;
 353
 354         mutex_lock(&vdev->context_list_lock);
 355
 356         xa_for_each(&vdev->context_xa, ctx_id, file_priv)
 357                 ivpu_cmdq_reset(file_priv);
 358
 359         mutex_unlock(&vdev->context_list_lock);
 360 }
 361
 362 void ivpu_context_abort_locked(struct ivpu_file_priv *file_priv)
 363 {
 364         struct ivpu_device *vdev = file_priv->vdev;
 365         struct ivpu_cmdq *cmdq;
 366         unsigned long cmdq_id;
 367
 368         lockdep_assert_held(&file_priv->lock);
 369         ivpu_dbg(vdev, JOB, "Context ID: %u abort\n", file_priv->ctx.id);
 370
 371         xa_for_each(&file_priv->cmdq_xa, cmdq_id, cmdq)
 372                 ivpu_cmdq_unregister(file_priv, cmdq);
 373
 374         if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_OS)
 375                 ivpu_jsm_context_release(vdev, file_priv->ctx.id);
 376
 377         ivpu_mmu_disable_ssid_events(vdev, file_priv->ctx.id);
 378         ivpu_mmu_discard_events(vdev);
 379
 380         file_priv->aborted = true;
 381 }
 382
 383 static int ivpu_cmdq_push_job(struct ivpu_cmdq *cmdq, struct ivpu_job *job)
 384 {
 385         struct ivpu_device *vdev = job->vdev;
 386         struct vpu_job_queue_header *header = &cmdq->jobq->header;
 387         struct vpu_job_queue_entry *entry;
 388         u32 tail = READ_ONCE(header->tail);
 389         u32 next_entry = (tail + 1) % cmdq->entry_count;
 390
 391         /* Check if there is space left in job queue */
 392         if (next_entry == header->head) {
 393                 ivpu_dbg(vdev, JOB, "Job queue full: ctx %d cmdq %d db %d head %d tail %d\n",
 394                          job->file_priv->ctx.id, cmdq->id, cmdq->db_id, header->head, tail);
 395                 return -EBUSY;
 396         }
 397
 398         entry = &cmdq->jobq->slot[tail].job;
 399         entry->batch_buf_addr = job->cmd_buf_vpu_addr;
 400         entry->job_id = job->job_id;
 401         entry->flags = 0;
 402         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_SUBMISSION))
 403                 entry->flags = VPU_JOB_FLAGS_NULL_SUBMISSION_MASK;
 404
 405         if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) {
 406                 if (cmdq->primary_preempt_buf) {
 407                         entry->primary_preempt_buf_addr = cmdq->primary_preempt_buf->vpu_addr;
 408                         entry->primary_preempt_buf_size = ivpu_bo_size(cmdq->primary_preempt_buf);
 409                 }
 410
 411                 if (cmdq->secondary_preempt_buf) {
 412                         entry->secondary_preempt_buf_addr = cmdq->secondary_preempt_buf->vpu_addr;
 413                         entry->secondary_preempt_buf_size =
 414                                 ivpu_bo_size(cmdq->secondary_preempt_buf);
 415                 }
 416         }
 417
 418         wmb(); /* Ensure that tail is updated after filling entry */
 419         header->tail = next_entry;
 420         wmb(); /* Flush WC buffer for jobq header */
 421
 422         return 0;
 423 }
 424
 425 struct ivpu_fence {
 426         struct dma_fence base;
 427         spinlock_t lock; /* protects base */
 428         struct ivpu_device *vdev;
 429 };
 430
 431 static inline struct ivpu_fence *to_vpu_fence(struct dma_fence *fence)
 432 {
 433         return container_of(fence, struct ivpu_fence, base);
 434 }
 435
 436 static const char *ivpu_fence_get_driver_name(struct dma_fence *fence)
 437 {
 438         return DRIVER_NAME;
 439 }
 440
 441 static const char *ivpu_fence_get_timeline_name(struct dma_fence *fence)
 442 {
 443         struct ivpu_fence *ivpu_fence = to_vpu_fence(fence);
 444
 445         return dev_name(ivpu_fence->vdev->drm.dev);
 446 }
 447
 448 static const struct dma_fence_ops ivpu_fence_ops = {
 449         .get_driver_name = ivpu_fence_get_driver_name,
 450         .get_timeline_name = ivpu_fence_get_timeline_name,
 451 };
 452
 453 static struct dma_fence *ivpu_fence_create(struct ivpu_device *vdev)
 454 {
 455         struct ivpu_fence *fence;
 456
 457         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 458         if (!fence)
 459                 return NULL;
 460
 461         fence->vdev = vdev;
 462         spin_lock_init(&fence->lock);
 463         dma_fence_init(&fence->base, &ivpu_fence_ops, &fence->lock, dma_fence_context_alloc(1), 1);
 464
 465         return &fence->base;
 466 }
 467
 468 static void ivpu_job_destroy(struct ivpu_job *job)
 469 {
 470         struct ivpu_device *vdev = job->vdev;
 471         u32 i;
 472
 473         ivpu_dbg(vdev, JOB, "Job destroyed: id %3u ctx %2d engine %d",
 474                  job->job_id, job->file_priv->ctx.id, job->engine_idx);
 475
 476         for (i = 0; i < job->bo_count; i++)
 477                 if (job->bos[i])
 478                         drm_gem_object_put(&job->bos[i]->base.base);
 479
 480         dma_fence_put(job->done_fence);
 481         ivpu_file_priv_put(&job->file_priv);
 482         kfree(job);
 483 }
 484
 485 static struct ivpu_job *
 486 ivpu_job_create(struct ivpu_file_priv *file_priv, u32 engine_idx, u32 bo_count)
 487 {
 488         struct ivpu_device *vdev = file_priv->vdev;
 489         struct ivpu_job *job;
 490
 491         job = kzalloc(struct_size(job, bos, bo_count), GFP_KERNEL);
 492         if (!job)
 493                 return NULL;
 494
 495         job->vdev = vdev;
 496         job->engine_idx = engine_idx;
 497         job->bo_count = bo_count;
 498         job->done_fence = ivpu_fence_create(vdev);
 499         if (!job->done_fence) {
 500                 ivpu_warn_ratelimited(vdev, "Failed to create a fence\n");
 501                 goto err_free_job;
 502         }
 503
 504         job->file_priv = ivpu_file_priv_get(file_priv);
 505
 506         trace_job("create", job);
 507         ivpu_dbg(vdev, JOB, "Job created: ctx %2d engine %d", file_priv->ctx.id, job->engine_idx);
 508         return job;
 509
 510 err_free_job:
 511         kfree(job);
 512         return NULL;
 513 }
 514
 515 static struct ivpu_job *ivpu_job_remove_from_submitted_jobs(struct ivpu_device *vdev, u32 job_id)
 516 {
 517         struct ivpu_job *job;
 518
 519         lockdep_assert_held(&vdev->submitted_jobs_lock);
 520
 521         job = xa_erase(&vdev->submitted_jobs_xa, job_id);
 522         if (xa_empty(&vdev->submitted_jobs_xa) && job) {
 523                 vdev->busy_time = ktime_add(ktime_sub(ktime_get(), vdev->busy_start_ts),
 524                                             vdev->busy_time);
 525         }
 526
 527         return job;
 528 }
 529
 530 static int ivpu_job_signal_and_destroy(struct ivpu_device *vdev, u32 job_id, u32 job_status)
 531 {
 532         struct ivpu_job *job;
 533
 534         lockdep_assert_held(&vdev->submitted_jobs_lock);
 535
 536         job = xa_load(&vdev->submitted_jobs_xa, job_id);
 537         if (!job)
 538                 return -ENOENT;
 539
 540         if (job_status == VPU_JSM_STATUS_MVNCI_CONTEXT_VIOLATION_HW) {
 541                 guard(mutex)(&job->file_priv->lock);
 542
 543                 if (job->file_priv->has_mmu_faults)
 544                         return 0;
 545
 546                 /*
 547                  * Mark context as faulty and defer destruction of the job to jobs abort thread
 548                  * handler to synchronize between both faults and jobs returning context violation
 549                  * status and ensure both are handled in the same way
 550                  */
 551                 job->file_priv->has_mmu_faults = true;
 552                 queue_work(system_wq, &vdev->context_abort_work);
 553                 return 0;
 554         }
 555
 556         job = ivpu_job_remove_from_submitted_jobs(vdev, job_id);
 557         if (!job)
 558                 return -ENOENT;
 559
 560         if (job->file_priv->has_mmu_faults)
 561                 job_status = DRM_IVPU_JOB_STATUS_ABORTED;
 562
 563         job->bos[CMD_BUF_IDX]->job_status = job_status;
 564         dma_fence_signal(job->done_fence);
 565
 566         trace_job("done", job);
 567         ivpu_dbg(vdev, JOB, "Job complete:  id %3u ctx %2d engine %d status 0x%x\n",
 568                  job->job_id, job->file_priv->ctx.id, job->engine_idx, job_status);
 569
 570         ivpu_job_destroy(job);
 571         ivpu_stop_job_timeout_detection(vdev);
 572
 573         ivpu_rpm_put(vdev);
 574
 575         if (!xa_empty(&vdev->submitted_jobs_xa))
 576                 ivpu_start_job_timeout_detection(vdev);
 577
 578         return 0;
 579 }
 580
 581 void ivpu_jobs_abort_all(struct ivpu_device *vdev)
 582 {
 583         struct ivpu_job *job;
 584         unsigned long id;
 585
 586         mutex_lock(&vdev->submitted_jobs_lock);
 587
 588         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 589                 ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
 590
 591         mutex_unlock(&vdev->submitted_jobs_lock);
 592 }
 593
 594 void ivpu_cmdq_abort_all_jobs(struct ivpu_device *vdev, u32 ctx_id, u32 cmdq_id)
 595 {
 596         struct ivpu_job *job;
 597         unsigned long id;
 598
 599         mutex_lock(&vdev->submitted_jobs_lock);
 600
 601         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 602                 if (job->file_priv->ctx.id == ctx_id && job->cmdq_id == cmdq_id)
 603                         ivpu_job_signal_and_destroy(vdev, id, DRM_IVPU_JOB_STATUS_ABORTED);
 604
 605         mutex_unlock(&vdev->submitted_jobs_lock);
 606 }
 607
 608 static int ivpu_job_submit(struct ivpu_job *job, u8 priority, u32 cmdq_id)
 609 {
 610         struct ivpu_file_priv *file_priv = job->file_priv;
 611         struct ivpu_device *vdev = job->vdev;
 612         struct ivpu_cmdq *cmdq;
 613         bool is_first_job;
 614         int ret;
 615
 616         ret = ivpu_rpm_get(vdev);
 617         if (ret < 0)
 618                 return ret;
 619
 620         mutex_lock(&vdev->submitted_jobs_lock);
 621         mutex_lock(&file_priv->lock);
 622
 623         if (cmdq_id == 0)
 624                 cmdq = ivpu_cmdq_acquire_legacy(file_priv, priority);
 625         else
 626                 cmdq = ivpu_cmdq_acquire(file_priv, cmdq_id);
 627         if (!cmdq) {
 628                 ivpu_warn_ratelimited(vdev, "Failed to get job queue, ctx %d\n", file_priv->ctx.id);
 629                 ret = -EINVAL;
 630                 goto err_unlock;
 631         }
 632
 633         ret = ivpu_cmdq_register(file_priv, cmdq);
 634         if (ret) {
 635                 ivpu_err(vdev, "Failed to register command queue: %d\n", ret);
 636                 goto err_unlock;
 637         }
 638
 639         job->cmdq_id = cmdq->id;
 640
 641         is_first_job = xa_empty(&vdev->submitted_jobs_xa);
 642         ret = xa_alloc_cyclic(&vdev->submitted_jobs_xa, &job->job_id, job, file_priv->job_limit,
 643                               &file_priv->job_id_next, GFP_KERNEL);
 644         if (ret < 0) {
 645                 ivpu_dbg(vdev, JOB, "Too many active jobs in ctx %d\n",
 646                          file_priv->ctx.id);
 647                 ret = -EBUSY;
 648                 goto err_unlock;
 649         }
 650
 651         ret = ivpu_cmdq_push_job(cmdq, job);
 652         if (ret)
 653                 goto err_erase_xa;
 654
 655         ivpu_start_job_timeout_detection(vdev);
 656
 657         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
 658                 cmdq->jobq->header.head = cmdq->jobq->header.tail;
 659                 wmb(); /* Flush WC buffer for jobq header */
 660         } else {
 661                 ivpu_cmdq_ring_db(vdev, cmdq);
 662                 if (is_first_job)
 663                         vdev->busy_start_ts = ktime_get();
 664         }
 665
 666         trace_job("submit", job);
 667         ivpu_dbg(vdev, JOB, "Job submitted: id %3u ctx %2d engine %d prio %d addr 0x%llx next %d\n",
 668                  job->job_id, file_priv->ctx.id, job->engine_idx, cmdq->priority,
 669                  job->cmd_buf_vpu_addr, cmdq->jobq->header.tail);
 670
 671         mutex_unlock(&file_priv->lock);
 672
 673         if (unlikely(ivpu_test_mode & IVPU_TEST_MODE_NULL_HW)) {
 674                 ivpu_job_signal_and_destroy(vdev, job->job_id, VPU_JSM_STATUS_SUCCESS);
 675         }
 676
 677         mutex_unlock(&vdev->submitted_jobs_lock);
 678
 679         return 0;
 680
 681 err_erase_xa:
 682         xa_erase(&vdev->submitted_jobs_xa, job->job_id);
 683 err_unlock:
 684         mutex_unlock(&vdev->submitted_jobs_lock);
 685         mutex_unlock(&file_priv->lock);
 686         ivpu_rpm_put(vdev);
 687         return ret;
 688 }
 689
 690 static int
 691 ivpu_job_prepare_bos_for_submit(struct drm_file *file, struct ivpu_job *job, u32 *buf_handles,
 692                                 u32 buf_count, u32 commands_offset)
 693 {
 694         struct ivpu_file_priv *file_priv = job->file_priv;
 695         struct ivpu_device *vdev = file_priv->vdev;
 696         struct ww_acquire_ctx acquire_ctx;
 697         enum dma_resv_usage usage;
 698         struct ivpu_bo *bo;
 699         int ret;
 700         u32 i;
 701
 702         for (i = 0; i < buf_count; i++) {
 703                 struct drm_gem_object *obj = drm_gem_object_lookup(file, buf_handles[i]);
 704
 705                 if (!obj)
 706                         return -ENOENT;
 707
 708                 job->bos[i] = to_ivpu_bo(obj);
 709
 710                 ret = ivpu_bo_pin(job->bos[i]);
 711                 if (ret)
 712                         return ret;
 713         }
 714
 715         bo = job->bos[CMD_BUF_IDX];
 716         if (!dma_resv_test_signaled(bo->base.base.resv, DMA_RESV_USAGE_READ)) {
 717                 ivpu_warn(vdev, "Buffer is already in use\n");
 718                 return -EBUSY;
 719         }
 720
 721         if (commands_offset >= ivpu_bo_size(bo)) {
 722                 ivpu_warn(vdev, "Invalid command buffer offset %u\n", commands_offset);
 723                 return -EINVAL;
 724         }
 725
 726         job->cmd_buf_vpu_addr = bo->vpu_addr + commands_offset;
 727
 728         ret = drm_gem_lock_reservations((struct drm_gem_object **)job->bos, buf_count,
 729                                         &acquire_ctx);
 730         if (ret) {
 731                 ivpu_warn(vdev, "Failed to lock reservations: %d\n", ret);
 732                 return ret;
 733         }
 734
 735         for (i = 0; i < buf_count; i++) {
 736                 ret = dma_resv_reserve_fences(job->bos[i]->base.base.resv, 1);
 737                 if (ret) {
 738                         ivpu_warn(vdev, "Failed to reserve fences: %d\n", ret);
 739                         goto unlock_reservations;
 740                 }
 741         }
 742
 743         for (i = 0; i < buf_count; i++) {
 744                 usage = (i == CMD_BUF_IDX) ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_BOOKKEEP;
 745                 dma_resv_add_fence(job->bos[i]->base.base.resv, job->done_fence, usage);
 746         }
 747
 748 unlock_reservations:
 749         drm_gem_unlock_reservations((struct drm_gem_object **)job->bos, buf_count, &acquire_ctx);
 750
 751         wmb(); /* Flush write combining buffers */
 752
 753         return ret;
 754 }
 755
 756 static int ivpu_submit(struct drm_file *file, struct ivpu_file_priv *file_priv, u32 cmdq_id,
 757                        u32 buffer_count, u32 engine, void __user *buffers_ptr, u32 cmds_offset,
 758                        u8 priority)
 759 {
 760         struct ivpu_device *vdev = file_priv->vdev;
 761         struct ivpu_job *job;
 762         u32 *buf_handles;
 763         int idx, ret;
 764
 765         buf_handles = kcalloc(buffer_count, sizeof(u32), GFP_KERNEL);
 766         if (!buf_handles)
 767                 return -ENOMEM;
 768
 769         ret = copy_from_user(buf_handles, buffers_ptr, buffer_count * sizeof(u32));
 770         if (ret) {
 771                 ret = -EFAULT;
 772                 goto err_free_handles;
 773         }
 774
 775         if (!drm_dev_enter(&vdev->drm, &idx)) {
 776                 ret = -ENODEV;
 777                 goto err_free_handles;
 778         }
 779
 780         ivpu_dbg(vdev, JOB, "Submit ioctl: ctx %u buf_count %u\n", file_priv->ctx.id, buffer_count);
 781
 782         job = ivpu_job_create(file_priv, engine, buffer_count);
 783         if (!job) {
 784                 ivpu_err(vdev, "Failed to create job\n");
 785                 ret = -ENOMEM;
 786                 goto err_exit_dev;
 787         }
 788
 789         ret = ivpu_job_prepare_bos_for_submit(file, job, buf_handles, buffer_count, cmds_offset);
 790         if (ret) {
 791                 ivpu_err(vdev, "Failed to prepare job: %d\n", ret);
 792                 goto err_destroy_job;
 793         }
 794
 795         down_read(&vdev->pm->reset_lock);
 796         ret = ivpu_job_submit(job, priority, cmdq_id);
 797         up_read(&vdev->pm->reset_lock);
 798         if (ret)
 799                 goto err_signal_fence;
 800
 801         drm_dev_exit(idx);
 802         kfree(buf_handles);
 803         return ret;
 804
 805 err_signal_fence:
 806         dma_fence_signal(job->done_fence);
 807 err_destroy_job:
 808         ivpu_job_destroy(job);
 809 err_exit_dev:
 810         drm_dev_exit(idx);
 811 err_free_handles:
 812         kfree(buf_handles);
 813         return ret;
 814 }
 815
 816 int ivpu_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 817 {
 818         struct ivpu_file_priv *file_priv = file->driver_priv;
 819         struct drm_ivpu_submit *args = data;
 820         u8 priority;
 821
 822         if (args->engine != DRM_IVPU_ENGINE_COMPUTE)
 823                 return -EINVAL;
 824
 825         if (args->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
 826                 return -EINVAL;
 827
 828         if (args->buffer_count == 0 || args->buffer_count > JOB_MAX_BUFFER_COUNT)
 829                 return -EINVAL;
 830
 831         if (!IS_ALIGNED(args->commands_offset, 8))
 832                 return -EINVAL;
 833
 834         if (!file_priv->ctx.id)
 835                 return -EINVAL;
 836
 837         if (file_priv->has_mmu_faults)
 838                 return -EBADFD;
 839
 840         priority = ivpu_job_to_jsm_priority(args->priority);
 841
 842         return ivpu_submit(file, file_priv, 0, args->buffer_count, args->engine,
 843                            (void __user *)args->buffers_ptr, args->commands_offset, priority);
 844 }
 845
 846 int ivpu_cmdq_submit_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 847 {
 848         struct ivpu_file_priv *file_priv = file->driver_priv;
 849         struct drm_ivpu_cmdq_submit *args = data;
 850
 851         if (!ivpu_is_capable(file_priv->vdev, DRM_IVPU_CAP_MANAGE_CMDQ))
 852                 return -ENODEV;
 853
 854         if (args->cmdq_id < IVPU_CMDQ_MIN_ID || args->cmdq_id > IVPU_CMDQ_MAX_ID)
 855                 return -EINVAL;
 856
 857         if (args->buffer_count == 0 || args->buffer_count > JOB_MAX_BUFFER_COUNT)
 858                 return -EINVAL;
 859
 860         if (!IS_ALIGNED(args->commands_offset, 8))
 861                 return -EINVAL;
 862
 863         if (!file_priv->ctx.id)
 864                 return -EINVAL;
 865
 866         if (file_priv->has_mmu_faults)
 867                 return -EBADFD;
 868
 869         return ivpu_submit(file, file_priv, args->cmdq_id, args->buffer_count, VPU_ENGINE_COMPUTE,
 870                            (void __user *)args->buffers_ptr, args->commands_offset, 0);
 871 }
 872
 873 int ivpu_cmdq_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 874 {
 875         struct ivpu_file_priv *file_priv = file->driver_priv;
 876         struct drm_ivpu_cmdq_create *args = data;
 877         struct ivpu_cmdq *cmdq;
 878
 879         if (!ivpu_is_capable(file_priv->vdev, DRM_IVPU_CAP_MANAGE_CMDQ))
 880                 return -ENODEV;
 881
 882         if (args->priority > DRM_IVPU_JOB_PRIORITY_REALTIME)
 883                 return -EINVAL;
 884
 885         mutex_lock(&file_priv->lock);
 886
 887         cmdq = ivpu_cmdq_create(file_priv, ivpu_job_to_jsm_priority(args->priority), false);
 888         if (cmdq)
 889                 args->cmdq_id = cmdq->id;
 890
 891         mutex_unlock(&file_priv->lock);
 892
 893         return cmdq ? 0 : -ENOMEM;
 894 }
 895
 896 int ivpu_cmdq_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 897 {
 898         struct ivpu_file_priv *file_priv = file->driver_priv;
 899         struct ivpu_device *vdev = file_priv->vdev;
 900         struct drm_ivpu_cmdq_destroy *args = data;
 901         struct ivpu_cmdq *cmdq;
 902         u32 cmdq_id;
 903         int ret;
 904
 905         if (!ivpu_is_capable(vdev, DRM_IVPU_CAP_MANAGE_CMDQ))
 906                 return -ENODEV;
 907
 908         mutex_lock(&file_priv->lock);
 909
 910         cmdq = xa_load(&file_priv->cmdq_xa, args->cmdq_id);
 911         if (!cmdq || cmdq->is_legacy) {
 912                 ret = -ENOENT;
 913                 goto err_unlock;
 914         }
 915
 916         cmdq_id = cmdq->id;
 917         ivpu_cmdq_destroy(file_priv, cmdq);
 918         mutex_unlock(&file_priv->lock);
 919         ivpu_cmdq_abort_all_jobs(vdev, file_priv->ctx.id, cmdq_id);
 920         return 0;
 921
 922 err_unlock:
 923         mutex_unlock(&file_priv->lock);
 924         return ret;
 925 }
 926
 927 static void
 928 ivpu_job_done_callback(struct ivpu_device *vdev, struct ivpu_ipc_hdr *ipc_hdr,
 929                        struct vpu_jsm_msg *jsm_msg)
 930 {
 931         struct vpu_ipc_msg_payload_job_done *payload;
 932
 933         if (!jsm_msg) {
 934                 ivpu_err(vdev, "IPC message has no JSM payload\n");
 935                 return;
 936         }
 937
 938         if (jsm_msg->result != VPU_JSM_STATUS_SUCCESS) {
 939                 ivpu_err(vdev, "Invalid JSM message result: %d\n", jsm_msg->result);
 940                 return;
 941         }
 942
 943         payload = (struct vpu_ipc_msg_payload_job_done *)&jsm_msg->payload;
 944
 945         mutex_lock(&vdev->submitted_jobs_lock);
 946         ivpu_job_signal_and_destroy(vdev, payload->job_id, payload->job_status);
 947         mutex_unlock(&vdev->submitted_jobs_lock);
 948 }
 949
 950 void ivpu_job_done_consumer_init(struct ivpu_device *vdev)
 951 {
 952         ivpu_ipc_consumer_add(vdev, &vdev->job_done_consumer,
 953                               VPU_IPC_CHAN_JOB_RET, ivpu_job_done_callback);
 954 }
 955
 956 void ivpu_job_done_consumer_fini(struct ivpu_device *vdev)
 957 {
 958         ivpu_ipc_consumer_del(vdev, &vdev->job_done_consumer);
 959 }
 960
 961 void ivpu_context_abort_work_fn(struct work_struct *work)
 962 {
 963         struct ivpu_device *vdev = container_of(work, struct ivpu_device, context_abort_work);
 964         struct ivpu_file_priv *file_priv;
 965         struct ivpu_job *job;
 966         unsigned long ctx_id;
 967         unsigned long id;
 968
 969         if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW)
 970                 ivpu_jsm_reset_engine(vdev, 0);
 971
 972         mutex_lock(&vdev->context_list_lock);
 973         xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
 974                 if (!file_priv->has_mmu_faults || file_priv->aborted)
 975                         continue;
 976
 977                 mutex_lock(&file_priv->lock);
 978                 ivpu_context_abort_locked(file_priv);
 979                 mutex_unlock(&file_priv->lock);
 980         }
 981         mutex_unlock(&vdev->context_list_lock);
 982
 983         if (vdev->fw->sched_mode != VPU_SCHEDULING_MODE_HW)
 984                 return;
 985
 986         ivpu_jsm_hws_resume_engine(vdev, 0);
 987         /*
 988          * In hardware scheduling mode NPU already has stopped processing jobs
 989          * and won't send us any further notifications, thus we have to free job related resources
 990          * and notify userspace
 991          */
 992         mutex_lock(&vdev->submitted_jobs_lock);
 993         xa_for_each(&vdev->submitted_jobs_xa, id, job)
 994                 if (job->file_priv->aborted)
 995                         ivpu_job_signal_and_destroy(vdev, job->job_id, DRM_IVPU_JOB_STATUS_ABORTED);
 996         mutex_unlock(&vdev->submitted_jobs_lock);
 997 }