drivers/accel/ivpu/ivpu_pm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2024 Intel Corporation
   4  */
   5
   6 #include <linux/highmem.h>
   7 #include <linux/moduleparam.h>
   8 #include <linux/pci.h>
   9 #include <linux/pm_runtime.h>
  10 #include <linux/reboot.h>
  11
  12 #include "ivpu_coredump.h"
  13 #include "ivpu_drv.h"
  14 #include "ivpu_fw.h"
  15 #include "ivpu_fw_log.h"
  16 #include "ivpu_hw.h"
  17 #include "ivpu_ipc.h"
  18 #include "ivpu_job.h"
  19 #include "ivpu_jsm_msg.h"
  20 #include "ivpu_mmu.h"
  21 #include "ivpu_ms.h"
  22 #include "ivpu_pm.h"
  23 #include "ivpu_trace.h"
  24 #include "vpu_boot_api.h"
  25
  26 static bool ivpu_disable_recovery;
  27 #if IS_ENABLED(CONFIG_DRM_ACCEL_IVPU_DEBUG)
  28 module_param_named_unsafe(disable_recovery, ivpu_disable_recovery, bool, 0644);
  29 MODULE_PARM_DESC(disable_recovery, "Disables recovery when NPU hang is detected");
  30 #endif
  31
  32 static unsigned long ivpu_tdr_timeout_ms;
  33 module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, ulong, 0644);
  34 MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");
  35
  36 #define PM_RESCHEDULE_LIMIT     5
  37
  38 static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev)
  39 {
  40         struct ivpu_fw_info *fw = vdev->fw;
  41
  42         ivpu_cmdq_reset_all_contexts(vdev);
  43         ivpu_ipc_reset(vdev);
  44         ivpu_fw_log_reset(vdev);
  45         ivpu_fw_load(vdev);
  46         fw->entry_point = fw->cold_boot_entry_point;
  47 }
  48
  49 static void ivpu_pm_prepare_warm_boot(struct ivpu_device *vdev)
  50 {
  51         struct ivpu_fw_info *fw = vdev->fw;
  52         struct vpu_boot_params *bp = ivpu_bo_vaddr(fw->mem);
  53
  54         if (!bp->save_restore_ret_address) {
  55                 ivpu_pm_prepare_cold_boot(vdev);
  56                 return;
  57         }
  58
  59         ivpu_dbg(vdev, FW_BOOT, "Save/restore entry point %llx", bp->save_restore_ret_address);
  60         fw->entry_point = bp->save_restore_ret_address;
  61 }
  62
  63 static int ivpu_suspend(struct ivpu_device *vdev)
  64 {
  65         int ret;
  66
  67         ivpu_prepare_for_reset(vdev);
  68
  69         ret = ivpu_shutdown(vdev);
  70         if (ret)
  71                 ivpu_err(vdev, "Failed to shutdown NPU: %d\n", ret);
  72
  73         return ret;
  74 }
  75
  76 static int ivpu_resume(struct ivpu_device *vdev)
  77 {
  78         int ret;
  79
  80 retry:
  81         pci_restore_state(to_pci_dev(vdev->drm.dev));
  82         pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D0);
  83
  84         ret = ivpu_hw_power_up(vdev);
  85         if (ret) {
  86                 ivpu_err(vdev, "Failed to power up HW: %d\n", ret);
  87                 goto err_power_down;
  88         }
  89
  90         ret = ivpu_mmu_enable(vdev);
  91         if (ret) {
  92                 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);
  93                 goto err_power_down;
  94         }
  95
  96         ret = ivpu_boot(vdev);
  97         if (ret)
  98                 goto err_mmu_disable;
  99
 100         return 0;
 101
 102 err_mmu_disable:
 103         ivpu_mmu_disable(vdev);
 104 err_power_down:
 105         ivpu_hw_power_down(vdev);
 106         pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);
 107
 108         if (!ivpu_fw_is_cold_boot(vdev)) {
 109                 ivpu_pm_prepare_cold_boot(vdev);
 110                 goto retry;
 111         } else {
 112                 ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);
 113         }
 114
 115         return ret;
 116 }
 117
 118 static void ivpu_pm_recovery_work(struct work_struct *work)
 119 {
 120         struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, recovery_work);
 121         struct ivpu_device *vdev = pm->vdev;
 122         char *evt[2] = {"IVPU_PM_EVENT=IVPU_RECOVER", NULL};
 123         int ret;
 124
 125         ivpu_err(vdev, "Recovering the NPU (reset #%d)\n", atomic_read(&vdev->pm->reset_counter));
 126
 127         ret = pm_runtime_resume_and_get(vdev->drm.dev);
 128         if (ret)
 129                 ivpu_err(vdev, "Failed to resume NPU: %d\n", ret);
 130
 131         ivpu_jsm_state_dump(vdev);
 132         ivpu_dev_coredump(vdev);
 133
 134         atomic_inc(&vdev->pm->reset_counter);
 135         atomic_set(&vdev->pm->reset_pending, 1);
 136         down_write(&vdev->pm->reset_lock);
 137
 138         ivpu_suspend(vdev);
 139         ivpu_pm_prepare_cold_boot(vdev);
 140         ivpu_jobs_abort_all(vdev);
 141         ivpu_ms_cleanup_all(vdev);
 142
 143         ret = ivpu_resume(vdev);
 144         if (ret)
 145                 ivpu_err(vdev, "Failed to resume NPU: %d\n", ret);
 146
 147         up_write(&vdev->pm->reset_lock);
 148         atomic_set(&vdev->pm->reset_pending, 0);
 149
 150         kobject_uevent_env(&vdev->drm.dev->kobj, KOBJ_CHANGE, evt);
 151         pm_runtime_mark_last_busy(vdev->drm.dev);
 152         pm_runtime_put_autosuspend(vdev->drm.dev);
 153 }
 154
 155 void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason)
 156 {
 157         ivpu_err(vdev, "Recovery triggered by %s\n", reason);
 158
 159         if (ivpu_disable_recovery) {
 160                 ivpu_err(vdev, "Recovery not available when disable_recovery param is set\n");
 161                 return;
 162         }
 163
 164         if (ivpu_is_fpga(vdev)) {
 165                 ivpu_err(vdev, "Recovery not available on FPGA\n");
 166                 return;
 167         }
 168
 169         /* Trigger recovery if it's not in progress */
 170         if (atomic_cmpxchg(&vdev->pm->reset_pending, 0, 1) == 0) {
 171                 ivpu_hw_diagnose_failure(vdev);
 172                 ivpu_hw_irq_disable(vdev); /* Disable IRQ early to protect from IRQ storm */
 173                 queue_work(system_long_wq, &vdev->pm->recovery_work);
 174         }
 175 }
 176
 177 static void ivpu_job_timeout_work(struct work_struct *work)
 178 {
 179         struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, job_timeout_work.work);
 180         struct ivpu_device *vdev = pm->vdev;
 181
 182         ivpu_pm_trigger_recovery(vdev, "TDR");
 183 }
 184
 185 void ivpu_start_job_timeout_detection(struct ivpu_device *vdev)
 186 {
 187         unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;
 188
 189         /* No-op if already queued */
 190         queue_delayed_work(system_wq, &vdev->pm->job_timeout_work, msecs_to_jiffies(timeout_ms));
 191 }
 192
 193 void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev)
 194 {
 195         cancel_delayed_work_sync(&vdev->pm->job_timeout_work);
 196 }
 197
 198 int ivpu_pm_suspend_cb(struct device *dev)
 199 {
 200         struct drm_device *drm = dev_get_drvdata(dev);
 201         struct ivpu_device *vdev = to_ivpu_device(drm);
 202         unsigned long timeout;
 203
 204         trace_pm("suspend");
 205         ivpu_dbg(vdev, PM, "Suspend..\n");
 206
 207         timeout = jiffies + msecs_to_jiffies(vdev->timeout.tdr);
 208         while (!ivpu_hw_is_idle(vdev)) {
 209                 cond_resched();
 210                 if (time_after_eq(jiffies, timeout)) {
 211                         ivpu_err(vdev, "Failed to enter idle on system suspend\n");
 212                         return -EBUSY;
 213                 }
 214         }
 215
 216         ivpu_jsm_pwr_d0i3_enter(vdev);
 217
 218         ivpu_suspend(vdev);
 219         ivpu_pm_prepare_warm_boot(vdev);
 220
 221         ivpu_dbg(vdev, PM, "Suspend done.\n");
 222         trace_pm("suspend done");
 223
 224         return 0;
 225 }
 226
 227 int ivpu_pm_resume_cb(struct device *dev)
 228 {
 229         struct drm_device *drm = dev_get_drvdata(dev);
 230         struct ivpu_device *vdev = to_ivpu_device(drm);
 231         int ret;
 232
 233         trace_pm("resume");
 234         ivpu_dbg(vdev, PM, "Resume..\n");
 235
 236         ret = ivpu_resume(vdev);
 237         if (ret)
 238                 ivpu_err(vdev, "Failed to resume: %d\n", ret);
 239
 240         ivpu_dbg(vdev, PM, "Resume done.\n");
 241         trace_pm("resume done");
 242
 243         return ret;
 244 }
 245
 246 int ivpu_pm_runtime_suspend_cb(struct device *dev)
 247 {
 248         struct drm_device *drm = dev_get_drvdata(dev);
 249         struct ivpu_device *vdev = to_ivpu_device(drm);
 250         int ret, ret_d0i3;
 251         bool is_idle;
 252
 253         drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));
 254         drm_WARN_ON(&vdev->drm, work_pending(&vdev->pm->recovery_work));
 255
 256         trace_pm("runtime suspend");
 257         ivpu_dbg(vdev, PM, "Runtime suspend..\n");
 258
 259         ivpu_mmu_disable(vdev);
 260
 261         is_idle = ivpu_hw_is_idle(vdev) || vdev->pm->dct_active_percent;
 262         if (!is_idle)
 263                 ivpu_err(vdev, "NPU is not idle before autosuspend\n");
 264
 265         ret_d0i3 = ivpu_jsm_pwr_d0i3_enter(vdev);
 266         if (ret_d0i3)
 267                 ivpu_err(vdev, "Failed to prepare for d0i3: %d\n", ret_d0i3);
 268
 269         ret = ivpu_suspend(vdev);
 270         if (ret)
 271                 ivpu_err(vdev, "Failed to suspend NPU: %d\n", ret);
 272
 273         if (!is_idle || ret_d0i3) {
 274                 ivpu_err(vdev, "Forcing cold boot due to previous errors\n");
 275                 atomic_inc(&vdev->pm->reset_counter);
 276                 ivpu_dev_coredump(vdev);
 277                 ivpu_pm_prepare_cold_boot(vdev);
 278         } else {
 279                 ivpu_pm_prepare_warm_boot(vdev);
 280         }
 281
 282         ivpu_dbg(vdev, PM, "Runtime suspend done.\n");
 283         trace_pm("runtime suspend done");
 284
 285         return 0;
 286 }
 287
 288 int ivpu_pm_runtime_resume_cb(struct device *dev)
 289 {
 290         struct drm_device *drm = dev_get_drvdata(dev);
 291         struct ivpu_device *vdev = to_ivpu_device(drm);
 292         int ret;
 293
 294         trace_pm("runtime resume");
 295         ivpu_dbg(vdev, PM, "Runtime resume..\n");
 296
 297         ret = ivpu_resume(vdev);
 298         if (ret)
 299                 ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);
 300
 301         ivpu_dbg(vdev, PM, "Runtime resume done.\n");
 302         trace_pm("runtime resume done");
 303
 304         return ret;
 305 }
 306
 307 int ivpu_rpm_get(struct ivpu_device *vdev)
 308 {
 309         int ret;
 310
 311         ret = pm_runtime_resume_and_get(vdev->drm.dev);
 312         drm_WARN_ON(&vdev->drm, ret < 0);
 313
 314         return ret;
 315 }
 316
 317 void ivpu_rpm_put(struct ivpu_device *vdev)
 318 {
 319         pm_runtime_mark_last_busy(vdev->drm.dev);
 320         pm_runtime_put_autosuspend(vdev->drm.dev);
 321 }
 322
 323 void ivpu_pm_reset_prepare_cb(struct pci_dev *pdev)
 324 {
 325         struct ivpu_device *vdev = pci_get_drvdata(pdev);
 326
 327         ivpu_dbg(vdev, PM, "Pre-reset..\n");
 328         atomic_inc(&vdev->pm->reset_counter);
 329         atomic_set(&vdev->pm->reset_pending, 1);
 330
 331         pm_runtime_get_sync(vdev->drm.dev);
 332         down_write(&vdev->pm->reset_lock);
 333         ivpu_prepare_for_reset(vdev);
 334         ivpu_hw_reset(vdev);
 335         ivpu_pm_prepare_cold_boot(vdev);
 336         ivpu_jobs_abort_all(vdev);
 337         ivpu_ms_cleanup_all(vdev);
 338
 339         ivpu_dbg(vdev, PM, "Pre-reset done.\n");
 340 }
 341
 342 void ivpu_pm_reset_done_cb(struct pci_dev *pdev)
 343 {
 344         struct ivpu_device *vdev = pci_get_drvdata(pdev);
 345         int ret;
 346
 347         ivpu_dbg(vdev, PM, "Post-reset..\n");
 348         ret = ivpu_resume(vdev);
 349         if (ret)
 350                 ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);
 351         up_write(&vdev->pm->reset_lock);
 352         atomic_set(&vdev->pm->reset_pending, 0);
 353         ivpu_dbg(vdev, PM, "Post-reset done.\n");
 354
 355         pm_runtime_mark_last_busy(vdev->drm.dev);
 356         pm_runtime_put_autosuspend(vdev->drm.dev);
 357 }
 358
 359 void ivpu_pm_init(struct ivpu_device *vdev)
 360 {
 361         struct device *dev = vdev->drm.dev;
 362         struct ivpu_pm_info *pm = vdev->pm;
 363         int delay;
 364
 365         pm->vdev = vdev;
 366
 367         init_rwsem(&pm->reset_lock);
 368         atomic_set(&pm->reset_pending, 0);
 369         atomic_set(&pm->reset_counter, 0);
 370
 371         INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work);
 372         INIT_DELAYED_WORK(&pm->job_timeout_work, ivpu_job_timeout_work);
 373
 374         if (ivpu_disable_recovery)
 375                 delay = -1;
 376         else
 377                 delay = vdev->timeout.autosuspend;
 378
 379         pm_runtime_use_autosuspend(dev);
 380         pm_runtime_set_autosuspend_delay(dev, delay);
 381
 382         ivpu_dbg(vdev, PM, "Autosuspend delay = %d\n", delay);
 383 }
 384
 385 void ivpu_pm_cancel_recovery(struct ivpu_device *vdev)
 386 {
 387         drm_WARN_ON(&vdev->drm, delayed_work_pending(&vdev->pm->job_timeout_work));
 388         cancel_work_sync(&vdev->pm->recovery_work);
 389 }
 390
 391 void ivpu_pm_enable(struct ivpu_device *vdev)
 392 {
 393         struct device *dev = vdev->drm.dev;
 394
 395         pm_runtime_set_active(dev);
 396         pm_runtime_allow(dev);
 397         pm_runtime_mark_last_busy(dev);
 398         pm_runtime_put_autosuspend(dev);
 399 }
 400
 401 void ivpu_pm_disable(struct ivpu_device *vdev)
 402 {
 403         pm_runtime_get_noresume(vdev->drm.dev);
 404         pm_runtime_forbid(vdev->drm.dev);
 405 }
 406
 407 int ivpu_pm_dct_init(struct ivpu_device *vdev)
 408 {
 409         if (vdev->pm->dct_active_percent)
 410                 return ivpu_pm_dct_enable(vdev, vdev->pm->dct_active_percent);
 411
 412         return 0;
 413 }
 414
 415 int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent)
 416 {
 417         u32 active_us, inactive_us;
 418         int ret;
 419
 420         if (active_percent == 0 || active_percent > 100)
 421                 return -EINVAL;
 422
 423         active_us = (DCT_PERIOD_US * active_percent) / 100;
 424         inactive_us = DCT_PERIOD_US - active_us;
 425
 426         ret = ivpu_jsm_dct_enable(vdev, active_us, inactive_us);
 427         if (ret) {
 428                 ivpu_err_ratelimited(vdev, "Failed to enable DCT: %d\n", ret);
 429                 return ret;
 430         }
 431
 432         vdev->pm->dct_active_percent = active_percent;
 433
 434         ivpu_dbg(vdev, PM, "DCT set to %u%% (D0: %uus, D0i2: %uus)\n",
 435                  active_percent, active_us, inactive_us);
 436         return 0;
 437 }
 438
 439 int ivpu_pm_dct_disable(struct ivpu_device *vdev)
 440 {
 441         int ret;
 442
 443         ret = ivpu_jsm_dct_disable(vdev);
 444         if (ret) {
 445                 ivpu_err_ratelimited(vdev, "Failed to disable DCT: %d\n", ret);
 446                 return ret;
 447         }
 448
 449         vdev->pm->dct_active_percent = 0;
 450
 451         ivpu_dbg(vdev, PM, "DCT disabled\n");
 452         return 0;
 453 }
 454
 455 void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev)
 456 {
 457         bool enable;
 458         int ret;
 459
 460         if (ivpu_hw_btrs_dct_get_request(vdev, &enable))
 461                 return;
 462
 463         if (vdev->pm->dct_active_percent)
 464                 ret = ivpu_pm_dct_enable(vdev, DCT_DEFAULT_ACTIVE_PERCENT);
 465         else
 466                 ret = ivpu_pm_dct_disable(vdev);
 467
 468         if (!ret)
 469                 ivpu_hw_btrs_dct_set_status(vdev, enable, vdev->pm->dct_active_percent);
 470 }