drivers/accel/ivpu/ivpu_hw.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 - 2024 Intel Corporation
   4  */
   5
   6 #include "ivpu_drv.h"
   7 #include "ivpu_hw.h"
   8 #include "ivpu_hw_btrs.h"
   9 #include "ivpu_hw_ip.h"
  10
  11 #include <linux/dmi.h>
  12
  13 static char *platform_to_str(u32 platform)
  14 {
  15         switch (platform) {
  16         case IVPU_PLATFORM_SILICON:
  17                 return "SILICON";
  18         case IVPU_PLATFORM_SIMICS:
  19                 return "SIMICS";
  20         case IVPU_PLATFORM_FPGA:
  21                 return "FPGA";
  22         default:
  23                 return "Invalid platform";
  24         }
  25 }
  26
  27 static const struct dmi_system_id dmi_platform_simulation[] = {
  28         {
  29                 .ident = "Intel Simics",
  30                 .matches = {
  31                         DMI_MATCH(DMI_BOARD_NAME, "lnlrvp"),
  32                         DMI_MATCH(DMI_BOARD_VERSION, "1.0"),
  33                         DMI_MATCH(DMI_BOARD_SERIAL, "123456789"),
  34                 },
  35         },
  36         {
  37                 .ident = "Intel Simics",
  38                 .matches = {
  39                         DMI_MATCH(DMI_BOARD_NAME, "Simics"),
  40                 },
  41         },
  42         { }
  43 };
  44
  45 static void platform_init(struct ivpu_device *vdev)
  46 {
  47         if (dmi_check_system(dmi_platform_simulation))
  48                 vdev->platform = IVPU_PLATFORM_SIMICS;
  49         else
  50                 vdev->platform = IVPU_PLATFORM_SILICON;
  51
  52         ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n",
  53                  platform_to_str(vdev->platform), vdev->platform);
  54 }
  55
  56 static void wa_init(struct ivpu_device *vdev)
  57 {
  58         vdev->wa.punit_disabled = ivpu_is_fpga(vdev);
  59         vdev->wa.clear_runtime_mem = false;
  60
  61         if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
  62                 vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev);
  63
  64         if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL &&
  65             ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0)
  66                 vdev->wa.disable_clock_relinquish = true;
  67
  68         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
  69                 vdev->wa.wp0_during_power_up = true;
  70
  71         IVPU_PRINT_WA(punit_disabled);
  72         IVPU_PRINT_WA(clear_runtime_mem);
  73         IVPU_PRINT_WA(interrupt_clear_with_0);
  74         IVPU_PRINT_WA(disable_clock_relinquish);
  75         IVPU_PRINT_WA(wp0_during_power_up);
  76 }
  77
  78 static void timeouts_init(struct ivpu_device *vdev)
  79 {
  80         if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) {
  81                 vdev->timeout.boot = -1;
  82                 vdev->timeout.jsm = -1;
  83                 vdev->timeout.tdr = -1;
  84                 vdev->timeout.autosuspend = -1;
  85                 vdev->timeout.d0i3_entry_msg = -1;
  86         } else if (ivpu_is_fpga(vdev)) {
  87                 vdev->timeout.boot = 100000;
  88                 vdev->timeout.jsm = 50000;
  89                 vdev->timeout.tdr = 2000000;
  90                 vdev->timeout.autosuspend = -1;
  91                 vdev->timeout.d0i3_entry_msg = 500;
  92                 vdev->timeout.state_dump_msg = 10;
  93         } else if (ivpu_is_simics(vdev)) {
  94                 vdev->timeout.boot = 50;
  95                 vdev->timeout.jsm = 500;
  96                 vdev->timeout.tdr = 10000;
  97                 vdev->timeout.autosuspend = 100;
  98                 vdev->timeout.d0i3_entry_msg = 100;
  99                 vdev->timeout.state_dump_msg = 10;
 100         } else {
 101                 vdev->timeout.boot = 1000;
 102                 vdev->timeout.jsm = 500;
 103                 vdev->timeout.tdr = 2000;
 104                 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 105                         vdev->timeout.autosuspend = 10;
 106                 else
 107                         vdev->timeout.autosuspend = 100;
 108                 vdev->timeout.d0i3_entry_msg = 5;
 109                 vdev->timeout.state_dump_msg = 10;
 110         }
 111 }
 112
 113 static void memory_ranges_init(struct ivpu_device *vdev)
 114 {
 115         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 116                 ivpu_hw_range_init(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
 117                 ivpu_hw_range_init(&vdev->hw->ranges.user,   0x88000000, 511 * SZ_1M);
 118                 ivpu_hw_range_init(&vdev->hw->ranges.shave, 0x180000000, SZ_2G);
 119                 ivpu_hw_range_init(&vdev->hw->ranges.dma,   0x200000000, SZ_128G);
 120         } else {
 121                 ivpu_hw_range_init(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
 122                 ivpu_hw_range_init(&vdev->hw->ranges.shave,  0x80000000, SZ_2G);
 123                 ivpu_hw_range_init(&vdev->hw->ranges.user,  0x100000000, SZ_256G);
 124                 vdev->hw->ranges.dma = vdev->hw->ranges.user;
 125         }
 126 }
 127
 128 static int wp_enable(struct ivpu_device *vdev)
 129 {
 130         return ivpu_hw_btrs_wp_drive(vdev, true);
 131 }
 132
 133 static int wp_disable(struct ivpu_device *vdev)
 134 {
 135         return ivpu_hw_btrs_wp_drive(vdev, false);
 136 }
 137
 138 int ivpu_hw_power_up(struct ivpu_device *vdev)
 139 {
 140         int ret;
 141
 142         if (IVPU_WA(wp0_during_power_up)) {
 143                 /* WP requests may fail when powering down, so issue WP 0 here */
 144                 ret = wp_disable(vdev);
 145                 if (ret)
 146                         ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret);
 147         }
 148
 149         ret = ivpu_hw_btrs_d0i3_disable(vdev);
 150         if (ret)
 151                 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
 152
 153         ret = wp_enable(vdev);
 154         if (ret) {
 155                 ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret);
 156                 return ret;
 157         }
 158
 159         if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) {
 160                 if (IVPU_WA(disable_clock_relinquish))
 161                         ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev);
 162                 ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev);
 163                 ivpu_hw_btrs_ats_print_lnl(vdev);
 164         }
 165
 166         ret = ivpu_hw_ip_host_ss_configure(vdev);
 167         if (ret) {
 168                 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
 169                 return ret;
 170         }
 171
 172         ivpu_hw_ip_idle_gen_disable(vdev);
 173
 174         ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev);
 175         if (ret) {
 176                 ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n");
 177                 return ret;
 178         }
 179
 180         ret = ivpu_hw_ip_pwr_domain_enable(vdev);
 181         if (ret) {
 182                 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
 183                 return ret;
 184         }
 185
 186         ret = ivpu_hw_ip_host_ss_axi_enable(vdev);
 187         if (ret) {
 188                 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
 189                 return ret;
 190         }
 191
 192         if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL)
 193                 ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev);
 194
 195         ret = ivpu_hw_ip_top_noc_enable(vdev);
 196         if (ret)
 197                 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
 198
 199         return ret;
 200 }
 201
 202 static void save_d0i3_entry_timestamp(struct ivpu_device *vdev)
 203 {
 204         vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
 205         vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev);
 206 }
 207
 208 int ivpu_hw_reset(struct ivpu_device *vdev)
 209 {
 210         int ret = 0;
 211
 212         if (ivpu_hw_btrs_ip_reset(vdev)) {
 213                 ivpu_err(vdev, "Failed to reset NPU IP\n");
 214                 ret = -EIO;
 215         }
 216
 217         if (wp_disable(vdev)) {
 218                 ivpu_err(vdev, "Failed to disable workpoint\n");
 219                 ret = -EIO;
 220         }
 221
 222         return ret;
 223 }
 224
 225 int ivpu_hw_power_down(struct ivpu_device *vdev)
 226 {
 227         int ret = 0;
 228
 229         save_d0i3_entry_timestamp(vdev);
 230
 231         if (!ivpu_hw_is_idle(vdev))
 232                 ivpu_warn(vdev, "NPU not idle during power down\n");
 233
 234         if (ivpu_hw_reset(vdev)) {
 235                 ivpu_err(vdev, "Failed to reset NPU\n");
 236                 ret = -EIO;
 237         }
 238
 239         if (ivpu_hw_btrs_d0i3_enable(vdev)) {
 240                 ivpu_err(vdev, "Failed to enter D0I3\n");
 241                 ret = -EIO;
 242         }
 243
 244         return ret;
 245 }
 246
 247 int ivpu_hw_init(struct ivpu_device *vdev)
 248 {
 249         ivpu_hw_btrs_info_init(vdev);
 250         ivpu_hw_btrs_freq_ratios_init(vdev);
 251         memory_ranges_init(vdev);
 252         platform_init(vdev);
 253         wa_init(vdev);
 254         timeouts_init(vdev);
 255         atomic_set(&vdev->hw->firewall_irq_counter, 0);
 256
 257         return 0;
 258 }
 259
 260 int ivpu_hw_boot_fw(struct ivpu_device *vdev)
 261 {
 262         int ret;
 263
 264         ivpu_hw_ip_snoop_disable(vdev);
 265         ivpu_hw_ip_tbu_mmu_enable(vdev);
 266         ret = ivpu_hw_ip_soc_cpu_boot(vdev);
 267         if (ret)
 268                 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
 269
 270         return ret;
 271 }
 272
 273 void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
 274 {
 275         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 276                 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
 277                 return;
 278         }
 279
 280         if (enable)
 281                 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
 282         else
 283                 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
 284 }
 285
 286 void ivpu_irq_handlers_init(struct ivpu_device *vdev)
 287 {
 288         INIT_KFIFO(vdev->hw->irq.fifo);
 289
 290         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 291                 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx;
 292         else
 293                 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx;
 294
 295         if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
 296                 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl;
 297         else
 298                 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl;
 299 }
 300
 301 void ivpu_hw_irq_enable(struct ivpu_device *vdev)
 302 {
 303         kfifo_reset(&vdev->hw->irq.fifo);
 304         ivpu_hw_ip_irq_enable(vdev);
 305         ivpu_hw_btrs_irq_enable(vdev);
 306 }
 307
 308 void ivpu_hw_irq_disable(struct ivpu_device *vdev)
 309 {
 310         ivpu_hw_btrs_irq_disable(vdev);
 311         ivpu_hw_ip_irq_disable(vdev);
 312 }
 313
 314 irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr)
 315 {
 316         struct ivpu_device *vdev = ptr;
 317         bool ip_handled, btrs_handled;
 318
 319         ivpu_hw_btrs_global_int_disable(vdev);
 320
 321         btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq);
 322         if (!ivpu_hw_is_idle((vdev)) || !btrs_handled)
 323                 ip_handled = ivpu_hw_ip_irq_handler(vdev, irq);
 324         else
 325                 ip_handled = false;
 326
 327         /* Re-enable global interrupts to re-trigger MSI for pending interrupts */
 328         ivpu_hw_btrs_global_int_enable(vdev);
 329
 330         if (!kfifo_is_empty(&vdev->hw->irq.fifo))
 331                 return IRQ_WAKE_THREAD;
 332         if (ip_handled || btrs_handled)
 333                 return IRQ_HANDLED;
 334         return IRQ_NONE;
 335 }