drivers/accel/ivpu/ivpu_hw_ip.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020-2024 Intel Corporation
   4  */
   5
   6 #include "ivpu_drv.h"
   7 #include "ivpu_fw.h"
   8 #include "ivpu_hw.h"
   9 #include "ivpu_hw_37xx_reg.h"
  10 #include "ivpu_hw_40xx_reg.h"
  11 #include "ivpu_hw_btrs.h"
  12 #include "ivpu_hw_ip.h"
  13 #include "ivpu_hw_reg_io.h"
  14 #include "ivpu_mmu.h"
  15 #include "ivpu_pm.h"
  16
  17 #define PWR_ISLAND_STATUS_TIMEOUT_US        (5 * USEC_PER_MSEC)
  18
  19 #define TIM_SAFE_ENABLE                     0xf1d0dead
  20 #define TIM_WATCHDOG_RESET_VALUE            0xffffffff
  21
  22 #define ICB_0_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
  23                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
  24                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
  25                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
  26                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
  27                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
  28                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
  29
  30 #define ICB_1_IRQ_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
  31                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
  32                              (REG_FLD(VPU_37XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
  33
  34 #define ICB_0_1_IRQ_MASK_37XX ((((u64)ICB_1_IRQ_MASK_37XX) << 32) | ICB_0_IRQ_MASK_37XX)
  35
  36 #define ICB_0_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT)) | \
  37                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT)) | \
  38                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT)) | \
  39                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT)) | \
  40                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT)) | \
  41                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT)) | \
  42                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT)))
  43
  44 #define ICB_1_IRQ_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_2_INT)) | \
  45                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_3_INT)) | \
  46                              (REG_FLD(VPU_40XX_HOST_SS_ICB_STATUS_1, CPU_INT_REDIRECT_4_INT)))
  47
  48 #define ICB_0_1_IRQ_MASK_40XX ((((u64)ICB_1_IRQ_MASK_40XX) << 32) | ICB_0_IRQ_MASK_40XX)
  49
  50 #define ITF_FIREWALL_VIOLATION_MASK_37XX ((REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
  51                                           (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
  52                                           (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
  53                                           (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
  54                                           (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
  55                                           (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
  56                                           (REG_FLD(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
  57
  58 #define ITF_FIREWALL_VIOLATION_MASK_40XX ((REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_ROM_CMX)) | \
  59                                           (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_DBG)) | \
  60                                           (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, CSS_CTRL)) | \
  61                                           (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, DEC400)) | \
  62                                           (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_NCE)) | \
  63                                           (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI)) | \
  64                                           (REG_FLD(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, MSS_MBI_CMX)))
  65
  66 static int wait_for_ip_bar(struct ivpu_device *vdev)
  67 {
  68         return REGV_POLL_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, AON, 0, 100);
  69 }
  70
  71 static void host_ss_rst_clr(struct ivpu_device *vdev)
  72 {
  73         u32 val = 0;
  74
  75         val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, TOP_NOC, val);
  76         val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, DSS_MAS, val);
  77         val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_CLR, MSS_MAS, val);
  78
  79         REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_CLR, val);
  80 }
  81
  82 static int host_ss_noc_qreqn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
  83 {
  84         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN);
  85
  86         if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
  87                 return -EIO;
  88
  89         return 0;
  90 }
  91
  92 static int host_ss_noc_qreqn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
  93 {
  94         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
  95
  96         if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, exp_val, val))
  97                 return -EIO;
  98
  99         return 0;
 100 }
 101
 102 static int host_ss_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
 103 {
 104         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 105                 return host_ss_noc_qreqn_check_37xx(vdev, exp_val);
 106         else
 107                 return host_ss_noc_qreqn_check_40xx(vdev, exp_val);
 108 }
 109
 110 static int host_ss_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 111 {
 112         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QACCEPTN);
 113
 114         if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
 115                 return -EIO;
 116
 117         return 0;
 118 }
 119
 120 static int host_ss_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 121 {
 122         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QACCEPTN);
 123
 124         if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QACCEPTN, TOP_SOCMMIO, exp_val, val))
 125                 return -EIO;
 126
 127         return 0;
 128 }
 129
 130 static int host_ss_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 131 {
 132         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 133                 return host_ss_noc_qacceptn_check_37xx(vdev, exp_val);
 134         else
 135                 return host_ss_noc_qacceptn_check_40xx(vdev, exp_val);
 136 }
 137
 138 static int host_ss_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 139 {
 140         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QDENY);
 141
 142         if (!REG_TEST_FLD_NUM(VPU_37XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
 143                 return -EIO;
 144
 145         return 0;
 146 }
 147
 148 static int host_ss_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 149 {
 150         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QDENY);
 151
 152         if (!REG_TEST_FLD_NUM(VPU_40XX_HOST_SS_NOC_QDENY, TOP_SOCMMIO, exp_val, val))
 153                 return -EIO;
 154
 155         return 0;
 156 }
 157
 158 static int host_ss_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 159 {
 160         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 161                 return host_ss_noc_qdeny_check_37xx(vdev, exp_val);
 162         else
 163                 return host_ss_noc_qdeny_check_40xx(vdev, exp_val);
 164 }
 165
 166 static int top_noc_qrenqn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 167 {
 168         u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN);
 169
 170         if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
 171             !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
 172                 return -EIO;
 173
 174         return 0;
 175 }
 176
 177 static int top_noc_qrenqn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 178 {
 179         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
 180
 181         if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, exp_val, val) ||
 182             !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, exp_val, val))
 183                 return -EIO;
 184
 185         return 0;
 186 }
 187
 188 static int top_noc_qreqn_check(struct ivpu_device *vdev, u32 exp_val)
 189 {
 190         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 191                 return top_noc_qrenqn_check_37xx(vdev, exp_val);
 192         else
 193                 return top_noc_qrenqn_check_40xx(vdev, exp_val);
 194 }
 195
 196 int ivpu_hw_ip_host_ss_configure(struct ivpu_device *vdev)
 197 {
 198         int ret;
 199
 200         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 201                 ret = wait_for_ip_bar(vdev);
 202                 if (ret) {
 203                         ivpu_err(vdev, "Timed out waiting for NPU IP bar\n");
 204                         return ret;
 205                 }
 206                 host_ss_rst_clr(vdev);
 207         }
 208
 209         ret = host_ss_noc_qreqn_check(vdev, 0x0);
 210         if (ret) {
 211                 ivpu_err(vdev, "Failed qreqn check: %d\n", ret);
 212                 return ret;
 213         }
 214
 215         ret = host_ss_noc_qacceptn_check(vdev, 0x0);
 216         if (ret) {
 217                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 218                 return ret;
 219         }
 220
 221         ret = host_ss_noc_qdeny_check(vdev, 0x0);
 222         if (ret)
 223                 ivpu_err(vdev, "Failed qdeny check %d\n", ret);
 224
 225         return ret;
 226 }
 227
 228 static void idle_gen_drive_37xx(struct ivpu_device *vdev, bool enable)
 229 {
 230         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN);
 231
 232         if (enable)
 233                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val);
 234         else
 235                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, EN, val);
 236
 237         REGV_WR32(VPU_37XX_HOST_SS_AON_VPU_IDLE_GEN, val);
 238 }
 239
 240 static void idle_gen_drive_40xx(struct ivpu_device *vdev, bool enable)
 241 {
 242         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_IDLE_GEN);
 243
 244         if (enable)
 245                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
 246         else
 247                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_IDLE_GEN, EN, val);
 248
 249         REGV_WR32(VPU_40XX_HOST_SS_AON_IDLE_GEN, val);
 250 }
 251
 252 void ivpu_hw_ip_idle_gen_enable(struct ivpu_device *vdev)
 253 {
 254         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 255                 idle_gen_drive_37xx(vdev, true);
 256         else
 257                 idle_gen_drive_40xx(vdev, true);
 258 }
 259
 260 void ivpu_hw_ip_idle_gen_disable(struct ivpu_device *vdev)
 261 {
 262         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 263                 idle_gen_drive_37xx(vdev, false);
 264         else
 265                 idle_gen_drive_40xx(vdev, false);
 266 }
 267
 268 static void
 269 pwr_island_delay_set_50xx(struct ivpu_device *vdev, u32 post, u32 post1, u32 post2, u32 status)
 270 {
 271         u32 val;
 272
 273         val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY);
 274         val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST_DLY, post, val);
 275         val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST1_DLY, post1, val);
 276         val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, POST2_DLY, post2, val);
 277         REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_EN_POST_DLY, val);
 278
 279         val = REGV_RD32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY);
 280         val = REG_SET_FLD_NUM(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, STATUS_DLY, status, val);
 281         REGV_WR32(VPU_50XX_HOST_SS_AON_PWR_ISLAND_STATUS_DLY, val);
 282 }
 283
 284 static void pwr_island_trickle_drive_37xx(struct ivpu_device *vdev, bool enable)
 285 {
 286         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
 287
 288         if (enable)
 289                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val);
 290         else
 291                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, MSS_CPU, val);
 292
 293         REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
 294 }
 295
 296 static void pwr_island_trickle_drive_40xx(struct ivpu_device *vdev, bool enable)
 297 {
 298         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0);
 299
 300         if (enable)
 301                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
 302         else
 303                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, CSS_CPU, val);
 304
 305         REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_TRICKLE_EN0, val);
 306 }
 307
 308 static void pwr_island_drive_37xx(struct ivpu_device *vdev, bool enable)
 309 {
 310         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0);
 311
 312         if (enable)
 313                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 314         else
 315                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 316
 317         REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 318 }
 319
 320 static void pwr_island_drive_40xx(struct ivpu_device *vdev, bool enable)
 321 {
 322         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0);
 323
 324         if (enable)
 325                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 326         else
 327                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 328
 329         REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 330 }
 331
 332 static void pwr_island_enable(struct ivpu_device *vdev)
 333 {
 334         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 335                 pwr_island_trickle_drive_37xx(vdev, true);
 336                 ndelay(500);
 337                 pwr_island_drive_37xx(vdev, true);
 338         } else {
 339                 pwr_island_trickle_drive_40xx(vdev, true);
 340                 ndelay(500);
 341                 pwr_island_drive_40xx(vdev, true);
 342         }
 343 }
 344
 345 static int wait_for_pwr_island_status(struct ivpu_device *vdev, u32 exp_val)
 346 {
 347         if (IVPU_WA(punit_disabled))
 348                 return 0;
 349
 350         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 351                 return REGV_POLL_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_STATUS0, MSS_CPU, exp_val,
 352                                      PWR_ISLAND_STATUS_TIMEOUT_US);
 353         else
 354                 return REGV_POLL_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_STATUS0, CSS_CPU, exp_val,
 355                                      PWR_ISLAND_STATUS_TIMEOUT_US);
 356 }
 357
 358 static void pwr_island_isolation_drive_37xx(struct ivpu_device *vdev, bool enable)
 359 {
 360         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0);
 361
 362         if (enable)
 363                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val);
 364         else
 365                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, MSS_CPU, val);
 366
 367         REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISO_EN0, val);
 368 }
 369
 370 static void pwr_island_isolation_drive_40xx(struct ivpu_device *vdev, bool enable)
 371 {
 372         u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0);
 373
 374         if (enable)
 375                 val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
 376         else
 377                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, CSS_CPU, val);
 378
 379         REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISO_EN0, val);
 380 }
 381
 382 static void pwr_island_isolation_drive(struct ivpu_device *vdev, bool enable)
 383 {
 384         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 385                 pwr_island_isolation_drive_37xx(vdev, enable);
 386         else
 387                 pwr_island_isolation_drive_40xx(vdev, enable);
 388 }
 389
 390 static void pwr_island_isolation_disable(struct ivpu_device *vdev)
 391 {
 392         pwr_island_isolation_drive(vdev, false);
 393 }
 394
 395 static void host_ss_clk_drive_37xx(struct ivpu_device *vdev, bool enable)
 396 {
 397         u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_CLK_SET);
 398
 399         if (enable) {
 400                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val);
 401                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val);
 402                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val);
 403         } else {
 404                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, TOP_NOC, val);
 405                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, DSS_MAS, val);
 406                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_CLK_SET, MSS_MAS, val);
 407         }
 408
 409         REGV_WR32(VPU_37XX_HOST_SS_CPR_CLK_SET, val);
 410 }
 411
 412 static void host_ss_clk_drive_40xx(struct ivpu_device *vdev, bool enable)
 413 {
 414         u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_CLK_EN);
 415
 416         if (enable) {
 417                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
 418                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
 419                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
 420         } else {
 421                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, TOP_NOC, val);
 422                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, DSS_MAS, val);
 423                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_CLK_EN, CSS_MAS, val);
 424         }
 425
 426         REGV_WR32(VPU_40XX_HOST_SS_CPR_CLK_EN, val);
 427 }
 428
 429 static void host_ss_clk_drive(struct ivpu_device *vdev, bool enable)
 430 {
 431         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 432                 host_ss_clk_drive_37xx(vdev, enable);
 433         else
 434                 host_ss_clk_drive_40xx(vdev, enable);
 435 }
 436
 437 static void host_ss_clk_enable(struct ivpu_device *vdev)
 438 {
 439         host_ss_clk_drive(vdev, true);
 440 }
 441
 442 static void host_ss_rst_drive_37xx(struct ivpu_device *vdev, bool enable)
 443 {
 444         u32 val = REGV_RD32(VPU_37XX_HOST_SS_CPR_RST_SET);
 445
 446         if (enable) {
 447                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val);
 448                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val);
 449                 val = REG_SET_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val);
 450         } else {
 451                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, TOP_NOC, val);
 452                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, DSS_MAS, val);
 453                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_CPR_RST_SET, MSS_MAS, val);
 454         }
 455
 456         REGV_WR32(VPU_37XX_HOST_SS_CPR_RST_SET, val);
 457 }
 458
 459 static void host_ss_rst_drive_40xx(struct ivpu_device *vdev, bool enable)
 460 {
 461         u32 val = REGV_RD32(VPU_40XX_HOST_SS_CPR_RST_EN);
 462
 463         if (enable) {
 464                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
 465                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
 466                 val = REG_SET_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
 467         } else {
 468                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, TOP_NOC, val);
 469                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, DSS_MAS, val);
 470                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_CPR_RST_EN, CSS_MAS, val);
 471         }
 472
 473         REGV_WR32(VPU_40XX_HOST_SS_CPR_RST_EN, val);
 474 }
 475
 476 static void host_ss_rst_drive(struct ivpu_device *vdev, bool enable)
 477 {
 478         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 479                 host_ss_rst_drive_37xx(vdev, enable);
 480         else
 481                 host_ss_rst_drive_40xx(vdev, enable);
 482 }
 483
 484 static void host_ss_rst_enable(struct ivpu_device *vdev)
 485 {
 486         host_ss_rst_drive(vdev, true);
 487 }
 488
 489 static void host_ss_noc_qreqn_top_socmmio_drive_37xx(struct ivpu_device *vdev, bool enable)
 490 {
 491         u32 val = REGV_RD32(VPU_37XX_HOST_SS_NOC_QREQN);
 492
 493         if (enable)
 494                 val = REG_SET_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 495         else
 496                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 497         REGV_WR32(VPU_37XX_HOST_SS_NOC_QREQN, val);
 498 }
 499
 500 static void host_ss_noc_qreqn_top_socmmio_drive_40xx(struct ivpu_device *vdev, bool enable)
 501 {
 502         u32 val = REGV_RD32(VPU_40XX_HOST_SS_NOC_QREQN);
 503
 504         if (enable)
 505                 val = REG_SET_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 506         else
 507                 val = REG_CLR_FLD(VPU_40XX_HOST_SS_NOC_QREQN, TOP_SOCMMIO, val);
 508         REGV_WR32(VPU_40XX_HOST_SS_NOC_QREQN, val);
 509 }
 510
 511 static void host_ss_noc_qreqn_top_socmmio_drive(struct ivpu_device *vdev, bool enable)
 512 {
 513         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 514                 host_ss_noc_qreqn_top_socmmio_drive_37xx(vdev, enable);
 515         else
 516                 host_ss_noc_qreqn_top_socmmio_drive_40xx(vdev, enable);
 517 }
 518
 519 static int host_ss_axi_drive(struct ivpu_device *vdev, bool enable)
 520 {
 521         int ret;
 522
 523         host_ss_noc_qreqn_top_socmmio_drive(vdev, enable);
 524
 525         ret = host_ss_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 526         if (ret) {
 527                 ivpu_err(vdev, "Failed HOST SS NOC QACCEPTN check: %d\n", ret);
 528                 return ret;
 529         }
 530
 531         ret = host_ss_noc_qdeny_check(vdev, 0x0);
 532         if (ret)
 533                 ivpu_err(vdev, "Failed HOST SS NOC QDENY check: %d\n", ret);
 534
 535         return ret;
 536 }
 537
 538 static void top_noc_qreqn_drive_40xx(struct ivpu_device *vdev, bool enable)
 539 {
 540         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QREQN);
 541
 542         if (enable) {
 543                 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
 544                 val = REG_SET_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 545         } else {
 546                 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, CPU_CTRL, val);
 547                 val = REG_CLR_FLD(VPU_40XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 548         }
 549
 550         REGV_WR32(VPU_40XX_TOP_NOC_QREQN, val);
 551 }
 552
 553 static void top_noc_qreqn_drive_37xx(struct ivpu_device *vdev, bool enable)
 554 {
 555         u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QREQN);
 556
 557         if (enable) {
 558                 val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val);
 559                 val = REG_SET_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 560         } else {
 561                 val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, CPU_CTRL, val);
 562                 val = REG_CLR_FLD(VPU_37XX_TOP_NOC_QREQN, HOSTIF_L2CACHE, val);
 563         }
 564
 565         REGV_WR32(VPU_37XX_TOP_NOC_QREQN, val);
 566 }
 567
 568 static void top_noc_qreqn_drive(struct ivpu_device *vdev, bool enable)
 569 {
 570         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 571                 top_noc_qreqn_drive_37xx(vdev, enable);
 572         else
 573                 top_noc_qreqn_drive_40xx(vdev, enable);
 574 }
 575
 576 int ivpu_hw_ip_host_ss_axi_enable(struct ivpu_device *vdev)
 577 {
 578         return host_ss_axi_drive(vdev, true);
 579 }
 580
 581 static int top_noc_qacceptn_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 582 {
 583         u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QACCEPTN);
 584
 585         if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
 586             !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
 587                 return -EIO;
 588
 589         return 0;
 590 }
 591
 592 static int top_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 593 {
 594         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QACCEPTN);
 595
 596         if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, CPU_CTRL, exp_val, val) ||
 597             !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QACCEPTN, HOSTIF_L2CACHE, exp_val, val))
 598                 return -EIO;
 599
 600         return 0;
 601 }
 602
 603 static int top_noc_qacceptn_check(struct ivpu_device *vdev, u32 exp_val)
 604 {
 605         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 606                 return top_noc_qacceptn_check_37xx(vdev, exp_val);
 607         else
 608                 return top_noc_qacceptn_check_40xx(vdev, exp_val);
 609 }
 610
 611 static int top_noc_qdeny_check_37xx(struct ivpu_device *vdev, u32 exp_val)
 612 {
 613         u32 val = REGV_RD32(VPU_37XX_TOP_NOC_QDENY);
 614
 615         if (!REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
 616             !REG_TEST_FLD_NUM(VPU_37XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
 617                 return -EIO;
 618
 619         return 0;
 620 }
 621
 622 static int top_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 623 {
 624         u32 val = REGV_RD32(VPU_40XX_TOP_NOC_QDENY);
 625
 626         if (!REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, CPU_CTRL, exp_val, val) ||
 627             !REG_TEST_FLD_NUM(VPU_40XX_TOP_NOC_QDENY, HOSTIF_L2CACHE, exp_val, val))
 628                 return -EIO;
 629
 630         return 0;
 631 }
 632
 633 static int top_noc_qdeny_check(struct ivpu_device *vdev, u32 exp_val)
 634 {
 635         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 636                 return top_noc_qdeny_check_37xx(vdev, exp_val);
 637         else
 638                 return top_noc_qdeny_check_40xx(vdev, exp_val);
 639 }
 640
 641 static int top_noc_drive(struct ivpu_device *vdev, bool enable)
 642 {
 643         int ret;
 644
 645         top_noc_qreqn_drive(vdev, enable);
 646
 647         ret = top_noc_qacceptn_check(vdev, enable ? 0x1 : 0x0);
 648         if (ret) {
 649                 ivpu_err(vdev, "Failed TOP NOC QACCEPTN check: %d\n", ret);
 650                 return ret;
 651         }
 652
 653         ret = top_noc_qdeny_check(vdev, 0x0);
 654         if (ret)
 655                 ivpu_err(vdev, "Failed TOP NOC QDENY check: %d\n", ret);
 656
 657         return ret;
 658 }
 659
 660 int ivpu_hw_ip_top_noc_enable(struct ivpu_device *vdev)
 661 {
 662         return top_noc_drive(vdev, true);
 663 }
 664
 665 static void dpu_active_drive_37xx(struct ivpu_device *vdev, bool enable)
 666 {
 667         u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE);
 668
 669         if (enable)
 670                 val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val);
 671         else
 672                 val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, DPU_ACTIVE, val);
 673
 674         REGV_WR32(VPU_37XX_HOST_SS_AON_DPU_ACTIVE, val);
 675 }
 676
 677 static void pwr_island_delay_set(struct ivpu_device *vdev)
 678 {
 679         bool high = vdev->hw->pll.profiling_freq == PLL_PROFILING_FREQ_HIGH;
 680         u32 post, post1, post2, status;
 681
 682         if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX)
 683                 return;
 684
 685         switch (ivpu_device_id(vdev)) {
 686         case PCI_DEVICE_ID_PTL_P:
 687                 post = high ? 18 : 0;
 688                 post1 = 0;
 689                 post2 = 0;
 690                 status = high ? 46 : 3;
 691                 break;
 692
 693         default:
 694                 dump_stack();
 695                 ivpu_err(vdev, "Unknown device ID\n");
 696                 return;
 697         }
 698
 699         pwr_island_delay_set_50xx(vdev, post, post1, post2, status);
 700 }
 701
 702 int ivpu_hw_ip_pwr_domain_enable(struct ivpu_device *vdev)
 703 {
 704         int ret;
 705
 706         pwr_island_delay_set(vdev);
 707         pwr_island_enable(vdev);
 708
 709         ret = wait_for_pwr_island_status(vdev, 0x1);
 710         if (ret) {
 711                 ivpu_err(vdev, "Timed out waiting for power island status\n");
 712                 return ret;
 713         }
 714
 715         ret = top_noc_qreqn_check(vdev, 0x0);
 716         if (ret) {
 717                 ivpu_err(vdev, "Failed TOP NOC QREQN check %d\n", ret);
 718                 return ret;
 719         }
 720
 721         host_ss_clk_enable(vdev);
 722         pwr_island_isolation_disable(vdev);
 723         host_ss_rst_enable(vdev);
 724
 725         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 726                 dpu_active_drive_37xx(vdev, true);
 727
 728         return ret;
 729 }
 730
 731 u64 ivpu_hw_ip_read_perf_timer_counter(struct ivpu_device *vdev)
 732 {
 733         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 734                 return REGV_RD64(VPU_37XX_CPU_SS_TIM_PERF_FREE_CNT);
 735         else
 736                 return REGV_RD64(VPU_40XX_CPU_SS_TIM_PERF_EXT_FREE_CNT);
 737 }
 738
 739 static void ivpu_hw_ip_snoop_disable_37xx(struct ivpu_device *vdev)
 740 {
 741         u32 val = REGV_RD32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES);
 742
 743         val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, NOSNOOP_OVERRIDE_EN, val);
 744         val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AW_NOSNOOP_OVERRIDE, val);
 745
 746         if (ivpu_is_force_snoop_enabled(vdev))
 747                 val = REG_CLR_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val);
 748         else
 749                 val = REG_SET_FLD(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, AR_NOSNOOP_OVERRIDE, val);
 750
 751         REGV_WR32(VPU_37XX_HOST_IF_TCU_PTW_OVERRIDES, val);
 752 }
 753
 754 static void ivpu_hw_ip_snoop_disable_40xx(struct ivpu_device *vdev)
 755 {
 756         u32 val = REGV_RD32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES);
 757
 758         val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, SNOOP_OVERRIDE_EN, val);
 759         val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AW_SNOOP_OVERRIDE, val);
 760
 761         if (ivpu_is_force_snoop_enabled(vdev))
 762                 val = REG_SET_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val);
 763         else
 764                 val = REG_CLR_FLD(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, AR_SNOOP_OVERRIDE, val);
 765
 766         REGV_WR32(VPU_40XX_HOST_IF_TCU_PTW_OVERRIDES, val);
 767 }
 768
 769 void ivpu_hw_ip_snoop_disable(struct ivpu_device *vdev)
 770 {
 771         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 772                 return ivpu_hw_ip_snoop_disable_37xx(vdev);
 773         else
 774                 return ivpu_hw_ip_snoop_disable_40xx(vdev);
 775 }
 776
 777 static void ivpu_hw_ip_tbu_mmu_enable_37xx(struct ivpu_device *vdev)
 778 {
 779         u32 val = REGV_RD32(VPU_37XX_HOST_IF_TBU_MMUSSIDV);
 780
 781         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
 782         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
 783         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
 784         val = REG_SET_FLD(VPU_37XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
 785
 786         REGV_WR32(VPU_37XX_HOST_IF_TBU_MMUSSIDV, val);
 787 }
 788
 789 static void ivpu_hw_ip_tbu_mmu_enable_40xx(struct ivpu_device *vdev)
 790 {
 791         u32 val = REGV_RD32(VPU_40XX_HOST_IF_TBU_MMUSSIDV);
 792
 793         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_AWMMUSSIDV, val);
 794         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU0_ARMMUSSIDV, val);
 795         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_AWMMUSSIDV, val);
 796         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU1_ARMMUSSIDV, val);
 797         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_AWMMUSSIDV, val);
 798         val = REG_SET_FLD(VPU_40XX_HOST_IF_TBU_MMUSSIDV, TBU2_ARMMUSSIDV, val);
 799
 800         REGV_WR32(VPU_40XX_HOST_IF_TBU_MMUSSIDV, val);
 801 }
 802
 803 void ivpu_hw_ip_tbu_mmu_enable(struct ivpu_device *vdev)
 804 {
 805         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 806                 return ivpu_hw_ip_tbu_mmu_enable_37xx(vdev);
 807         else
 808                 return ivpu_hw_ip_tbu_mmu_enable_40xx(vdev);
 809 }
 810
 811 static int soc_cpu_boot_37xx(struct ivpu_device *vdev)
 812 {
 813         u32 val;
 814
 815         val = REGV_RD32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC);
 816         val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTRUN0, val);
 817
 818         val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RSTVEC, val);
 819         REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 820
 821         val = REG_SET_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val);
 822         REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 823
 824         val = REG_CLR_FLD(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, IRQI_RESUME0, val);
 825         REGV_WR32(VPU_37XX_CPU_SS_MSSCPU_CPR_LEON_RT_VEC, val);
 826
 827         val = vdev->fw->entry_point >> 9;
 828         REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val);
 829
 830         val = REG_SET_FLD(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, DONE, val);
 831         REGV_WR32(VPU_37XX_HOST_SS_LOADING_ADDRESS_LO, val);
 832
 833         ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
 834                  vdev->fw->entry_point == vdev->fw->cold_boot_entry_point ? "cold boot" : "resume");
 835
 836         return 0;
 837 }
 838
 839 static int cpu_noc_qacceptn_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 840 {
 841         u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN);
 842
 843         if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QACCEPTN, TOP_MMIO, exp_val, val))
 844                 return -EIO;
 845
 846         return 0;
 847 }
 848
 849 static int cpu_noc_qdeny_check_40xx(struct ivpu_device *vdev, u32 exp_val)
 850 {
 851         u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QDENY);
 852
 853         if (!REG_TEST_FLD_NUM(VPU_40XX_CPU_SS_CPR_NOC_QDENY, TOP_MMIO, exp_val, val))
 854                 return -EIO;
 855
 856         return 0;
 857 }
 858
 859 static void cpu_noc_top_mmio_drive_40xx(struct ivpu_device *vdev, bool enable)
 860 {
 861         u32 val = REGV_RD32(VPU_40XX_CPU_SS_CPR_NOC_QREQN);
 862
 863         if (enable)
 864                 val = REG_SET_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
 865         else
 866                 val = REG_CLR_FLD(VPU_40XX_CPU_SS_CPR_NOC_QREQN, TOP_MMIO, val);
 867         REGV_WR32(VPU_40XX_CPU_SS_CPR_NOC_QREQN, val);
 868 }
 869
 870 static int soc_cpu_drive_40xx(struct ivpu_device *vdev, bool enable)
 871 {
 872         int ret;
 873
 874         cpu_noc_top_mmio_drive_40xx(vdev, enable);
 875
 876         ret = cpu_noc_qacceptn_check_40xx(vdev, enable ? 0x1 : 0x0);
 877         if (ret) {
 878                 ivpu_err(vdev, "Failed qacceptn check: %d\n", ret);
 879                 return ret;
 880         }
 881
 882         ret = cpu_noc_qdeny_check_40xx(vdev, 0x0);
 883         if (ret)
 884                 ivpu_err(vdev, "Failed qdeny check: %d\n", ret);
 885
 886         return ret;
 887 }
 888
 889 static int soc_cpu_enable(struct ivpu_device *vdev)
 890 {
 891         return soc_cpu_drive_40xx(vdev, true);
 892 }
 893
 894 static int soc_cpu_boot_40xx(struct ivpu_device *vdev)
 895 {
 896         int ret;
 897         u32 val;
 898         u64 val64;
 899
 900         ret = soc_cpu_enable(vdev);
 901         if (ret) {
 902                 ivpu_err(vdev, "Failed to enable SOC CPU: %d\n", ret);
 903                 return ret;
 904         }
 905
 906         val64 = vdev->fw->entry_point;
 907         val64 <<= ffs(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO_IMAGE_LOCATION_MASK) - 1;
 908         REGV_WR64(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val64);
 909
 910         val = REGV_RD32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO);
 911         val = REG_SET_FLD(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, DONE, val);
 912         REGV_WR32(VPU_40XX_HOST_SS_VERIFICATION_ADDRESS_LO, val);
 913
 914         ivpu_dbg(vdev, PM, "Booting firmware, mode: %s\n",
 915                  ivpu_fw_is_cold_boot(vdev) ? "cold boot" : "resume");
 916
 917         return 0;
 918 }
 919
 920 int ivpu_hw_ip_soc_cpu_boot(struct ivpu_device *vdev)
 921 {
 922         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 923                 return soc_cpu_boot_37xx(vdev);
 924         else
 925                 return soc_cpu_boot_40xx(vdev);
 926 }
 927
 928 static void wdt_disable_37xx(struct ivpu_device *vdev)
 929 {
 930         u32 val;
 931
 932         /* Enable writing and set non-zero WDT value */
 933         REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 934         REGV_WR32(VPU_37XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
 935
 936         /* Enable writing and disable watchdog timer */
 937         REGV_WR32(VPU_37XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 938         REGV_WR32(VPU_37XX_CPU_SS_TIM_WDOG_EN, 0);
 939
 940         /* Now clear the timeout interrupt */
 941         val = REGV_RD32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG);
 942         val = REG_CLR_FLD(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
 943         REGV_WR32(VPU_37XX_CPU_SS_TIM_GEN_CONFIG, val);
 944 }
 945
 946 static void wdt_disable_40xx(struct ivpu_device *vdev)
 947 {
 948         u32 val;
 949
 950         REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 951         REGV_WR32(VPU_40XX_CPU_SS_TIM_WATCHDOG, TIM_WATCHDOG_RESET_VALUE);
 952
 953         REGV_WR32(VPU_40XX_CPU_SS_TIM_SAFE, TIM_SAFE_ENABLE);
 954         REGV_WR32(VPU_40XX_CPU_SS_TIM_WDOG_EN, 0);
 955
 956         val = REGV_RD32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG);
 957         val = REG_CLR_FLD(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, WDOG_TO_INT_CLR, val);
 958         REGV_WR32(VPU_40XX_CPU_SS_TIM_GEN_CONFIG, val);
 959 }
 960
 961 void ivpu_hw_ip_wdt_disable(struct ivpu_device *vdev)
 962 {
 963         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 964                 return wdt_disable_37xx(vdev);
 965         else
 966                 return wdt_disable_40xx(vdev);
 967 }
 968
 969 static u32 ipc_rx_count_get_37xx(struct ivpu_device *vdev)
 970 {
 971         u32 count = REGV_RD32_SILENT(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT);
 972
 973         return REG_GET_FLD(VPU_37XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
 974 }
 975
 976 static u32 ipc_rx_count_get_40xx(struct ivpu_device *vdev)
 977 {
 978         u32 count = REGV_RD32_SILENT(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT);
 979
 980         return REG_GET_FLD(VPU_40XX_HOST_SS_TIM_IPC_FIFO_STAT, FILL_LEVEL, count);
 981 }
 982
 983 u32 ivpu_hw_ip_ipc_rx_count_get(struct ivpu_device *vdev)
 984 {
 985         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
 986                 return ipc_rx_count_get_37xx(vdev);
 987         else
 988                 return ipc_rx_count_get_40xx(vdev);
 989 }
 990
 991 void ivpu_hw_ip_irq_enable(struct ivpu_device *vdev)
 992 {
 993         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
 994                 REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_37XX);
 995                 REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_37XX);
 996         } else {
 997                 REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, ITF_FIREWALL_VIOLATION_MASK_40XX);
 998                 REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, ICB_0_1_IRQ_MASK_40XX);
 999         }
1000 }
1001
1002 void ivpu_hw_ip_irq_disable(struct ivpu_device *vdev)
1003 {
1004         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
1005                 REGV_WR64(VPU_37XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
1006                 REGV_WR32(VPU_37XX_HOST_SS_FW_SOC_IRQ_EN, 0x0);
1007         } else {
1008                 REGV_WR64(VPU_40XX_HOST_SS_ICB_ENABLE_0, 0x0ull);
1009                 REGV_WR32(VPU_40XX_HOST_SS_FW_SOC_IRQ_EN, 0x0ul);
1010         }
1011 }
1012
1013 static void diagnose_failure_37xx(struct ivpu_device *vdev)
1014 {
1015         u32 reg = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX;
1016
1017         if (ipc_rx_count_get_37xx(vdev))
1018                 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1019
1020         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg))
1021                 ivpu_err(vdev, "WDT MSS timeout detected\n");
1022
1023         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg))
1024                 ivpu_err(vdev, "WDT NCE timeout detected\n");
1025
1026         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg))
1027                 ivpu_err(vdev, "NOC Firewall irq detected\n");
1028 }
1029
1030 static void diagnose_failure_40xx(struct ivpu_device *vdev)
1031 {
1032         u32 reg = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX;
1033
1034         if (ipc_rx_count_get_40xx(vdev))
1035                 ivpu_err(vdev, "IPC FIFO queue not empty, missed IPC IRQ");
1036
1037         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, reg))
1038                 ivpu_err(vdev, "WDT MSS timeout detected\n");
1039
1040         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, reg))
1041                 ivpu_err(vdev, "WDT NCE timeout detected\n");
1042
1043         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, reg))
1044                 ivpu_err(vdev, "NOC Firewall irq detected\n");
1045 }
1046
1047 void ivpu_hw_ip_diagnose_failure(struct ivpu_device *vdev)
1048 {
1049         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1050                 diagnose_failure_37xx(vdev);
1051         else
1052                 diagnose_failure_40xx(vdev);
1053 }
1054
1055 void ivpu_hw_ip_irq_clear(struct ivpu_device *vdev)
1056 {
1057         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1058                 REGV_WR64(VPU_37XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_37XX);
1059         else
1060                 REGV_WR64(VPU_40XX_HOST_SS_ICB_CLEAR_0, ICB_0_1_IRQ_MASK_40XX);
1061 }
1062
1063 static void irq_wdt_nce_handler(struct ivpu_device *vdev)
1064 {
1065         ivpu_pm_trigger_recovery(vdev, "WDT NCE IRQ");
1066 }
1067
1068 static void irq_wdt_mss_handler(struct ivpu_device *vdev)
1069 {
1070         ivpu_hw_ip_wdt_disable(vdev);
1071         ivpu_pm_trigger_recovery(vdev, "WDT MSS IRQ");
1072 }
1073
1074 static void irq_noc_firewall_handler(struct ivpu_device *vdev)
1075 {
1076         atomic_inc(&vdev->hw->firewall_irq_counter);
1077
1078         ivpu_dbg(vdev, IRQ, "NOC Firewall interrupt detected, counter %d\n",
1079                  atomic_read(&vdev->hw->firewall_irq_counter));
1080 }
1081
1082 /* Handler for IRQs from NPU core */
1083 bool ivpu_hw_ip_irq_handler_37xx(struct ivpu_device *vdev, int irq)
1084 {
1085         u32 status = REGV_RD32(VPU_37XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_37XX;
1086
1087         if (!status)
1088                 return false;
1089
1090         REGV_WR32(VPU_37XX_HOST_SS_ICB_CLEAR_0, status);
1091
1092         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
1093                 ivpu_mmu_irq_evtq_handler(vdev);
1094
1095         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
1096                 ivpu_ipc_irq_handler(vdev);
1097
1098         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
1099                 ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
1100
1101         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
1102                 ivpu_mmu_irq_gerr_handler(vdev);
1103
1104         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
1105                 irq_wdt_mss_handler(vdev);
1106
1107         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
1108                 irq_wdt_nce_handler(vdev);
1109
1110         if (REG_TEST_FLD(VPU_37XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
1111                 irq_noc_firewall_handler(vdev);
1112
1113         return true;
1114 }
1115
1116 /* Handler for IRQs from NPU core */
1117 bool ivpu_hw_ip_irq_handler_40xx(struct ivpu_device *vdev, int irq)
1118 {
1119         u32 status = REGV_RD32(VPU_40XX_HOST_SS_ICB_STATUS_0) & ICB_0_IRQ_MASK_40XX;
1120
1121         if (!status)
1122                 return false;
1123
1124         REGV_WR32(VPU_40XX_HOST_SS_ICB_CLEAR_0, status);
1125
1126         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_0_INT, status))
1127                 ivpu_mmu_irq_evtq_handler(vdev);
1128
1129         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, HOST_IPC_FIFO_INT, status))
1130                 ivpu_ipc_irq_handler(vdev);
1131
1132         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_1_INT, status))
1133                 ivpu_dbg(vdev, IRQ, "MMU sync complete\n");
1134
1135         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, MMU_IRQ_2_INT, status))
1136                 ivpu_mmu_irq_gerr_handler(vdev);
1137
1138         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_0_INT, status))
1139                 irq_wdt_mss_handler(vdev);
1140
1141         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, CPU_INT_REDIRECT_1_INT, status))
1142                 irq_wdt_nce_handler(vdev);
1143
1144         if (REG_TEST_FLD(VPU_40XX_HOST_SS_ICB_STATUS_0, NOC_FIREWALL_INT, status))
1145                 irq_noc_firewall_handler(vdev);
1146
1147         return true;
1148 }
1149
1150 static void db_set_37xx(struct ivpu_device *vdev, u32 db_id)
1151 {
1152         u32 reg_stride = VPU_37XX_CPU_SS_DOORBELL_1 - VPU_37XX_CPU_SS_DOORBELL_0;
1153         u32 val = REG_FLD(VPU_37XX_CPU_SS_DOORBELL_0, SET);
1154
1155         REGV_WR32I(VPU_37XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
1156 }
1157
1158 static void db_set_40xx(struct ivpu_device *vdev, u32 db_id)
1159 {
1160         u32 reg_stride = VPU_40XX_CPU_SS_DOORBELL_1 - VPU_40XX_CPU_SS_DOORBELL_0;
1161         u32 val = REG_FLD(VPU_40XX_CPU_SS_DOORBELL_0, SET);
1162
1163         REGV_WR32I(VPU_40XX_CPU_SS_DOORBELL_0, reg_stride, db_id, val);
1164 }
1165
1166 void ivpu_hw_ip_db_set(struct ivpu_device *vdev, u32 db_id)
1167 {
1168         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1169                 db_set_37xx(vdev, db_id);
1170         else
1171                 db_set_40xx(vdev, db_id);
1172 }
1173
1174 u32 ivpu_hw_ip_ipc_rx_addr_get(struct ivpu_device *vdev)
1175 {
1176         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1177                 return REGV_RD32(VPU_37XX_HOST_SS_TIM_IPC_FIFO_ATM);
1178         else
1179                 return REGV_RD32(VPU_40XX_HOST_SS_TIM_IPC_FIFO_ATM);
1180 }
1181
1182 void ivpu_hw_ip_ipc_tx_set(struct ivpu_device *vdev, u32 vpu_addr)
1183 {
1184         if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
1185                 REGV_WR32(VPU_37XX_CPU_SS_TIM_IPC_FIFO, vpu_addr);
1186         else
1187                 REGV_WR32(VPU_40XX_CPU_SS_TIM_IPC_FIFO, vpu_addr);
1188 }