drivers/gpu/drm/amd/amdgpu/vega10_ih.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/pci.h>
  25
  26 #include "amdgpu.h"
  27 #include "amdgpu_ih.h"
  28 #include "soc15.h"
  29
  30 #include "oss/osssys_4_0_offset.h"
  31 #include "oss/osssys_4_0_sh_mask.h"
  32
  33 #include "soc15_common.h"
  34 #include "vega10_ih.h"
  35
  36 #define MAX_REARM_RETRY 10
  37
  38 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
  39
  40 /**
  41  * vega10_ih_enable_interrupts - Enable the interrupt ring buffer
  42  *
  43  * @adev: amdgpu_device pointer
  44  *
  45  * Enable the interrupt ring buffer (VEGA10).
  46  */
  47 static void vega10_ih_enable_interrupts(struct amdgpu_device *adev)
  48 {
  49         u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
  50
  51         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
  52         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
  53         if (amdgpu_sriov_vf(adev)) {
  54                 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
  55                         DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
  56                         return;
  57                 }
  58         } else {
  59                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
  60         }
  61         adev->irq.ih.enabled = true;
  62
  63         if (adev->irq.ih1.ring_size) {
  64                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
  65                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
  66                                            RB_ENABLE, 1);
  67                 if (amdgpu_sriov_vf(adev)) {
  68                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
  69                                                 ih_rb_cntl)) {
  70                                 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
  71                                 return;
  72                         }
  73                 } else {
  74                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
  75                 }
  76                 adev->irq.ih1.enabled = true;
  77         }
  78
  79         if (adev->irq.ih2.ring_size) {
  80                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
  81                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
  82                                            RB_ENABLE, 1);
  83                 if (amdgpu_sriov_vf(adev)) {
  84                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
  85                                                 ih_rb_cntl)) {
  86                                 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
  87                                 return;
  88                         }
  89                 } else {
  90                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
  91                 }
  92                 adev->irq.ih2.enabled = true;
  93         }
  94 }
  95
  96 /**
  97  * vega10_ih_disable_interrupts - Disable the interrupt ring buffer
  98  *
  99  * @adev: amdgpu_device pointer
 100  *
 101  * Disable the interrupt ring buffer (VEGA10).
 102  */
 103 static void vega10_ih_disable_interrupts(struct amdgpu_device *adev)
 104 {
 105         u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 106
 107         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
 108         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
 109         if (amdgpu_sriov_vf(adev)) {
 110                 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
 111                         DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
 112                         return;
 113                 }
 114         } else {
 115                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 116         }
 117
 118         /* set rptr, wptr to 0 */
 119         WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 120         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 121         adev->irq.ih.enabled = false;
 122         adev->irq.ih.rptr = 0;
 123
 124         if (adev->irq.ih1.ring_size) {
 125                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 126                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
 127                                            RB_ENABLE, 0);
 128                 if (amdgpu_sriov_vf(adev)) {
 129                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
 130                                                 ih_rb_cntl)) {
 131                                 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
 132                                 return;
 133                         }
 134                 } else {
 135                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
 136                 }
 137                 /* set rptr, wptr to 0 */
 138                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
 139                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
 140                 adev->irq.ih1.enabled = false;
 141                 adev->irq.ih1.rptr = 0;
 142         }
 143
 144         if (adev->irq.ih2.ring_size) {
 145                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 146                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
 147                                            RB_ENABLE, 0);
 148                 if (amdgpu_sriov_vf(adev)) {
 149                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
 150                                                 ih_rb_cntl)) {
 151                                 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
 152                                 return;
 153                         }
 154                 } else {
 155                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
 156                 }
 157
 158                 /* set rptr, wptr to 0 */
 159                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
 160                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
 161                 adev->irq.ih2.enabled = false;
 162                 adev->irq.ih2.rptr = 0;
 163         }
 164 }
 165
 166 static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
 167 {
 168         int rb_bufsz = order_base_2(ih->ring_size / 4);
 169
 170         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 171                                    MC_SPACE, ih->use_bus_addr ? 1 : 4);
 172         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 173                                    WPTR_OVERFLOW_CLEAR, 1);
 174         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 175                                    WPTR_OVERFLOW_ENABLE, 1);
 176         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
 177         /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
 178          * value is written to memory
 179          */
 180         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 181                                    WPTR_WRITEBACK_ENABLE, 1);
 182         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
 183         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
 184         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
 185
 186         return ih_rb_cntl;
 187 }
 188
 189 static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)
 190 {
 191         u32 ih_doorbell_rtpr = 0;
 192
 193         if (ih->use_doorbell) {
 194                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 195                                                  IH_DOORBELL_RPTR, OFFSET,
 196                                                  ih->doorbell_index);
 197                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 198                                                  IH_DOORBELL_RPTR,
 199                                                  ENABLE, 1);
 200         } else {
 201                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 202                                                  IH_DOORBELL_RPTR,
 203                                                  ENABLE, 0);
 204         }
 205         return ih_doorbell_rtpr;
 206 }
 207
 208 /**
 209  * vega10_ih_irq_init - init and enable the interrupt ring
 210  *
 211  * @adev: amdgpu_device pointer
 212  *
 213  * Allocate a ring buffer for the interrupt controller,
 214  * enable the RLC, disable interrupts, enable the IH
 215  * ring buffer and enable it (VI).
 216  * Called at device load and reume.
 217  * Returns 0 for success, errors for failure.
 218  */
 219 static int vega10_ih_irq_init(struct amdgpu_device *adev)
 220 {
 221         struct amdgpu_ih_ring *ih;
 222         u32 ih_rb_cntl, ih_chicken;
 223         int ret = 0;
 224         u32 tmp;
 225
 226         /* disable irqs */
 227         vega10_ih_disable_interrupts(adev);
 228
 229         adev->nbio.funcs->ih_control(adev);
 230
 231         ih = &adev->irq.ih;
 232         /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
 233         WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
 234         WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
 235
 236         ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 237         ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 238         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
 239                                    !!adev->irq.msi_enabled);
 240         if (amdgpu_sriov_vf(adev)) {
 241                 if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
 242                         DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
 243                         return -ETIMEDOUT;
 244                 }
 245         } else {
 246                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 247         }
 248
 249         if ((adev->asic_type == CHIP_ARCTURUS &&
 250              adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
 251             adev->asic_type == CHIP_RENOIR) {
 252                 ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
 253                 if (adev->irq.ih.use_bus_addr) {
 254                         ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
 255                                                    MC_SPACE_GPA_ENABLE, 1);
 256                 } else {
 257                         ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
 258                                                    MC_SPACE_FBPA_ENABLE, 1);
 259                 }
 260                 WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
 261         }
 262
 263         /* set the writeback address whether it's enabled or not */
 264         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
 265                      lower_32_bits(ih->wptr_addr));
 266         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
 267                      upper_32_bits(ih->wptr_addr) & 0xFFFF);
 268
 269         /* set rptr, wptr to 0 */
 270         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 271         WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 272
 273         WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR,
 274                      vega10_ih_doorbell_rptr(ih));
 275
 276         ih = &adev->irq.ih1;
 277         if (ih->ring_size) {
 278                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8);
 279                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1,
 280                              (ih->gpu_addr >> 40) & 0xff);
 281
 282                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 283                 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 284                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 285                                            WPTR_OVERFLOW_ENABLE, 0);
 286                 ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
 287                                            RB_FULL_DRAIN_ENABLE, 1);
 288                 if (amdgpu_sriov_vf(adev)) {
 289                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
 290                                                 ih_rb_cntl)) {
 291                                 DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
 292                                 return -ETIMEDOUT;
 293                         }
 294                 } else {
 295                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
 296                 }
 297
 298                 /* set rptr, wptr to 0 */
 299                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
 300                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
 301
 302                 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1,
 303                              vega10_ih_doorbell_rptr(ih));
 304         }
 305
 306         ih = &adev->irq.ih2;
 307         if (ih->ring_size) {
 308                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8);
 309                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2,
 310                              (ih->gpu_addr >> 40) & 0xff);
 311
 312                 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 313                 ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
 314
 315                 if (amdgpu_sriov_vf(adev)) {
 316                         if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
 317                                                 ih_rb_cntl)) {
 318                                 DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
 319                                 return -ETIMEDOUT;
 320                         }
 321                 } else {
 322                         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
 323                 }
 324
 325                 /* set rptr, wptr to 0 */
 326                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
 327                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
 328
 329                 WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2,
 330                              vega10_ih_doorbell_rptr(ih));
 331         }
 332
 333         tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
 334         tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
 335                             CLIENT18_IS_STORM_CLIENT, 1);
 336         WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
 337
 338         tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
 339         tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
 340         WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
 341
 342         pci_set_master(adev->pdev);
 343
 344         /* enable interrupts */
 345         vega10_ih_enable_interrupts(adev);
 346
 347         return ret;
 348 }
 349
 350 /**
 351  * vega10_ih_irq_disable - disable interrupts
 352  *
 353  * @adev: amdgpu_device pointer
 354  *
 355  * Disable interrupts on the hw (VEGA10).
 356  */
 357 static void vega10_ih_irq_disable(struct amdgpu_device *adev)
 358 {
 359         vega10_ih_disable_interrupts(adev);
 360
 361         /* Wait and acknowledge irq */
 362         mdelay(1);
 363 }
 364
 365 /**
 366  * vega10_ih_get_wptr - get the IH ring buffer wptr
 367  *
 368  * @adev: amdgpu_device pointer
 369  *
 370  * Get the IH ring buffer wptr from either the register
 371  * or the writeback memory buffer (VEGA10).  Also check for
 372  * ring buffer overflow and deal with it.
 373  * Returns the value of the wptr.
 374  */
 375 static u32 vega10_ih_get_wptr(struct amdgpu_device *adev,
 376                               struct amdgpu_ih_ring *ih)
 377 {
 378         u32 wptr, reg, tmp;
 379
 380         wptr = le32_to_cpu(*ih->wptr_cpu);
 381
 382         if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 383                 goto out;
 384
 385         /* Double check that the overflow wasn't already cleared. */
 386
 387         if (ih == &adev->irq.ih)
 388                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
 389         else if (ih == &adev->irq.ih1)
 390                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1);
 391         else if (ih == &adev->irq.ih2)
 392                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2);
 393         else
 394                 BUG();
 395
 396         wptr = RREG32_NO_KIQ(reg);
 397         if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 398                 goto out;
 399
 400         wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
 401
 402         /* When a ring buffer overflow happen start parsing interrupt
 403          * from the last not overwritten vector (wptr + 32). Hopefully
 404          * this should allow us to catchup.
 405          */
 406         tmp = (wptr + 32) & ih->ptr_mask;
 407         dev_warn(adev->dev, "IH ring buffer overflow "
 408                  "(0x%08X, 0x%08X, 0x%08X)\n",
 409                  wptr, ih->rptr, tmp);
 410         ih->rptr = tmp;
 411
 412         if (ih == &adev->irq.ih)
 413                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
 414         else if (ih == &adev->irq.ih1)
 415                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1);
 416         else if (ih == &adev->irq.ih2)
 417                 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2);
 418         else
 419                 BUG();
 420
 421         tmp = RREG32_NO_KIQ(reg);
 422         tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
 423         WREG32_NO_KIQ(reg, tmp);
 424
 425 out:
 426         return (wptr & ih->ptr_mask);
 427 }
 428
 429 /**
 430  * vega10_ih_decode_iv - decode an interrupt vector
 431  *
 432  * @adev: amdgpu_device pointer
 433  *
 434  * Decodes the interrupt vector at the current rptr
 435  * position and also advance the position.
 436  */
 437 static void vega10_ih_decode_iv(struct amdgpu_device *adev,
 438                                 struct amdgpu_ih_ring *ih,
 439                                 struct amdgpu_iv_entry *entry)
 440 {
 441         /* wptr/rptr are in bytes! */
 442         u32 ring_index = ih->rptr >> 2;
 443         uint32_t dw[8];
 444
 445         dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
 446         dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
 447         dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
 448         dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
 449         dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
 450         dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
 451         dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
 452         dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
 453
 454         entry->client_id = dw[0] & 0xff;
 455         entry->src_id = (dw[0] >> 8) & 0xff;
 456         entry->ring_id = (dw[0] >> 16) & 0xff;
 457         entry->vmid = (dw[0] >> 24) & 0xf;
 458         entry->vmid_src = (dw[0] >> 31);
 459         entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
 460         entry->timestamp_src = dw[2] >> 31;
 461         entry->pasid = dw[3] & 0xffff;
 462         entry->pasid_src = dw[3] >> 31;
 463         entry->src_data[0] = dw[4];
 464         entry->src_data[1] = dw[5];
 465         entry->src_data[2] = dw[6];
 466         entry->src_data[3] = dw[7];
 467
 468         /* wptr/rptr are in bytes! */
 469         ih->rptr += 32;
 470 }
 471
 472 /**
 473  * vega10_ih_irq_rearm - rearm IRQ if lost
 474  *
 475  * @adev: amdgpu_device pointer
 476  *
 477  */
 478 static void vega10_ih_irq_rearm(struct amdgpu_device *adev,
 479                                struct amdgpu_ih_ring *ih)
 480 {
 481         uint32_t reg_rptr = 0;
 482         uint32_t v = 0;
 483         uint32_t i = 0;
 484
 485         if (ih == &adev->irq.ih)
 486                 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR);
 487         else if (ih == &adev->irq.ih1)
 488                 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1);
 489         else if (ih == &adev->irq.ih2)
 490                 reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2);
 491         else
 492                 return;
 493
 494         /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */
 495         for (i = 0; i < MAX_REARM_RETRY; i++) {
 496                 v = RREG32_NO_KIQ(reg_rptr);
 497                 if ((v < ih->ring_size) && (v != ih->rptr))
 498                         WDOORBELL32(ih->doorbell_index, ih->rptr);
 499                 else
 500                         break;
 501         }
 502 }
 503
 504 /**
 505  * vega10_ih_set_rptr - set the IH ring buffer rptr
 506  *
 507  * @adev: amdgpu_device pointer
 508  *
 509  * Set the IH ring buffer rptr.
 510  */
 511 static void vega10_ih_set_rptr(struct amdgpu_device *adev,
 512                                struct amdgpu_ih_ring *ih)
 513 {
 514         if (ih->use_doorbell) {
 515                 /* XXX check if swapping is necessary on BE */
 516                 *ih->rptr_cpu = ih->rptr;
 517                 WDOORBELL32(ih->doorbell_index, ih->rptr);
 518
 519                 if (amdgpu_sriov_vf(adev))
 520                         vega10_ih_irq_rearm(adev, ih);
 521         } else if (ih == &adev->irq.ih) {
 522                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
 523         } else if (ih == &adev->irq.ih1) {
 524                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr);
 525         } else if (ih == &adev->irq.ih2) {
 526                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr);
 527         }
 528 }
 529
 530 /**
 531  * vega10_ih_self_irq - dispatch work for ring 1 and 2
 532  *
 533  * @adev: amdgpu_device pointer
 534  * @source: irq source
 535  * @entry: IV with WPTR update
 536  *
 537  * Update the WPTR from the IV and schedule work to handle the entries.
 538  */
 539 static int vega10_ih_self_irq(struct amdgpu_device *adev,
 540                               struct amdgpu_irq_src *source,
 541                               struct amdgpu_iv_entry *entry)
 542 {
 543         uint32_t wptr = cpu_to_le32(entry->src_data[0]);
 544
 545         switch (entry->ring_id) {
 546         case 1:
 547                 *adev->irq.ih1.wptr_cpu = wptr;
 548                 schedule_work(&adev->irq.ih1_work);
 549                 break;
 550         case 2:
 551                 *adev->irq.ih2.wptr_cpu = wptr;
 552                 schedule_work(&adev->irq.ih2_work);
 553                 break;
 554         default: break;
 555         }
 556         return 0;
 557 }
 558
 559 static const struct amdgpu_irq_src_funcs vega10_ih_self_irq_funcs = {
 560         .process = vega10_ih_self_irq,
 561 };
 562
 563 static void vega10_ih_set_self_irq_funcs(struct amdgpu_device *adev)
 564 {
 565         adev->irq.self_irq.num_types = 0;
 566         adev->irq.self_irq.funcs = &vega10_ih_self_irq_funcs;
 567 }
 568
 569 static int vega10_ih_early_init(void *handle)
 570 {
 571         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 572
 573         vega10_ih_set_interrupt_funcs(adev);
 574         vega10_ih_set_self_irq_funcs(adev);
 575         return 0;
 576 }
 577
 578 static int vega10_ih_sw_init(void *handle)
 579 {
 580         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 581         int r;
 582
 583         r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0,
 584                               &adev->irq.self_irq);
 585         if (r)
 586                 return r;
 587
 588         r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
 589         if (r)
 590                 return r;
 591
 592         adev->irq.ih.use_doorbell = true;
 593         adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
 594
 595         r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
 596         if (r)
 597                 return r;
 598
 599         adev->irq.ih1.use_doorbell = true;
 600         adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
 601
 602         r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
 603         if (r)
 604                 return r;
 605
 606         adev->irq.ih2.use_doorbell = true;
 607         adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
 608
 609         r = amdgpu_irq_init(adev);
 610
 611         return r;
 612 }
 613
 614 static int vega10_ih_sw_fini(void *handle)
 615 {
 616         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 617
 618         amdgpu_irq_fini(adev);
 619         amdgpu_ih_ring_fini(adev, &adev->irq.ih2);
 620         amdgpu_ih_ring_fini(adev, &adev->irq.ih1);
 621         amdgpu_ih_ring_fini(adev, &adev->irq.ih);
 622
 623         return 0;
 624 }
 625
 626 static int vega10_ih_hw_init(void *handle)
 627 {
 628         int r;
 629         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 630
 631         r = vega10_ih_irq_init(adev);
 632         if (r)
 633                 return r;
 634
 635         return 0;
 636 }
 637
 638 static int vega10_ih_hw_fini(void *handle)
 639 {
 640         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 641
 642         vega10_ih_irq_disable(adev);
 643
 644         return 0;
 645 }
 646
 647 static int vega10_ih_suspend(void *handle)
 648 {
 649         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 650
 651         return vega10_ih_hw_fini(adev);
 652 }
 653
 654 static int vega10_ih_resume(void *handle)
 655 {
 656         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 657
 658         return vega10_ih_hw_init(adev);
 659 }
 660
 661 static bool vega10_ih_is_idle(void *handle)
 662 {
 663         /* todo */
 664         return true;
 665 }
 666
 667 static int vega10_ih_wait_for_idle(void *handle)
 668 {
 669         /* todo */
 670         return -ETIMEDOUT;
 671 }
 672
 673 static int vega10_ih_soft_reset(void *handle)
 674 {
 675         /* todo */
 676
 677         return 0;
 678 }
 679
 680 static void vega10_ih_update_clockgating_state(struct amdgpu_device *adev,
 681                                                bool enable)
 682 {
 683         uint32_t data, def, field_val;
 684
 685         if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
 686                 def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
 687                 field_val = enable ? 0 : 1;
 688                 /**
 689                  * Vega10 does not have IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE
 690                  * and IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field.
 691                  */
 692                 if (adev->asic_type > CHIP_VEGA10) {
 693                         data = REG_SET_FIELD(data, IH_CLK_CTRL,
 694                                      IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val);
 695                         data = REG_SET_FIELD(data, IH_CLK_CTRL,
 696                                      IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val);
 697                 }
 698
 699                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 700                                      DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
 701                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 702                                      OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
 703                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 704                                      LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
 705                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 706                                      DYN_CLK_SOFT_OVERRIDE, field_val);
 707                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 708                                      REG_CLK_SOFT_OVERRIDE, field_val);
 709                 if (def != data)
 710                         WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
 711         }
 712 }
 713
 714 static int vega10_ih_set_clockgating_state(void *handle,
 715                                           enum amd_clockgating_state state)
 716 {
 717         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 718
 719         vega10_ih_update_clockgating_state(adev,
 720                                 state == AMD_CG_STATE_GATE ? true : false);
 721         return 0;
 722
 723 }
 724
 725 static int vega10_ih_set_powergating_state(void *handle,
 726                                           enum amd_powergating_state state)
 727 {
 728         return 0;
 729 }
 730
 731 const struct amd_ip_funcs vega10_ih_ip_funcs = {
 732         .name = "vega10_ih",
 733         .early_init = vega10_ih_early_init,
 734         .late_init = NULL,
 735         .sw_init = vega10_ih_sw_init,
 736         .sw_fini = vega10_ih_sw_fini,
 737         .hw_init = vega10_ih_hw_init,
 738         .hw_fini = vega10_ih_hw_fini,
 739         .suspend = vega10_ih_suspend,
 740         .resume = vega10_ih_resume,
 741         .is_idle = vega10_ih_is_idle,
 742         .wait_for_idle = vega10_ih_wait_for_idle,
 743         .soft_reset = vega10_ih_soft_reset,
 744         .set_clockgating_state = vega10_ih_set_clockgating_state,
 745         .set_powergating_state = vega10_ih_set_powergating_state,
 746 };
 747
 748 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
 749         .get_wptr = vega10_ih_get_wptr,
 750         .decode_iv = vega10_ih_decode_iv,
 751         .set_rptr = vega10_ih_set_rptr
 752 };
 753
 754 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 755 {
 756         adev->irq.ih_funcs = &vega10_ih_funcs;
 757 }
 758
 759 const struct amdgpu_ip_block_version vega10_ih_ip_block =
 760 {
 761         .type = AMD_IP_BLOCK_TYPE_IH,
 762         .major = 4,
 763         .minor = 0,
 764         .rev = 0,
 765         .funcs = &vega10_ih_ip_funcs,
 766 };