drivers/gpu/drm/amd/amdgpu/navi10_ih.c

   1 /*
   2  * Copyright 2019 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/pci.h>
  25
  26 #include "amdgpu.h"
  27 #include "amdgpu_ih.h"
  28
  29 #include "oss/osssys_5_0_0_offset.h"
  30 #include "oss/osssys_5_0_0_sh_mask.h"
  31
  32 #include "soc15_common.h"
  33 #include "navi10_ih.h"
  34
  35
  36 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
  37
  38 /**
  39  * navi10_ih_enable_interrupts - Enable the interrupt ring buffer
  40  *
  41  * @adev: amdgpu_device pointer
  42  *
  43  * Enable the interrupt ring buffer (NAVI10).
  44  */
  45 static void navi10_ih_enable_interrupts(struct amdgpu_device *adev)
  46 {
  47         u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
  48
  49         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
  50         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
  51         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
  52         adev->irq.ih.enabled = true;
  53 }
  54
  55 /**
  56  * navi10_ih_disable_interrupts - Disable the interrupt ring buffer
  57  *
  58  * @adev: amdgpu_device pointer
  59  *
  60  * Disable the interrupt ring buffer (NAVI10).
  61  */
  62 static void navi10_ih_disable_interrupts(struct amdgpu_device *adev)
  63 {
  64         u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
  65
  66         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
  67         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
  68         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
  69         /* set rptr, wptr to 0 */
  70         WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
  71         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
  72         adev->irq.ih.enabled = false;
  73         adev->irq.ih.rptr = 0;
  74 }
  75
  76 static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
  77 {
  78         int rb_bufsz = order_base_2(ih->ring_size / 4);
  79
  80         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
  81                                    MC_SPACE, ih->use_bus_addr ? 1 : 4);
  82         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
  83                                    WPTR_OVERFLOW_CLEAR, 1);
  84         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
  85                                    WPTR_OVERFLOW_ENABLE, 1);
  86         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
  87         /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
  88          * value is written to memory
  89          */
  90         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
  91                                    WPTR_WRITEBACK_ENABLE, 1);
  92         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
  93         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
  94         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
  95
  96         return ih_rb_cntl;
  97 }
  98
  99 /**
 100  * navi10_ih_irq_init - init and enable the interrupt ring
 101  *
 102  * @adev: amdgpu_device pointer
 103  *
 104  * Allocate a ring buffer for the interrupt controller,
 105  * enable the RLC, disable interrupts, enable the IH
 106  * ring buffer and enable it (NAVI).
 107  * Called at device load and reume.
 108  * Returns 0 for success, errors for failure.
 109  */
 110 static int navi10_ih_irq_init(struct amdgpu_device *adev)
 111 {
 112         struct amdgpu_ih_ring *ih = &adev->irq.ih;
 113         u32 ih_rb_cntl, ih_doorbell_rtpr, ih_chicken;
 114         u32 tmp;
 115
 116         /* disable irqs */
 117         navi10_ih_disable_interrupts(adev);
 118
 119         adev->nbio.funcs->ih_control(adev);
 120
 121         /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
 122         WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
 123         WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
 124
 125         ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
 126         ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl);
 127         ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
 128                                    !!adev->irq.msi_enabled);
 129
 130         if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) {
 131                 if (ih->use_bus_addr) {
 132                         ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
 133                         ih_chicken = REG_SET_FIELD(ih_chicken,
 134                                         IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1);
 135                         WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
 136                 }
 137         }
 138
 139         WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
 140
 141         /* set the writeback address whether it's enabled or not */
 142         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
 143                      lower_32_bits(ih->wptr_addr));
 144         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
 145                      upper_32_bits(ih->wptr_addr) & 0xFFFF);
 146
 147         /* set rptr, wptr to 0 */
 148         WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
 149         WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
 150
 151         ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
 152         if (ih->use_doorbell) {
 153                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 154                                                  IH_DOORBELL_RPTR, OFFSET,
 155                                                  ih->doorbell_index);
 156                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 157                                                  IH_DOORBELL_RPTR, ENABLE, 1);
 158         } else {
 159                 ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
 160                                                  IH_DOORBELL_RPTR, ENABLE, 0);
 161         }
 162         WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
 163
 164         adev->nbio.funcs->ih_doorbell_range(adev, ih->use_doorbell,
 165                                             ih->doorbell_index);
 166
 167         tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
 168         tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
 169                             CLIENT18_IS_STORM_CLIENT, 1);
 170         WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp);
 171
 172         tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL);
 173         tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1);
 174         WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp);
 175
 176         pci_set_master(adev->pdev);
 177
 178         /* enable interrupts */
 179         navi10_ih_enable_interrupts(adev);
 180
 181         return 0;
 182 }
 183
 184 /**
 185  * navi10_ih_irq_disable - disable interrupts
 186  *
 187  * @adev: amdgpu_device pointer
 188  *
 189  * Disable interrupts on the hw (NAVI10).
 190  */
 191 static void navi10_ih_irq_disable(struct amdgpu_device *adev)
 192 {
 193         navi10_ih_disable_interrupts(adev);
 194
 195         /* Wait and acknowledge irq */
 196         mdelay(1);
 197 }
 198
 199 /**
 200  * navi10_ih_get_wptr - get the IH ring buffer wptr
 201  *
 202  * @adev: amdgpu_device pointer
 203  *
 204  * Get the IH ring buffer wptr from either the register
 205  * or the writeback memory buffer (NAVI10).  Also check for
 206  * ring buffer overflow and deal with it.
 207  * Returns the value of the wptr.
 208  */
 209 static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,
 210                               struct amdgpu_ih_ring *ih)
 211 {
 212         u32 wptr, reg, tmp;
 213
 214         wptr = le32_to_cpu(*ih->wptr_cpu);
 215
 216         if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 217                 goto out;
 218
 219         reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
 220         wptr = RREG32_NO_KIQ(reg);
 221         if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
 222                 goto out;
 223         wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
 224
 225         /* When a ring buffer overflow happen start parsing interrupt
 226          * from the last not overwritten vector (wptr + 32). Hopefully
 227          * this should allow us to catch up.
 228          */
 229         tmp = (wptr + 32) & ih->ptr_mask;
 230         dev_warn(adev->dev, "IH ring buffer overflow "
 231                  "(0x%08X, 0x%08X, 0x%08X)\n",
 232                  wptr, ih->rptr, tmp);
 233         ih->rptr = tmp;
 234
 235         reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
 236         tmp = RREG32_NO_KIQ(reg);
 237         tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
 238         WREG32_NO_KIQ(reg, tmp);
 239 out:
 240         return (wptr & ih->ptr_mask);
 241 }
 242
 243 /**
 244  * navi10_ih_decode_iv - decode an interrupt vector
 245  *
 246  * @adev: amdgpu_device pointer
 247  *
 248  * Decodes the interrupt vector at the current rptr
 249  * position and also advance the position.
 250  */
 251 static void navi10_ih_decode_iv(struct amdgpu_device *adev,
 252                                 struct amdgpu_ih_ring *ih,
 253                                 struct amdgpu_iv_entry *entry)
 254 {
 255         /* wptr/rptr are in bytes! */
 256         u32 ring_index = ih->rptr >> 2;
 257         uint32_t dw[8];
 258
 259         dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
 260         dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
 261         dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
 262         dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
 263         dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
 264         dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
 265         dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
 266         dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
 267
 268         entry->client_id = dw[0] & 0xff;
 269         entry->src_id = (dw[0] >> 8) & 0xff;
 270         entry->ring_id = (dw[0] >> 16) & 0xff;
 271         entry->vmid = (dw[0] >> 24) & 0xf;
 272         entry->vmid_src = (dw[0] >> 31);
 273         entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32);
 274         entry->timestamp_src = dw[2] >> 31;
 275         entry->pasid = dw[3] & 0xffff;
 276         entry->pasid_src = dw[3] >> 31;
 277         entry->src_data[0] = dw[4];
 278         entry->src_data[1] = dw[5];
 279         entry->src_data[2] = dw[6];
 280         entry->src_data[3] = dw[7];
 281
 282         /* wptr/rptr are in bytes! */
 283         ih->rptr += 32;
 284 }
 285
 286 /**
 287  * navi10_ih_set_rptr - set the IH ring buffer rptr
 288  *
 289  * @adev: amdgpu_device pointer
 290  *
 291  * Set the IH ring buffer rptr.
 292  */
 293 static void navi10_ih_set_rptr(struct amdgpu_device *adev,
 294                                struct amdgpu_ih_ring *ih)
 295 {
 296         if (ih->use_doorbell) {
 297                 /* XXX check if swapping is necessary on BE */
 298                 *ih->rptr_cpu = ih->rptr;
 299                 WDOORBELL32(ih->doorbell_index, ih->rptr);
 300         } else
 301                 WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
 302 }
 303
 304 static int navi10_ih_early_init(void *handle)
 305 {
 306         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 307
 308         navi10_ih_set_interrupt_funcs(adev);
 309         return 0;
 310 }
 311
 312 static int navi10_ih_sw_init(void *handle)
 313 {
 314         int r;
 315         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 316         bool use_bus_addr;
 317
 318         /* use gpu virtual address for ih ring
 319          * until ih_checken is programmed to allow
 320          * use bus address for ih ring by psp bl */
 321         use_bus_addr =
 322                 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) ? false : true;
 323         r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, use_bus_addr);
 324         if (r)
 325                 return r;
 326
 327         adev->irq.ih.use_doorbell = true;
 328         adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
 329
 330         r = amdgpu_irq_init(adev);
 331
 332         return r;
 333 }
 334
 335 static int navi10_ih_sw_fini(void *handle)
 336 {
 337         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 338
 339         amdgpu_irq_fini(adev);
 340         amdgpu_ih_ring_fini(adev, &adev->irq.ih);
 341
 342         return 0;
 343 }
 344
 345 static int navi10_ih_hw_init(void *handle)
 346 {
 347         int r;
 348         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 349
 350         r = navi10_ih_irq_init(adev);
 351         if (r)
 352                 return r;
 353
 354         return 0;
 355 }
 356
 357 static int navi10_ih_hw_fini(void *handle)
 358 {
 359         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 360
 361         navi10_ih_irq_disable(adev);
 362
 363         return 0;
 364 }
 365
 366 static int navi10_ih_suspend(void *handle)
 367 {
 368         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 369
 370         return navi10_ih_hw_fini(adev);
 371 }
 372
 373 static int navi10_ih_resume(void *handle)
 374 {
 375         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 376
 377         return navi10_ih_hw_init(adev);
 378 }
 379
 380 static bool navi10_ih_is_idle(void *handle)
 381 {
 382         /* todo */
 383         return true;
 384 }
 385
 386 static int navi10_ih_wait_for_idle(void *handle)
 387 {
 388         /* todo */
 389         return -ETIMEDOUT;
 390 }
 391
 392 static int navi10_ih_soft_reset(void *handle)
 393 {
 394         /* todo */
 395         return 0;
 396 }
 397
 398 static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev,
 399                                                bool enable)
 400 {
 401         uint32_t data, def, field_val;
 402
 403         if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
 404                 def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
 405                 field_val = enable ? 0 : 1;
 406                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 407                                      DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
 408                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 409                                      OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
 410                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 411                                      LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
 412                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 413                                      DYN_CLK_SOFT_OVERRIDE, field_val);
 414                 data = REG_SET_FIELD(data, IH_CLK_CTRL,
 415                                      REG_CLK_SOFT_OVERRIDE, field_val);
 416                 if (def != data)
 417                         WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
 418         }
 419
 420         return;
 421 }
 422
 423 static int navi10_ih_set_clockgating_state(void *handle,
 424                                            enum amd_clockgating_state state)
 425 {
 426         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 427
 428         navi10_ih_update_clockgating_state(adev,
 429                                 state == AMD_CG_STATE_GATE ? true : false);
 430         return 0;
 431 }
 432
 433 static int navi10_ih_set_powergating_state(void *handle,
 434                                            enum amd_powergating_state state)
 435 {
 436         return 0;
 437 }
 438
 439 static void navi10_ih_get_clockgating_state(void *handle, u32 *flags)
 440 {
 441         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 442
 443         if (!RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL))
 444                 *flags |= AMD_CG_SUPPORT_IH_CG;
 445
 446         return;
 447 }
 448
 449 static const struct amd_ip_funcs navi10_ih_ip_funcs = {
 450         .name = "navi10_ih",
 451         .early_init = navi10_ih_early_init,
 452         .late_init = NULL,
 453         .sw_init = navi10_ih_sw_init,
 454         .sw_fini = navi10_ih_sw_fini,
 455         .hw_init = navi10_ih_hw_init,
 456         .hw_fini = navi10_ih_hw_fini,
 457         .suspend = navi10_ih_suspend,
 458         .resume = navi10_ih_resume,
 459         .is_idle = navi10_ih_is_idle,
 460         .wait_for_idle = navi10_ih_wait_for_idle,
 461         .soft_reset = navi10_ih_soft_reset,
 462         .set_clockgating_state = navi10_ih_set_clockgating_state,
 463         .set_powergating_state = navi10_ih_set_powergating_state,
 464         .get_clockgating_state = navi10_ih_get_clockgating_state,
 465 };
 466
 467 static const struct amdgpu_ih_funcs navi10_ih_funcs = {
 468         .get_wptr = navi10_ih_get_wptr,
 469         .decode_iv = navi10_ih_decode_iv,
 470         .set_rptr = navi10_ih_set_rptr
 471 };
 472
 473 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
 474 {
 475         if (adev->irq.ih_funcs == NULL)
 476                 adev->irq.ih_funcs = &navi10_ih_funcs;
 477 }
 478
 479 const struct amdgpu_ip_block_version navi10_ih_ip_block =
 480 {
 481         .type = AMD_IP_BLOCK_TYPE_IH,
 482         .major = 5,
 483         .minor = 0,
 484         .rev = 0,
 485         .funcs = &navi10_ih_ip_funcs,
 486 };