?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / gpu / drm / amd / amdgpu / nv.c
blob2e0f8933410e656034dffb96bac8b827d3be21e4
1 /*
2 * Copyright 2019 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include <linux/firmware.h>
24 #include <linux/slab.h>
25 #include <linux/module.h>
26 #include <linux/pci.h>
27
28 #include "amdgpu.h"
29 #include "amdgpu_atombios.h"
30 #include "amdgpu_ih.h"
31 #include "amdgpu_uvd.h"
32 #include "amdgpu_vce.h"
33 #include "amdgpu_ucode.h"
34 #include "amdgpu_psp.h"
35 #include "amdgpu_smu.h"
36 #include "atom.h"
37 #include "amd_pcie.h"
38
39 #include "gc/gc_10_1_0_offset.h"
40 #include "gc/gc_10_1_0_sh_mask.h"
41 #include "hdp/hdp_5_0_0_offset.h"
42 #include "hdp/hdp_5_0_0_sh_mask.h"
43 #include "smuio/smuio_11_0_0_offset.h"
44
45 #include "soc15.h"
46 #include "soc15_common.h"
47 #include "gmc_v10_0.h"
48 #include "gfxhub_v2_0.h"
49 #include "mmhub_v2_0.h"
50 #include "nbio_v2_3.h"
51 #include "nv.h"
52 #include "navi10_ih.h"
53 #include "gfx_v10_0.h"
54 #include "sdma_v5_0.h"
55 #include "vcn_v2_0.h"
56 #include "jpeg_v2_0.h"
57 #include "dce_virtual.h"
58 #include "mes_v10_1.h"
59 #include "mxgpu_nv.h"
60
61 static const struct amd_ip_funcs nv_common_ip_funcs;
62
63 /*
64 * Indirect registers accessor
65 */
66 static u32 nv_pcie_rreg(struct amdgpu_device *adev, u32 reg)
67 {
68 unsigned long flags, address, data;
69 u32 r;
70 address = adev->nbio.funcs->get_pcie_index_offset(adev);
71 data = adev->nbio.funcs->get_pcie_data_offset(adev);
72
73 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
74 WREG32(address, reg);
75 (void)RREG32(address);
76 r = RREG32(data);
77 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
78 return r;
79 }
80
81 static void nv_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
82 {
83 unsigned long flags, address, data;
84
85 address = adev->nbio.funcs->get_pcie_index_offset(adev);
86 data = adev->nbio.funcs->get_pcie_data_offset(adev);
87
88 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
89 WREG32(address, reg);
90 (void)RREG32(address);
91 WREG32(data, v);
92 (void)RREG32(data);
93 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
94 }
95
96 static u32 nv_didt_rreg(struct amdgpu_device *adev, u32 reg)
97 {
98 unsigned long flags, address, data;
99 u32 r;
100
101 address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
102 data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
103
104 spin_lock_irqsave(&adev->didt_idx_lock, flags);
105 WREG32(address, (reg));
106 r = RREG32(data);
107 spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
108 return r;
109 }
110
111 static void nv_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
112 {
113 unsigned long flags, address, data;
114
115 address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX);
116 data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA);
117
118 spin_lock_irqsave(&adev->didt_idx_lock, flags);
119 WREG32(address, (reg));
120 WREG32(data, (v));
121 spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
122 }
123
124 static u32 nv_get_config_memsize(struct amdgpu_device *adev)
125 {
126 return adev->nbio.funcs->get_memsize(adev);
127 }
128
129 static u32 nv_get_xclk(struct amdgpu_device *adev)
130 {
131 return adev->clock.spll.reference_freq;
132 }
133
134
135 void nv_grbm_select(struct amdgpu_device *adev,
136 u32 me, u32 pipe, u32 queue, u32 vmid)
137 {
138 u32 grbm_gfx_cntl = 0;
139 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, PIPEID, pipe);
140 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, MEID, me);
141 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid);
142 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue);
143
144 WREG32(SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL), grbm_gfx_cntl);
145 }
146
147 static void nv_vga_set_state(struct amdgpu_device *adev, bool state)
148 {
149 /* todo */
150 }
151
152 static bool nv_read_disabled_bios(struct amdgpu_device *adev)
153 {
154 /* todo */
155 return false;
156 }
157
158 static bool nv_read_bios_from_rom(struct amdgpu_device *adev,
159 u8 *bios, u32 length_bytes)
160 {
161 u32 *dw_ptr;
162 u32 i, length_dw;
163
164 if (bios == NULL)
165 return false;
166 if (length_bytes == 0)
167 return false;
168 /* APU vbios image is part of sbios image */
169 if (adev->flags & AMD_IS_APU)
170 return false;
171
172 dw_ptr = (u32 *)bios;
173 length_dw = ALIGN(length_bytes, 4) / 4;
174
175 /* set rom index to 0 */
176 WREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_INDEX), 0);
177 /* read out the rom data */
178 for (i = 0; i < length_dw; i++)
179 dw_ptr[i] = RREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_DATA));
180
181 return true;
182 }
183
184 static struct soc15_allowed_register_entry nv_allowed_read_registers[] = {
185 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS)},
186 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS2)},
187 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE0)},
188 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE1)},
189 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE2)},
190 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE3)},
191 #if 0 /* TODO: will set it when SDMA header is available */
192 { SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_STATUS_REG)},
193 { SOC15_REG_ENTRY(SDMA1, 0, mmSDMA1_STATUS_REG)},
194 #endif
195 { SOC15_REG_ENTRY(GC, 0, mmCP_STAT)},
196 { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT1)},
197 { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT2)},
198 { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT3)},
199 { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_BUSY_STAT)},
200 { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STALLED_STAT1)},
201 { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STATUS)},
202 { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_BUSY_STAT)},
203 { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STALLED_STAT1)},
204 { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STATUS)},
205 { SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
206 };
207
208 static uint32_t nv_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
209 u32 sh_num, u32 reg_offset)
210 {
211 uint32_t val;
212
213 mutex_lock(&adev->grbm_idx_mutex);
214 if (se_num != 0xffffffff || sh_num != 0xffffffff)
215 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
216
217 val = RREG32(reg_offset);
218
219 if (se_num != 0xffffffff || sh_num != 0xffffffff)
220 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
221 mutex_unlock(&adev->grbm_idx_mutex);
222 return val;
223 }
224
225 static uint32_t nv_get_register_value(struct amdgpu_device *adev,
226 bool indexed, u32 se_num,
227 u32 sh_num, u32 reg_offset)
228 {
229 if (indexed) {
230 return nv_read_indexed_register(adev, se_num, sh_num, reg_offset);
231 } else {
232 if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
233 return adev->gfx.config.gb_addr_config;
234 return RREG32(reg_offset);
235 }
236 }
237
238 static int nv_read_register(struct amdgpu_device *adev, u32 se_num,
239 u32 sh_num, u32 reg_offset, u32 *value)
240 {
241 uint32_t i;
242 struct soc15_allowed_register_entry *en;
243
244 *value = 0;
245 for (i = 0; i < ARRAY_SIZE(nv_allowed_read_registers); i++) {
246 en = &nv_allowed_read_registers[i];
247 if (reg_offset !=
248 (adev->reg_offset[en->hwip][en->inst][en->seg] + en->reg_offset))
249 continue;
250
251 *value = nv_get_register_value(adev,
252 nv_allowed_read_registers[i].grbm_indexed,
253 se_num, sh_num, reg_offset);
254 return 0;
255 }
256 return -EINVAL;
257 }
258
259 #if 0
260 static void nv_gpu_pci_config_reset(struct amdgpu_device *adev)
261 {
262 u32 i;
263
264 dev_info(adev->dev, "GPU pci config reset\n");
265
266 /* disable BM */
267 pci_clear_master(adev->pdev);
268 /* reset */
269 amdgpu_pci_config_reset(adev);
270
271 udelay(100);
272
273 /* wait for asic to come out of reset */
274 for (i = 0; i < adev->usec_timeout; i++) {
275 u32 memsize = nbio_v2_3_get_memsize(adev);
276 if (memsize != 0xffffffff)
277 break;
278 udelay(1);
279 }
280
281 }
282 #endif
283
284 static int nv_asic_mode1_reset(struct amdgpu_device *adev)
285 {
286 u32 i;
287 int ret = 0;
288
289 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
290
291 dev_info(adev->dev, "GPU mode1 reset\n");
292
293 /* disable BM */
294 pci_clear_master(adev->pdev);
295
296 pci_save_state(adev->pdev);
297
298 ret = psp_gpu_reset(adev);
299 if (ret)
300 dev_err(adev->dev, "GPU mode1 reset failed\n");
301
302 pci_restore_state(adev->pdev);
303
304 /* wait for asic to come out of reset */
305 for (i = 0; i < adev->usec_timeout; i++) {
306 u32 memsize = adev->nbio.funcs->get_memsize(adev);
307
308 if (memsize != 0xffffffff)
309 break;
310 udelay(1);
311 }
312
313 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
314
315 return ret;
316 }
317
318 static bool nv_asic_supports_baco(struct amdgpu_device *adev)
319 {
320 struct smu_context *smu = &adev->smu;
321
322 if (smu_baco_is_support(smu))
323 return true;
324 else
325 return false;
326 }
327
328 static enum amd_reset_method
329 nv_asic_reset_method(struct amdgpu_device *adev)
330 {
331 struct smu_context *smu = &adev->smu;
332
333 if (!amdgpu_sriov_vf(adev) && smu_baco_is_support(smu))
334 return AMD_RESET_METHOD_BACO;
335 else
336 return AMD_RESET_METHOD_MODE1;
337 }
338
339 static int nv_asic_reset(struct amdgpu_device *adev)
340 {
341
342 /* FIXME: it doesn't work since vega10 */
343 #if 0
344 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
345
346 nv_gpu_pci_config_reset(adev);
347
348 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
349 #endif
350 int ret = 0;
351 struct smu_context *smu = &adev->smu;
352
353 if (nv_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
354 if (!adev->in_suspend)
355 amdgpu_inc_vram_lost(adev);
356 ret = smu_baco_enter(smu);
357 if (ret)
358 return ret;
359 ret = smu_baco_exit(smu);
360 if (ret)
361 return ret;
362 } else {
363 if (!adev->in_suspend)
364 amdgpu_inc_vram_lost(adev);
365 ret = nv_asic_mode1_reset(adev);
366 }
367
368 return ret;
369 }
370
371 static int nv_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
372 {
373 /* todo */
374 return 0;
375 }
376
377 static int nv_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
378 {
379 /* todo */
380 return 0;
381 }
382
383 static void nv_pcie_gen3_enable(struct amdgpu_device *adev)
384 {
385 if (pci_is_root_bus(adev->pdev->bus))
386 return;
387
388 if (amdgpu_pcie_gen2 == 0)
389 return;
390
391 if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
392 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
393 return;
394
395 /* todo */
396 }
397
398 static void nv_program_aspm(struct amdgpu_device *adev)
399 {
400
401 if (amdgpu_aspm == 0)
402 return;
403
404 /* todo */
405 }
406
407 static void nv_enable_doorbell_aperture(struct amdgpu_device *adev,
408 bool enable)
409 {
410 adev->nbio.funcs->enable_doorbell_aperture(adev, enable);
411 adev->nbio.funcs->enable_doorbell_selfring_aperture(adev, enable);
412 }
413
414 static const struct amdgpu_ip_block_version nv_common_ip_block =
415 {
416 .type = AMD_IP_BLOCK_TYPE_COMMON,
417 .major = 1,
418 .minor = 0,
419 .rev = 0,
420 .funcs = &nv_common_ip_funcs,
421 };
422
423 static int nv_reg_base_init(struct amdgpu_device *adev)
424 {
425 int r;
426
427 if (amdgpu_discovery) {
428 r = amdgpu_discovery_reg_base_init(adev);
429 if (r) {
430 DRM_WARN("failed to init reg base from ip discovery table, "
431 "fallback to legacy init method\n");
432 goto legacy_init;
433 }
434
435 return 0;
436 }
437
438 legacy_init:
439 switch (adev->asic_type) {
440 case CHIP_NAVI10:
441 navi10_reg_base_init(adev);
442 break;
443 case CHIP_NAVI14:
444 navi14_reg_base_init(adev);
445 break;
446 case CHIP_NAVI12:
447 navi12_reg_base_init(adev);
448 break;
449 default:
450 return -EINVAL;
451 }
452
453 return 0;
454 }
455
456 int nv_set_ip_blocks(struct amdgpu_device *adev)
457 {
458 int r;
459
460 /* Set IP register base before any HW register access */
461 r = nv_reg_base_init(adev);
462 if (r)
463 return r;
464
465 adev->nbio.funcs = &nbio_v2_3_funcs;
466 adev->nbio.hdp_flush_reg = &nbio_v2_3_hdp_flush_reg;
467
468 adev->nbio.funcs->detect_hw_virt(adev);
469
470 if (amdgpu_sriov_vf(adev))
471 adev->virt.ops = &xgpu_nv_virt_ops;
472
473 switch (adev->asic_type) {
474 case CHIP_NAVI10:
475 case CHIP_NAVI14:
476 amdgpu_device_ip_block_add(adev, &nv_common_ip_block);
477 amdgpu_device_ip_block_add(adev, &gmc_v10_0_ip_block);
478 amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block);
479 amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
480 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
481 !amdgpu_sriov_vf(adev))
482 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
483 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
484 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
485 #if defined(CONFIG_DRM_AMD_DC)
486 else if (amdgpu_device_has_dc_support(adev))
487 amdgpu_device_ip_block_add(adev, &dm_ip_block);
488 #endif
489 amdgpu_device_ip_block_add(adev, &gfx_v10_0_ip_block);
490 amdgpu_device_ip_block_add(adev, &sdma_v5_0_ip_block);
491 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT &&
492 !amdgpu_sriov_vf(adev))
493 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
494 amdgpu_device_ip_block_add(adev, &vcn_v2_0_ip_block);
495 amdgpu_device_ip_block_add(adev, &jpeg_v2_0_ip_block);
496 if (adev->enable_mes)
497 amdgpu_device_ip_block_add(adev, &mes_v10_1_ip_block);
498 break;
499 case CHIP_NAVI12:
500 amdgpu_device_ip_block_add(adev, &nv_common_ip_block);
501 amdgpu_device_ip_block_add(adev, &gmc_v10_0_ip_block);
502 amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block);
503 amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block);
504 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP &&
505 !amdgpu_sriov_vf(adev))
506 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
507 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev))
508 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
509 #if defined(CONFIG_DRM_AMD_DC)
510 else if (amdgpu_device_has_dc_support(adev))
511 amdgpu_device_ip_block_add(adev, &dm_ip_block);
512 #endif
513 amdgpu_device_ip_block_add(adev, &gfx_v10_0_ip_block);
514 amdgpu_device_ip_block_add(adev, &sdma_v5_0_ip_block);
515 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT &&
516 !amdgpu_sriov_vf(adev))
517 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block);
518 amdgpu_device_ip_block_add(adev, &vcn_v2_0_ip_block);
519 amdgpu_device_ip_block_add(adev, &jpeg_v2_0_ip_block);
520 break;
521 default:
522 return -EINVAL;
523 }
524
525 return 0;
526 }
527
528 static uint32_t nv_get_rev_id(struct amdgpu_device *adev)
529 {
530 return adev->nbio.funcs->get_rev_id(adev);
531 }
532
533 static void nv_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
534 {
535 adev->nbio.funcs->hdp_flush(adev, ring);
536 }
537
538 static void nv_invalidate_hdp(struct amdgpu_device *adev,
539 struct amdgpu_ring *ring)
540 {
541 if (!ring || !ring->funcs->emit_wreg) {
542 WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_READ_CACHE_INVALIDATE, 1);
543 } else {
544 amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET(
545 HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
546 }
547 }
548
549 static bool nv_need_full_reset(struct amdgpu_device *adev)
550 {
551 return true;
552 }
553
554 static void nv_get_pcie_usage(struct amdgpu_device *adev,
555 uint64_t *count0,
556 uint64_t *count1)
557 {
558 /*TODO*/
559 }
560
561 static bool nv_need_reset_on_init(struct amdgpu_device *adev)
562 {
563 #if 0
564 u32 sol_reg;
565
566 if (adev->flags & AMD_IS_APU)
567 return false;
568
569 /* Check sOS sign of life register to confirm sys driver and sOS
570 * are already been loaded.
571 */
572 sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81);
573 if (sol_reg)
574 return true;
575 #endif
576 /* TODO: re-enable it when mode1 reset is functional */
577 return false;
578 }
579
580 static uint64_t nv_get_pcie_replay_count(struct amdgpu_device *adev)
581 {
582
583 /* TODO
584 * dummy implement for pcie_replay_count sysfs interface
585 * */
586
587 return 0;
588 }
589
590 static void nv_init_doorbell_index(struct amdgpu_device *adev)
591 {
592 adev->doorbell_index.kiq = AMDGPU_NAVI10_DOORBELL_KIQ;
593 adev->doorbell_index.mec_ring0 = AMDGPU_NAVI10_DOORBELL_MEC_RING0;
594 adev->doorbell_index.mec_ring1 = AMDGPU_NAVI10_DOORBELL_MEC_RING1;
595 adev->doorbell_index.mec_ring2 = AMDGPU_NAVI10_DOORBELL_MEC_RING2;
596 adev->doorbell_index.mec_ring3 = AMDGPU_NAVI10_DOORBELL_MEC_RING3;
597 adev->doorbell_index.mec_ring4 = AMDGPU_NAVI10_DOORBELL_MEC_RING4;
598 adev->doorbell_index.mec_ring5 = AMDGPU_NAVI10_DOORBELL_MEC_RING5;
599 adev->doorbell_index.mec_ring6 = AMDGPU_NAVI10_DOORBELL_MEC_RING6;
600 adev->doorbell_index.mec_ring7 = AMDGPU_NAVI10_DOORBELL_MEC_RING7;
601 adev->doorbell_index.userqueue_start = AMDGPU_NAVI10_DOORBELL_USERQUEUE_START;
602 adev->doorbell_index.userqueue_end = AMDGPU_NAVI10_DOORBELL_USERQUEUE_END;
603 adev->doorbell_index.gfx_ring0 = AMDGPU_NAVI10_DOORBELL_GFX_RING0;
604 adev->doorbell_index.gfx_ring1 = AMDGPU_NAVI10_DOORBELL_GFX_RING1;
605 adev->doorbell_index.sdma_engine[0] = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0;
606 adev->doorbell_index.sdma_engine[1] = AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1;
607 adev->doorbell_index.ih = AMDGPU_NAVI10_DOORBELL_IH;
608 adev->doorbell_index.vcn.vcn_ring0_1 = AMDGPU_NAVI10_DOORBELL64_VCN0_1;
609 adev->doorbell_index.vcn.vcn_ring2_3 = AMDGPU_NAVI10_DOORBELL64_VCN2_3;
610 adev->doorbell_index.vcn.vcn_ring4_5 = AMDGPU_NAVI10_DOORBELL64_VCN4_5;
611 adev->doorbell_index.vcn.vcn_ring6_7 = AMDGPU_NAVI10_DOORBELL64_VCN6_7;
612 adev->doorbell_index.first_non_cp = AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP;
613 adev->doorbell_index.last_non_cp = AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP;
614
615 adev->doorbell_index.max_assignment = AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT << 1;
616 adev->doorbell_index.sdma_doorbell_range = 20;
617 }
618
619 static const struct amdgpu_asic_funcs nv_asic_funcs =
620 {
621 .read_disabled_bios = &nv_read_disabled_bios,
622 .read_bios_from_rom = &nv_read_bios_from_rom,
623 .read_register = &nv_read_register,
624 .reset = &nv_asic_reset,
625 .reset_method = &nv_asic_reset_method,
626 .set_vga_state = &nv_vga_set_state,
627 .get_xclk = &nv_get_xclk,
628 .set_uvd_clocks = &nv_set_uvd_clocks,
629 .set_vce_clocks = &nv_set_vce_clocks,
630 .get_config_memsize = &nv_get_config_memsize,
631 .flush_hdp = &nv_flush_hdp,
632 .invalidate_hdp = &nv_invalidate_hdp,
633 .init_doorbell_index = &nv_init_doorbell_index,
634 .need_full_reset = &nv_need_full_reset,
635 .get_pcie_usage = &nv_get_pcie_usage,
636 .need_reset_on_init = &nv_need_reset_on_init,
637 .get_pcie_replay_count = &nv_get_pcie_replay_count,
638 .supports_baco = &nv_asic_supports_baco,
639 };
640
641 static int nv_common_early_init(void *handle)
642 {
643 #define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE)
644 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
645
646 adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET;
647 adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET;
648 adev->smc_rreg = NULL;
649 adev->smc_wreg = NULL;
650 adev->pcie_rreg = &nv_pcie_rreg;
651 adev->pcie_wreg = &nv_pcie_wreg;
652
653 /* TODO: will add them during VCN v2 implementation */
654 adev->uvd_ctx_rreg = NULL;
655 adev->uvd_ctx_wreg = NULL;
656
657 adev->didt_rreg = &nv_didt_rreg;
658 adev->didt_wreg = &nv_didt_wreg;
659
660 adev->asic_funcs = &nv_asic_funcs;
661
662 adev->rev_id = nv_get_rev_id(adev);
663 adev->external_rev_id = 0xff;
664 switch (adev->asic_type) {
665 case CHIP_NAVI10:
666 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
667 AMD_CG_SUPPORT_GFX_CGCG |
668 AMD_CG_SUPPORT_IH_CG |
669 AMD_CG_SUPPORT_HDP_MGCG |
670 AMD_CG_SUPPORT_HDP_LS |
671 AMD_CG_SUPPORT_SDMA_MGCG |
672 AMD_CG_SUPPORT_SDMA_LS |
673 AMD_CG_SUPPORT_MC_MGCG |
674 AMD_CG_SUPPORT_MC_LS |
675 AMD_CG_SUPPORT_ATHUB_MGCG |
676 AMD_CG_SUPPORT_ATHUB_LS |
677 AMD_CG_SUPPORT_VCN_MGCG |
678 AMD_CG_SUPPORT_JPEG_MGCG |
679 AMD_CG_SUPPORT_BIF_MGCG |
680 AMD_CG_SUPPORT_BIF_LS;
681 adev->pg_flags = AMD_PG_SUPPORT_VCN |
682 AMD_PG_SUPPORT_VCN_DPG |
683 AMD_PG_SUPPORT_JPEG |
684 AMD_PG_SUPPORT_ATHUB;
685 adev->external_rev_id = adev->rev_id + 0x1;
686 break;
687 case CHIP_NAVI14:
688 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
689 AMD_CG_SUPPORT_GFX_CGCG |
690 AMD_CG_SUPPORT_IH_CG |
691 AMD_CG_SUPPORT_HDP_MGCG |
692 AMD_CG_SUPPORT_HDP_LS |
693 AMD_CG_SUPPORT_SDMA_MGCG |
694 AMD_CG_SUPPORT_SDMA_LS |
695 AMD_CG_SUPPORT_MC_MGCG |
696 AMD_CG_SUPPORT_MC_LS |
697 AMD_CG_SUPPORT_ATHUB_MGCG |
698 AMD_CG_SUPPORT_ATHUB_LS |
699 AMD_CG_SUPPORT_VCN_MGCG |
700 AMD_CG_SUPPORT_JPEG_MGCG |
701 AMD_CG_SUPPORT_BIF_MGCG |
702 AMD_CG_SUPPORT_BIF_LS;
703 adev->pg_flags = AMD_PG_SUPPORT_VCN |
704 AMD_PG_SUPPORT_JPEG |
705 AMD_PG_SUPPORT_VCN_DPG;
706 adev->external_rev_id = adev->rev_id + 20;
707 break;
708 case CHIP_NAVI12:
709 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
710 AMD_CG_SUPPORT_GFX_MGLS |
711 AMD_CG_SUPPORT_GFX_CGCG |
712 AMD_CG_SUPPORT_GFX_CP_LS |
713 AMD_CG_SUPPORT_GFX_RLC_LS |
714 AMD_CG_SUPPORT_IH_CG |
715 AMD_CG_SUPPORT_HDP_MGCG |
716 AMD_CG_SUPPORT_HDP_LS |
717 AMD_CG_SUPPORT_SDMA_MGCG |
718 AMD_CG_SUPPORT_SDMA_LS |
719 AMD_CG_SUPPORT_MC_MGCG |
720 AMD_CG_SUPPORT_MC_LS |
721 AMD_CG_SUPPORT_ATHUB_MGCG |
722 AMD_CG_SUPPORT_ATHUB_LS |
723 AMD_CG_SUPPORT_VCN_MGCG |
724 AMD_CG_SUPPORT_JPEG_MGCG;
725 adev->pg_flags = AMD_PG_SUPPORT_VCN |
726 AMD_PG_SUPPORT_VCN_DPG |
727 AMD_PG_SUPPORT_JPEG |
728 AMD_PG_SUPPORT_ATHUB;
729 /* guest vm gets 0xffffffff when reading RCC_DEV0_EPF0_STRAP0,
730 * as a consequence, the rev_id and external_rev_id are wrong.
731 * workaround it by hardcoding rev_id to 0 (default value).
732 */
733 if (amdgpu_sriov_vf(adev))
734 adev->rev_id = 0;
735 adev->external_rev_id = adev->rev_id + 0xa;
736 break;
737 default:
738 /* FIXME: not supported yet */
739 return -EINVAL;
740 }
741
742 if (amdgpu_sriov_vf(adev)) {
743 amdgpu_virt_init_setting(adev);
744 xgpu_nv_mailbox_set_irq_funcs(adev);
745 }
746
747 return 0;
748 }
749
750 static int nv_common_late_init(void *handle)
751 {
752 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
753
754 if (amdgpu_sriov_vf(adev))
755 xgpu_nv_mailbox_get_irq(adev);
756
757 return 0;
758 }
759
760 static int nv_common_sw_init(void *handle)
761 {
762 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
763
764 if (amdgpu_sriov_vf(adev))
765 xgpu_nv_mailbox_add_irq_id(adev);
766
767 return 0;
768 }
769
770 static int nv_common_sw_fini(void *handle)
771 {
772 return 0;
773 }
774
775 static int nv_common_hw_init(void *handle)
776 {
777 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
778
779 /* enable pcie gen2/3 link */
780 nv_pcie_gen3_enable(adev);
781 /* enable aspm */
782 nv_program_aspm(adev);
783 /* setup nbio registers */
784 adev->nbio.funcs->init_registers(adev);
785 /* remap HDP registers to a hole in mmio space,
786 * for the purpose of expose those registers
787 * to process space
788 */
789 if (adev->nbio.funcs->remap_hdp_registers)
790 adev->nbio.funcs->remap_hdp_registers(adev);
791 /* enable the doorbell aperture */
792 nv_enable_doorbell_aperture(adev, true);
793
794 return 0;
795 }
796
797 static int nv_common_hw_fini(void *handle)
798 {
799 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
800
801 /* disable the doorbell aperture */
802 nv_enable_doorbell_aperture(adev, false);
803
804 return 0;
805 }
806
807 static int nv_common_suspend(void *handle)
808 {
809 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
810
811 return nv_common_hw_fini(adev);
812 }
813
814 static int nv_common_resume(void *handle)
815 {
816 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
817
818 return nv_common_hw_init(adev);
819 }
820
821 static bool nv_common_is_idle(void *handle)
822 {
823 return true;
824 }
825
826 static int nv_common_wait_for_idle(void *handle)
827 {
828 return 0;
829 }
830
831 static int nv_common_soft_reset(void *handle)
832 {
833 return 0;
834 }
835
836 static void nv_update_hdp_mem_power_gating(struct amdgpu_device *adev,
837 bool enable)
838 {
839 uint32_t hdp_clk_cntl, hdp_clk_cntl1;
840 uint32_t hdp_mem_pwr_cntl;
841
842 if (!(adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS |
843 AMD_CG_SUPPORT_HDP_DS |
844 AMD_CG_SUPPORT_HDP_SD)))
845 return;
846
847 hdp_clk_cntl = hdp_clk_cntl1 = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL);
848 hdp_mem_pwr_cntl = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL);
849
850 /* Before doing clock/power mode switch,
851 * forced on IPH & RC clock */
852 hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL,
853 IPH_MEM_CLK_SOFT_OVERRIDE, 1);
854 hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL,
855 RC_MEM_CLK_SOFT_OVERRIDE, 1);
856 WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl);
857
858 /* HDP 5.0 doesn't support dynamic power mode switch,
859 * disable clock and power gating before any changing */
860 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
861 IPH_MEM_POWER_CTRL_EN, 0);
862 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
863 IPH_MEM_POWER_LS_EN, 0);
864 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
865 IPH_MEM_POWER_DS_EN, 0);
866 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
867 IPH_MEM_POWER_SD_EN, 0);
868 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
869 RC_MEM_POWER_CTRL_EN, 0);
870 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
871 RC_MEM_POWER_LS_EN, 0);
872 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
873 RC_MEM_POWER_DS_EN, 0);
874 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL,
875 RC_MEM_POWER_SD_EN, 0);
876 WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl);
877
878 /* only one clock gating mode (LS/DS/SD) can be enabled */
879 if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) {
880 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
881 HDP_MEM_POWER_CTRL,
882 IPH_MEM_POWER_LS_EN, enable);
883 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
884 HDP_MEM_POWER_CTRL,
885 RC_MEM_POWER_LS_EN, enable);
886 } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_DS) {
887 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
888 HDP_MEM_POWER_CTRL,
889 IPH_MEM_POWER_DS_EN, enable);
890 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
891 HDP_MEM_POWER_CTRL,
892 RC_MEM_POWER_DS_EN, enable);
893 } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_SD) {
894 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
895 HDP_MEM_POWER_CTRL,
896 IPH_MEM_POWER_SD_EN, enable);
897 /* RC should not use shut down mode, fallback to ds */
898 hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl,
899 HDP_MEM_POWER_CTRL,
900 RC_MEM_POWER_DS_EN, enable);
901 }
902
903 WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl);
904
905 /* restore IPH & RC clock override after clock/power mode changing */
906 WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl1);
907 }
908
909 static void nv_update_hdp_clock_gating(struct amdgpu_device *adev,
910 bool enable)
911 {
912 uint32_t hdp_clk_cntl;
913
914 if (!(adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
915 return;
916
917 hdp_clk_cntl = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL);
918
919 if (enable) {
920 hdp_clk_cntl &=
921 ~(uint32_t)
922 (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK |
923 HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK |
924 HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK |
925 HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK |
926 HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK |
927 HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK);
928 } else {
929 hdp_clk_cntl |= HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK |
930 HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK |
931 HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK |
932 HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK |
933 HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK |
934 HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK;
935 }
936
937 WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl);
938 }
939
940 static int nv_common_set_clockgating_state(void *handle,
941 enum amd_clockgating_state state)
942 {
943 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
944
945 if (amdgpu_sriov_vf(adev))
946 return 0;
947
948 switch (adev->asic_type) {
949 case CHIP_NAVI10:
950 case CHIP_NAVI14:
951 case CHIP_NAVI12:
952 adev->nbio.funcs->update_medium_grain_clock_gating(adev,
953 state == AMD_CG_STATE_GATE ? true : false);
954 adev->nbio.funcs->update_medium_grain_light_sleep(adev,
955 state == AMD_CG_STATE_GATE ? true : false);
956 nv_update_hdp_mem_power_gating(adev,
957 state == AMD_CG_STATE_GATE ? true : false);
958 nv_update_hdp_clock_gating(adev,
959 state == AMD_CG_STATE_GATE ? true : false);
960 break;
961 default:
962 break;
963 }
964 return 0;
965 }
966
967 static int nv_common_set_powergating_state(void *handle,
968 enum amd_powergating_state state)
969 {
970 /* TODO */
971 return 0;
972 }
973
974 static void nv_common_get_clockgating_state(void *handle, u32 *flags)
975 {
976 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
977 uint32_t tmp;
978
979 if (amdgpu_sriov_vf(adev))
980 *flags = 0;
981
982 adev->nbio.funcs->get_clockgating_state(adev, flags);
983
984 /* AMD_CG_SUPPORT_HDP_MGCG */
985 tmp = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL);
986 if (!(tmp & (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK |
987 HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK |
988 HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK |
989 HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK |
990 HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK |
991 HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK)))
992 *flags |= AMD_CG_SUPPORT_HDP_MGCG;
993
994 /* AMD_CG_SUPPORT_HDP_LS/DS/SD */
995 tmp = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL);
996 if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK)
997 *flags |= AMD_CG_SUPPORT_HDP_LS;
998 else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK)
999 *flags |= AMD_CG_SUPPORT_HDP_DS;
1000 else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK)
1001 *flags |= AMD_CG_SUPPORT_HDP_SD;
1002
1003 return;
1004 }
1005
1006 static const struct amd_ip_funcs nv_common_ip_funcs = {
1007 .name = "nv_common",
1008 .early_init = nv_common_early_init,
1009 .late_init = nv_common_late_init,
1010 .sw_init = nv_common_sw_init,
1011 .sw_fini = nv_common_sw_fini,
1012 .hw_init = nv_common_hw_init,
1013 .hw_fini = nv_common_hw_fini,
1014 .suspend = nv_common_suspend,
1015 .resume = nv_common_resume,
1016 .is_idle = nv_common_is_idle,
1017 .wait_for_idle = nv_common_wait_for_idle,
1018 .soft_reset = nv_common_soft_reset,
1019 .set_clockgating_state = nv_common_set_clockgating_state,
1020 .set_powergating_state = nv_common_set_powergating_state,
1021 .get_clockgating_state = nv_common_get_clockgating_state,
1022 };
Linux with added FPC-III support