?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / gpu / drm / amd / amdgpu / gfx_v9_0.c
blob46ab46757b25756afa0c8e37ce57b9441ec702bd
1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "soc15.h"
33 #include "soc15d.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
36
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
39
40 #include "vega10_enum.h"
41 #include "hdp/hdp_4_0_offset.h"
42
43 #include "soc15_common.h"
44 #include "clearstate_gfx9.h"
45 #include "v9_structs.h"
46
47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
48
49 #include "amdgpu_ras.h"
50
51 #define GFX9_NUM_GFX_RINGS 1
52 #define GFX9_MEC_HPD_SIZE 4096
53 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
54 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
55
56 #define mmPWR_MISC_CNTL_STATUS 0x0183
57 #define mmPWR_MISC_CNTL_STATUS_BASE_IDX 0
58 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN__SHIFT 0x0
59 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT 0x1
60 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK 0x00000001L
61 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK 0x00000006L
62
63 #define mmGCEA_PROBE_MAP 0x070c
64 #define mmGCEA_PROBE_MAP_BASE_IDX 0
65
66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
68 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
72
73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
75 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
79
80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
82 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
86
87 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
89 MODULE_FIRMWARE("amdgpu/raven_me.bin");
90 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
93
94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
96 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
101
102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
104 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
109
110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
111 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin");
112 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
113
114 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
115 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
116 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
118 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin");
119 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
120
121 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03
122 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
123 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04
124 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
125 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09
126 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
127 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
128 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
129 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
130 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
131 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
132 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
133
134 enum ta_ras_gfx_subblock {
135 /*CPC*/
136 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
137 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
138 TA_RAS_BLOCK__GFX_CPC_UCODE,
139 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
140 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
141 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
142 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
143 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
144 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
145 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
146 /* CPF*/
147 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
148 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
149 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
150 TA_RAS_BLOCK__GFX_CPF_TAG,
151 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
152 /* CPG*/
153 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
154 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
155 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
156 TA_RAS_BLOCK__GFX_CPG_TAG,
157 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
158 /* GDS*/
159 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
160 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
161 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
162 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
163 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
164 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
165 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
166 /* SPI*/
167 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
168 /* SQ*/
169 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
170 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
171 TA_RAS_BLOCK__GFX_SQ_LDS_D,
172 TA_RAS_BLOCK__GFX_SQ_LDS_I,
173 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
174 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
175 /* SQC (3 ranges)*/
176 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
177 /* SQC range 0*/
178 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
179 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
180 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
181 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
182 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
183 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
184 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
185 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
186 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
187 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
188 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
189 /* SQC range 1*/
190 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
191 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
192 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
193 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
194 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
195 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
196 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
197 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
198 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
199 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
200 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
201 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
202 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
203 /* SQC range 2*/
204 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
205 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
206 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
207 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
208 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
209 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
210 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
212 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
213 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
215 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
217 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
218 /* TA*/
219 TA_RAS_BLOCK__GFX_TA_INDEX_START,
220 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
221 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
222 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
223 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
224 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
225 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
226 /* TCA*/
227 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
228 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
229 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
230 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
231 /* TCC (5 sub-ranges)*/
232 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
233 /* TCC range 0*/
234 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
235 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
236 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
237 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
238 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
239 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
240 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
241 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
242 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
243 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
244 /* TCC range 1*/
245 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
246 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
247 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
248 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
249 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
250 /* TCC range 2*/
251 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
252 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
253 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
254 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
255 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
256 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
257 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
258 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
259 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
260 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
261 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
262 /* TCC range 3*/
263 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
264 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
265 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
266 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
267 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
268 /* TCC range 4*/
269 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
270 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
271 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
272 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
273 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
274 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
275 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
276 /* TCI*/
277 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
278 /* TCP*/
279 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
280 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
281 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
282 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
283 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
284 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
285 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
286 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
287 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
288 /* TD*/
289 TA_RAS_BLOCK__GFX_TD_INDEX_START,
290 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
291 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
292 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
293 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
294 /* EA (3 sub-ranges)*/
295 TA_RAS_BLOCK__GFX_EA_INDEX_START,
296 /* EA range 0*/
297 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
298 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
299 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
300 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
301 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
302 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
303 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
304 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
305 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
306 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
307 /* EA range 1*/
308 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
309 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
310 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
311 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
312 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
313 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
314 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
315 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
316 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
317 /* EA range 2*/
318 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
319 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
320 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
321 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
322 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
323 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
324 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
325 /* UTC VM L2 bank*/
326 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
327 /* UTC VM walker*/
328 TA_RAS_BLOCK__UTC_VML2_WALKER,
329 /* UTC ATC L2 2MB cache*/
330 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
331 /* UTC ATC L2 4KB cache*/
332 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
333 TA_RAS_BLOCK__GFX_MAX
334 };
335
336 struct ras_gfx_subblock {
337 unsigned char *name;
338 int ta_subblock;
339 int hw_supported_error_type;
340 int sw_supported_error_type;
341 };
342
343 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
344 [AMDGPU_RAS_BLOCK__##subblock] = { \
345 #subblock, \
346 TA_RAS_BLOCK__##subblock, \
347 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
348 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
349 }
350
351 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
352 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
353 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
354 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
355 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
356 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
357 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
358 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
359 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
360 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
361 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
362 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
363 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
364 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
365 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
366 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
367 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
368 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
369 0),
370 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
371 0),
372 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
373 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
374 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
375 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
376 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
377 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
378 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
379 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
380 0, 0),
381 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
382 0),
383 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
384 0, 0),
385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
386 0),
387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
388 0, 0),
389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
390 0),
391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
392 1),
393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
394 0, 0, 0),
395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
396 0),
397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
398 0),
399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
400 0),
401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
402 0),
403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
404 0),
405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
406 0, 0),
407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
408 0),
409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
410 0),
411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
412 0, 0, 0),
413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
414 0),
415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
416 0),
417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
418 0),
419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
420 0),
421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
422 0),
423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
424 0, 0),
425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
426 0),
427 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
428 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
429 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
430 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
431 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
432 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
433 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
434 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
435 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
436 1),
437 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
438 1),
439 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
440 1),
441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
442 0),
443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
444 0),
445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
457 0),
458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
460 0),
461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
462 0, 0),
463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
464 0),
465 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
466 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
467 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
468 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
469 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
470 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
471 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
472 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
473 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
474 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
475 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
476 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
477 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
478 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
479 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
480 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
481 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
495 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
496 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
497 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
498 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
499 };
500
501 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
502 {
503 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
504 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
505 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
506 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
507 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
508 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
523 };
524
525 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
526 {
527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
545 };
546
547 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
548 {
549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
560 };
561
562 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
563 {
564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
588 };
589
590 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
591 {
592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
597 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
599 };
600
601 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
602 {
603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
622 };
623
624 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
625 {
626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
638 };
639
640 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
641 {
642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
645 };
646
647 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
648 {
649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
650 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
665 };
666
667 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
668 {
669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
682 };
683
684 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
685 {
686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
696 };
697
698 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
699 {
700 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
701 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
702 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
703 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
704 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
705 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
706 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
707 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
708 };
709
710 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
711 {
712 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
713 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
714 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
715 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
716 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
717 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
718 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
719 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
720 };
721
722 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
723 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
724 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
725 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
726
727 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
728 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
729 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
730 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
731 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
732 struct amdgpu_cu_info *cu_info);
733 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
734 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
735 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
736 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
737 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
738 void *ras_error_status);
739 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
740 void *inject_if);
741
742 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
743 uint64_t queue_mask)
744 {
745 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
746 amdgpu_ring_write(kiq_ring,
747 PACKET3_SET_RESOURCES_VMID_MASK(0) |
748 /* vmid_mask:0* queue_type:0 (KIQ) */
749 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
750 amdgpu_ring_write(kiq_ring,
751 lower_32_bits(queue_mask)); /* queue mask lo */
752 amdgpu_ring_write(kiq_ring,
753 upper_32_bits(queue_mask)); /* queue mask hi */
754 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
755 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
756 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
757 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
758 }
759
760 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
761 struct amdgpu_ring *ring)
762 {
763 struct amdgpu_device *adev = kiq_ring->adev;
764 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
765 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
766 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
767
768 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
769 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
770 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
771 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
772 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
773 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
774 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
775 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
776 /*queue_type: normal compute queue */
777 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
778 /* alloc format: all_on_one_pipe */
779 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
780 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
781 /* num_queues: must be 1 */
782 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
783 amdgpu_ring_write(kiq_ring,
784 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
785 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
786 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
787 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
788 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
789 }
790
791 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
792 struct amdgpu_ring *ring,
793 enum amdgpu_unmap_queues_action action,
794 u64 gpu_addr, u64 seq)
795 {
796 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
797
798 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
799 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
800 PACKET3_UNMAP_QUEUES_ACTION(action) |
801 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
802 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
803 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
804 amdgpu_ring_write(kiq_ring,
805 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
806
807 if (action == PREEMPT_QUEUES_NO_UNMAP) {
808 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
809 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
810 amdgpu_ring_write(kiq_ring, seq);
811 } else {
812 amdgpu_ring_write(kiq_ring, 0);
813 amdgpu_ring_write(kiq_ring, 0);
814 amdgpu_ring_write(kiq_ring, 0);
815 }
816 }
817
818 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
819 struct amdgpu_ring *ring,
820 u64 addr,
821 u64 seq)
822 {
823 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
824
825 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
826 amdgpu_ring_write(kiq_ring,
827 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
828 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
829 PACKET3_QUERY_STATUS_COMMAND(2));
830 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
831 amdgpu_ring_write(kiq_ring,
832 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
833 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
834 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
835 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
836 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
837 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
838 }
839
840 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
841 uint16_t pasid, uint32_t flush_type,
842 bool all_hub)
843 {
844 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
845 amdgpu_ring_write(kiq_ring,
846 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
847 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
848 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
849 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
850 }
851
852 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
853 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
854 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
855 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
856 .kiq_query_status = gfx_v9_0_kiq_query_status,
857 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
858 .set_resources_size = 8,
859 .map_queues_size = 7,
860 .unmap_queues_size = 6,
861 .query_status_size = 7,
862 .invalidate_tlbs_size = 12,
863 };
864
865 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
866 {
867 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
868 }
869
870 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
871 {
872 switch (adev->asic_type) {
873 case CHIP_VEGA10:
874 soc15_program_register_sequence(adev,
875 golden_settings_gc_9_0,
876 ARRAY_SIZE(golden_settings_gc_9_0));
877 soc15_program_register_sequence(adev,
878 golden_settings_gc_9_0_vg10,
879 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
880 break;
881 case CHIP_VEGA12:
882 soc15_program_register_sequence(adev,
883 golden_settings_gc_9_2_1,
884 ARRAY_SIZE(golden_settings_gc_9_2_1));
885 soc15_program_register_sequence(adev,
886 golden_settings_gc_9_2_1_vg12,
887 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
888 break;
889 case CHIP_VEGA20:
890 soc15_program_register_sequence(adev,
891 golden_settings_gc_9_0,
892 ARRAY_SIZE(golden_settings_gc_9_0));
893 soc15_program_register_sequence(adev,
894 golden_settings_gc_9_0_vg20,
895 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
896 break;
897 case CHIP_ARCTURUS:
898 soc15_program_register_sequence(adev,
899 golden_settings_gc_9_4_1_arct,
900 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
901 break;
902 case CHIP_RAVEN:
903 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
904 ARRAY_SIZE(golden_settings_gc_9_1));
905 if (adev->rev_id >= 8)
906 soc15_program_register_sequence(adev,
907 golden_settings_gc_9_1_rv2,
908 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
909 else
910 soc15_program_register_sequence(adev,
911 golden_settings_gc_9_1_rv1,
912 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
913 break;
914 case CHIP_RENOIR:
915 soc15_program_register_sequence(adev,
916 golden_settings_gc_9_1_rn,
917 ARRAY_SIZE(golden_settings_gc_9_1_rn));
918 return; /* for renoir, don't need common goldensetting */
919 default:
920 break;
921 }
922
923 if (adev->asic_type != CHIP_ARCTURUS)
924 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
925 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
926 }
927
928 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
929 {
930 adev->gfx.scratch.num_reg = 8;
931 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
932 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
933 }
934
935 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
936 bool wc, uint32_t reg, uint32_t val)
937 {
938 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
939 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
940 WRITE_DATA_DST_SEL(0) |
941 (wc ? WR_CONFIRM : 0));
942 amdgpu_ring_write(ring, reg);
943 amdgpu_ring_write(ring, 0);
944 amdgpu_ring_write(ring, val);
945 }
946
947 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
948 int mem_space, int opt, uint32_t addr0,
949 uint32_t addr1, uint32_t ref, uint32_t mask,
950 uint32_t inv)
951 {
952 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
953 amdgpu_ring_write(ring,
954 /* memory (1) or register (0) */
955 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
956 WAIT_REG_MEM_OPERATION(opt) | /* wait */
957 WAIT_REG_MEM_FUNCTION(3) | /* equal */
958 WAIT_REG_MEM_ENGINE(eng_sel)));
959
960 if (mem_space)
961 BUG_ON(addr0 & 0x3); /* Dword align */
962 amdgpu_ring_write(ring, addr0);
963 amdgpu_ring_write(ring, addr1);
964 amdgpu_ring_write(ring, ref);
965 amdgpu_ring_write(ring, mask);
966 amdgpu_ring_write(ring, inv); /* poll interval */
967 }
968
969 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
970 {
971 struct amdgpu_device *adev = ring->adev;
972 uint32_t scratch;
973 uint32_t tmp = 0;
974 unsigned i;
975 int r;
976
977 r = amdgpu_gfx_scratch_get(adev, &scratch);
978 if (r)
979 return r;
980
981 WREG32(scratch, 0xCAFEDEAD);
982 r = amdgpu_ring_alloc(ring, 3);
983 if (r)
984 goto error_free_scratch;
985
986 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
987 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
988 amdgpu_ring_write(ring, 0xDEADBEEF);
989 amdgpu_ring_commit(ring);
990
991 for (i = 0; i < adev->usec_timeout; i++) {
992 tmp = RREG32(scratch);
993 if (tmp == 0xDEADBEEF)
994 break;
995 udelay(1);
996 }
997
998 if (i >= adev->usec_timeout)
999 r = -ETIMEDOUT;
1000
1001 error_free_scratch:
1002 amdgpu_gfx_scratch_free(adev, scratch);
1003 return r;
1004 }
1005
1006 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1007 {
1008 struct amdgpu_device *adev = ring->adev;
1009 struct amdgpu_ib ib;
1010 struct dma_fence *f = NULL;
1011
1012 unsigned index;
1013 uint64_t gpu_addr;
1014 uint32_t tmp;
1015 long r;
1016
1017 r = amdgpu_device_wb_get(adev, &index);
1018 if (r)
1019 return r;
1020
1021 gpu_addr = adev->wb.gpu_addr + (index * 4);
1022 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1023 memset(&ib, 0, sizeof(ib));
1024 r = amdgpu_ib_get(adev, NULL, 16, &ib);
1025 if (r)
1026 goto err1;
1027
1028 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1029 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1030 ib.ptr[2] = lower_32_bits(gpu_addr);
1031 ib.ptr[3] = upper_32_bits(gpu_addr);
1032 ib.ptr[4] = 0xDEADBEEF;
1033 ib.length_dw = 5;
1034
1035 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1036 if (r)
1037 goto err2;
1038
1039 r = dma_fence_wait_timeout(f, false, timeout);
1040 if (r == 0) {
1041 r = -ETIMEDOUT;
1042 goto err2;
1043 } else if (r < 0) {
1044 goto err2;
1045 }
1046
1047 tmp = adev->wb.wb[index];
1048 if (tmp == 0xDEADBEEF)
1049 r = 0;
1050 else
1051 r = -EINVAL;
1052
1053 err2:
1054 amdgpu_ib_free(adev, &ib, NULL);
1055 dma_fence_put(f);
1056 err1:
1057 amdgpu_device_wb_free(adev, index);
1058 return r;
1059 }
1060
1061
1062 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1063 {
1064 release_firmware(adev->gfx.pfp_fw);
1065 adev->gfx.pfp_fw = NULL;
1066 release_firmware(adev->gfx.me_fw);
1067 adev->gfx.me_fw = NULL;
1068 release_firmware(adev->gfx.ce_fw);
1069 adev->gfx.ce_fw = NULL;
1070 release_firmware(adev->gfx.rlc_fw);
1071 adev->gfx.rlc_fw = NULL;
1072 release_firmware(adev->gfx.mec_fw);
1073 adev->gfx.mec_fw = NULL;
1074 release_firmware(adev->gfx.mec2_fw);
1075 adev->gfx.mec2_fw = NULL;
1076
1077 kfree(adev->gfx.rlc.register_list_format);
1078 }
1079
1080 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1081 {
1082 const struct rlc_firmware_header_v2_1 *rlc_hdr;
1083
1084 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1085 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1086 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1087 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1088 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1089 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1090 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1091 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1092 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1093 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1094 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1095 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1096 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1097 adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1098 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1099 }
1100
1101 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1102 {
1103 adev->gfx.me_fw_write_wait = false;
1104 adev->gfx.mec_fw_write_wait = false;
1105
1106 if ((adev->gfx.mec_fw_version < 0x000001a5) ||
1107 (adev->gfx.mec_feature_version < 46) ||
1108 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1109 (adev->gfx.pfp_feature_version < 46))
1110 DRM_WARN_ONCE("CP firmware version too old, please update!");
1111
1112 switch (adev->asic_type) {
1113 case CHIP_VEGA10:
1114 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1115 (adev->gfx.me_feature_version >= 42) &&
1116 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1117 (adev->gfx.pfp_feature_version >= 42))
1118 adev->gfx.me_fw_write_wait = true;
1119
1120 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1121 (adev->gfx.mec_feature_version >= 42))
1122 adev->gfx.mec_fw_write_wait = true;
1123 break;
1124 case CHIP_VEGA12:
1125 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1126 (adev->gfx.me_feature_version >= 44) &&
1127 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1128 (adev->gfx.pfp_feature_version >= 44))
1129 adev->gfx.me_fw_write_wait = true;
1130
1131 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1132 (adev->gfx.mec_feature_version >= 44))
1133 adev->gfx.mec_fw_write_wait = true;
1134 break;
1135 case CHIP_VEGA20:
1136 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1137 (adev->gfx.me_feature_version >= 44) &&
1138 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1139 (adev->gfx.pfp_feature_version >= 44))
1140 adev->gfx.me_fw_write_wait = true;
1141
1142 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1143 (adev->gfx.mec_feature_version >= 44))
1144 adev->gfx.mec_fw_write_wait = true;
1145 break;
1146 case CHIP_RAVEN:
1147 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1148 (adev->gfx.me_feature_version >= 42) &&
1149 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1150 (adev->gfx.pfp_feature_version >= 42))
1151 adev->gfx.me_fw_write_wait = true;
1152
1153 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1154 (adev->gfx.mec_feature_version >= 42))
1155 adev->gfx.mec_fw_write_wait = true;
1156 break;
1157 default:
1158 break;
1159 }
1160 }
1161
1162 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1163 {
1164 switch (adev->asic_type) {
1165 case CHIP_VEGA10:
1166 case CHIP_VEGA12:
1167 case CHIP_VEGA20:
1168 break;
1169 case CHIP_RAVEN:
1170 if (!(adev->rev_id >= 0x8 ||
1171 adev->pdev->device == 0x15d8) &&
1172 (adev->pm.fw_version < 0x41e2b || /* not raven1 fresh */
1173 !adev->gfx.rlc.is_rlc_v2_1)) /* without rlc save restore ucodes */
1174 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1175
1176 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1177 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1178 AMD_PG_SUPPORT_CP |
1179 AMD_PG_SUPPORT_RLC_SMU_HS;
1180 break;
1181 case CHIP_RENOIR:
1182 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1183 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1184 AMD_PG_SUPPORT_CP |
1185 AMD_PG_SUPPORT_RLC_SMU_HS;
1186 break;
1187 default:
1188 break;
1189 }
1190 }
1191
1192 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1193 const char *chip_name)
1194 {
1195 char fw_name[30];
1196 int err;
1197 struct amdgpu_firmware_info *info = NULL;
1198 const struct common_firmware_header *header = NULL;
1199 const struct gfx_firmware_header_v1_0 *cp_hdr;
1200
1201 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1202 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1203 if (err)
1204 goto out;
1205 err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1206 if (err)
1207 goto out;
1208 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1209 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1210 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1211
1212 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1213 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1214 if (err)
1215 goto out;
1216 err = amdgpu_ucode_validate(adev->gfx.me_fw);
1217 if (err)
1218 goto out;
1219 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1220 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1221 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1222
1223 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1224 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1225 if (err)
1226 goto out;
1227 err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1228 if (err)
1229 goto out;
1230 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1231 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1232 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1233
1234 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1235 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1236 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1237 info->fw = adev->gfx.pfp_fw;
1238 header = (const struct common_firmware_header *)info->fw->data;
1239 adev->firmware.fw_size +=
1240 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1241
1242 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1243 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1244 info->fw = adev->gfx.me_fw;
1245 header = (const struct common_firmware_header *)info->fw->data;
1246 adev->firmware.fw_size +=
1247 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1248
1249 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1250 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1251 info->fw = adev->gfx.ce_fw;
1252 header = (const struct common_firmware_header *)info->fw->data;
1253 adev->firmware.fw_size +=
1254 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1255 }
1256
1257 out:
1258 if (err) {
1259 dev_err(adev->dev,
1260 "gfx9: Failed to load firmware \"%s\"\n",
1261 fw_name);
1262 release_firmware(adev->gfx.pfp_fw);
1263 adev->gfx.pfp_fw = NULL;
1264 release_firmware(adev->gfx.me_fw);
1265 adev->gfx.me_fw = NULL;
1266 release_firmware(adev->gfx.ce_fw);
1267 adev->gfx.ce_fw = NULL;
1268 }
1269 return err;
1270 }
1271
1272 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1273 const char *chip_name)
1274 {
1275 char fw_name[30];
1276 int err;
1277 struct amdgpu_firmware_info *info = NULL;
1278 const struct common_firmware_header *header = NULL;
1279 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1280 unsigned int *tmp = NULL;
1281 unsigned int i = 0;
1282 uint16_t version_major;
1283 uint16_t version_minor;
1284 uint32_t smu_version;
1285
1286 /*
1287 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1288 * instead of picasso_rlc.bin.
1289 * Judgment method:
1290 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1291 * or revision >= 0xD8 && revision <= 0xDF
1292 * otherwise is PCO FP5
1293 */
1294 if (!strcmp(chip_name, "picasso") &&
1295 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1296 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1297 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1298 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1299 (smu_version >= 0x41e2b))
1300 /**
1301 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1302 */
1303 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1304 else
1305 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1306 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1307 if (err)
1308 goto out;
1309 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1310 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1311
1312 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1313 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1314 if (version_major == 2 && version_minor == 1)
1315 adev->gfx.rlc.is_rlc_v2_1 = true;
1316
1317 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1318 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1319 adev->gfx.rlc.save_and_restore_offset =
1320 le32_to_cpu(rlc_hdr->save_and_restore_offset);
1321 adev->gfx.rlc.clear_state_descriptor_offset =
1322 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1323 adev->gfx.rlc.avail_scratch_ram_locations =
1324 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1325 adev->gfx.rlc.reg_restore_list_size =
1326 le32_to_cpu(rlc_hdr->reg_restore_list_size);
1327 adev->gfx.rlc.reg_list_format_start =
1328 le32_to_cpu(rlc_hdr->reg_list_format_start);
1329 adev->gfx.rlc.reg_list_format_separate_start =
1330 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1331 adev->gfx.rlc.starting_offsets_start =
1332 le32_to_cpu(rlc_hdr->starting_offsets_start);
1333 adev->gfx.rlc.reg_list_format_size_bytes =
1334 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1335 adev->gfx.rlc.reg_list_size_bytes =
1336 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1337 adev->gfx.rlc.register_list_format =
1338 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1339 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1340 if (!adev->gfx.rlc.register_list_format) {
1341 err = -ENOMEM;
1342 goto out;
1343 }
1344
1345 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1346 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1347 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1348 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1349
1350 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1351
1352 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1353 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1354 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1355 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1356
1357 if (adev->gfx.rlc.is_rlc_v2_1)
1358 gfx_v9_0_init_rlc_ext_microcode(adev);
1359
1360 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1361 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1362 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1363 info->fw = adev->gfx.rlc_fw;
1364 header = (const struct common_firmware_header *)info->fw->data;
1365 adev->firmware.fw_size +=
1366 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1367
1368 if (adev->gfx.rlc.is_rlc_v2_1 &&
1369 adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1370 adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1371 adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1372 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1373 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1374 info->fw = adev->gfx.rlc_fw;
1375 adev->firmware.fw_size +=
1376 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1377
1378 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1379 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1380 info->fw = adev->gfx.rlc_fw;
1381 adev->firmware.fw_size +=
1382 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1383
1384 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1385 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1386 info->fw = adev->gfx.rlc_fw;
1387 adev->firmware.fw_size +=
1388 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1389 }
1390 }
1391
1392 out:
1393 if (err) {
1394 dev_err(adev->dev,
1395 "gfx9: Failed to load firmware \"%s\"\n",
1396 fw_name);
1397 release_firmware(adev->gfx.rlc_fw);
1398 adev->gfx.rlc_fw = NULL;
1399 }
1400 return err;
1401 }
1402
1403 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1404 const char *chip_name)
1405 {
1406 char fw_name[30];
1407 int err;
1408 struct amdgpu_firmware_info *info = NULL;
1409 const struct common_firmware_header *header = NULL;
1410 const struct gfx_firmware_header_v1_0 *cp_hdr;
1411
1412 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1413 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1414 if (err)
1415 goto out;
1416 err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1417 if (err)
1418 goto out;
1419 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1420 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1421 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1422
1423
1424 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1425 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1426 if (!err) {
1427 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1428 if (err)
1429 goto out;
1430 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1431 adev->gfx.mec2_fw->data;
1432 adev->gfx.mec2_fw_version =
1433 le32_to_cpu(cp_hdr->header.ucode_version);
1434 adev->gfx.mec2_feature_version =
1435 le32_to_cpu(cp_hdr->ucode_feature_version);
1436 } else {
1437 err = 0;
1438 adev->gfx.mec2_fw = NULL;
1439 }
1440
1441 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1442 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1443 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1444 info->fw = adev->gfx.mec_fw;
1445 header = (const struct common_firmware_header *)info->fw->data;
1446 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1447 adev->firmware.fw_size +=
1448 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1449
1450 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1451 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1452 info->fw = adev->gfx.mec_fw;
1453 adev->firmware.fw_size +=
1454 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1455
1456 if (adev->gfx.mec2_fw) {
1457 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1458 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1459 info->fw = adev->gfx.mec2_fw;
1460 header = (const struct common_firmware_header *)info->fw->data;
1461 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1462 adev->firmware.fw_size +=
1463 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1464
1465 /* TODO: Determine if MEC2 JT FW loading can be removed
1466 for all GFX V9 asic and above */
1467 if (adev->asic_type != CHIP_ARCTURUS &&
1468 adev->asic_type != CHIP_RENOIR) {
1469 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1470 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1471 info->fw = adev->gfx.mec2_fw;
1472 adev->firmware.fw_size +=
1473 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1474 PAGE_SIZE);
1475 }
1476 }
1477 }
1478
1479 out:
1480 gfx_v9_0_check_if_need_gfxoff(adev);
1481 gfx_v9_0_check_fw_write_wait(adev);
1482 if (err) {
1483 dev_err(adev->dev,
1484 "gfx9: Failed to load firmware \"%s\"\n",
1485 fw_name);
1486 release_firmware(adev->gfx.mec_fw);
1487 adev->gfx.mec_fw = NULL;
1488 release_firmware(adev->gfx.mec2_fw);
1489 adev->gfx.mec2_fw = NULL;
1490 }
1491 return err;
1492 }
1493
1494 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1495 {
1496 const char *chip_name;
1497 int r;
1498
1499 DRM_DEBUG("\n");
1500
1501 switch (adev->asic_type) {
1502 case CHIP_VEGA10:
1503 chip_name = "vega10";
1504 break;
1505 case CHIP_VEGA12:
1506 chip_name = "vega12";
1507 break;
1508 case CHIP_VEGA20:
1509 chip_name = "vega20";
1510 break;
1511 case CHIP_RAVEN:
1512 if (adev->rev_id >= 8)
1513 chip_name = "raven2";
1514 else if (adev->pdev->device == 0x15d8)
1515 chip_name = "picasso";
1516 else
1517 chip_name = "raven";
1518 break;
1519 case CHIP_ARCTURUS:
1520 chip_name = "arcturus";
1521 break;
1522 case CHIP_RENOIR:
1523 chip_name = "renoir";
1524 break;
1525 default:
1526 BUG();
1527 }
1528
1529 /* No CPG in Arcturus */
1530 if (adev->asic_type != CHIP_ARCTURUS) {
1531 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1532 if (r)
1533 return r;
1534 }
1535
1536 r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1537 if (r)
1538 return r;
1539
1540 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1541 if (r)
1542 return r;
1543
1544 return r;
1545 }
1546
1547 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1548 {
1549 u32 count = 0;
1550 const struct cs_section_def *sect = NULL;
1551 const struct cs_extent_def *ext = NULL;
1552
1553 /* begin clear state */
1554 count += 2;
1555 /* context control state */
1556 count += 3;
1557
1558 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1559 for (ext = sect->section; ext->extent != NULL; ++ext) {
1560 if (sect->id == SECT_CONTEXT)
1561 count += 2 + ext->reg_count;
1562 else
1563 return 0;
1564 }
1565 }
1566
1567 /* end clear state */
1568 count += 2;
1569 /* clear state */
1570 count += 2;
1571
1572 return count;
1573 }
1574
1575 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1576 volatile u32 *buffer)
1577 {
1578 u32 count = 0, i;
1579 const struct cs_section_def *sect = NULL;
1580 const struct cs_extent_def *ext = NULL;
1581
1582 if (adev->gfx.rlc.cs_data == NULL)
1583 return;
1584 if (buffer == NULL)
1585 return;
1586
1587 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1588 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1589
1590 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1591 buffer[count++] = cpu_to_le32(0x80000000);
1592 buffer[count++] = cpu_to_le32(0x80000000);
1593
1594 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1595 for (ext = sect->section; ext->extent != NULL; ++ext) {
1596 if (sect->id == SECT_CONTEXT) {
1597 buffer[count++] =
1598 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1599 buffer[count++] = cpu_to_le32(ext->reg_index -
1600 PACKET3_SET_CONTEXT_REG_START);
1601 for (i = 0; i < ext->reg_count; i++)
1602 buffer[count++] = cpu_to_le32(ext->extent[i]);
1603 } else {
1604 return;
1605 }
1606 }
1607 }
1608
1609 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1610 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1611
1612 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1613 buffer[count++] = cpu_to_le32(0);
1614 }
1615
1616 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1617 {
1618 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1619 uint32_t pg_always_on_cu_num = 2;
1620 uint32_t always_on_cu_num;
1621 uint32_t i, j, k;
1622 uint32_t mask, cu_bitmap, counter;
1623
1624 if (adev->flags & AMD_IS_APU)
1625 always_on_cu_num = 4;
1626 else if (adev->asic_type == CHIP_VEGA12)
1627 always_on_cu_num = 8;
1628 else
1629 always_on_cu_num = 12;
1630
1631 mutex_lock(&adev->grbm_idx_mutex);
1632 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1633 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1634 mask = 1;
1635 cu_bitmap = 0;
1636 counter = 0;
1637 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1638
1639 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1640 if (cu_info->bitmap[i][j] & mask) {
1641 if (counter == pg_always_on_cu_num)
1642 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1643 if (counter < always_on_cu_num)
1644 cu_bitmap |= mask;
1645 else
1646 break;
1647 counter++;
1648 }
1649 mask <<= 1;
1650 }
1651
1652 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1653 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1654 }
1655 }
1656 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1657 mutex_unlock(&adev->grbm_idx_mutex);
1658 }
1659
1660 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1661 {
1662 uint32_t data;
1663
1664 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1665 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1666 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1667 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1668 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1669
1670 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1671 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1672
1673 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1674 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1675
1676 mutex_lock(&adev->grbm_idx_mutex);
1677 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1678 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1679 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1680
1681 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1682 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1683 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1684 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1685 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1686
1687 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1688 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1689 data &= 0x0000FFFF;
1690 data |= 0x00C00000;
1691 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1692
1693 /*
1694 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1695 * programmed in gfx_v9_0_init_always_on_cu_mask()
1696 */
1697
1698 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1699 * but used for RLC_LB_CNTL configuration */
1700 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1701 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1702 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1703 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1704 mutex_unlock(&adev->grbm_idx_mutex);
1705
1706 gfx_v9_0_init_always_on_cu_mask(adev);
1707 }
1708
1709 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1710 {
1711 uint32_t data;
1712
1713 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1714 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1715 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1716 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1717 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1718
1719 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1720 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1721
1722 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1723 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1724
1725 mutex_lock(&adev->grbm_idx_mutex);
1726 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1727 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1728 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1729
1730 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1731 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1732 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1733 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1734 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1735
1736 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1737 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1738 data &= 0x0000FFFF;
1739 data |= 0x00C00000;
1740 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1741
1742 /*
1743 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1744 * programmed in gfx_v9_0_init_always_on_cu_mask()
1745 */
1746
1747 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1748 * but used for RLC_LB_CNTL configuration */
1749 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1750 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1751 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1752 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1753 mutex_unlock(&adev->grbm_idx_mutex);
1754
1755 gfx_v9_0_init_always_on_cu_mask(adev);
1756 }
1757
1758 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1759 {
1760 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1761 }
1762
1763 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1764 {
1765 return 5;
1766 }
1767
1768 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1769 {
1770 const struct cs_section_def *cs_data;
1771 int r;
1772
1773 adev->gfx.rlc.cs_data = gfx9_cs_data;
1774
1775 cs_data = adev->gfx.rlc.cs_data;
1776
1777 if (cs_data) {
1778 /* init clear state block */
1779 r = amdgpu_gfx_rlc_init_csb(adev);
1780 if (r)
1781 return r;
1782 }
1783
1784 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) {
1785 /* TODO: double check the cp_table_size for RV */
1786 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1787 r = amdgpu_gfx_rlc_init_cpt(adev);
1788 if (r)
1789 return r;
1790 }
1791
1792 switch (adev->asic_type) {
1793 case CHIP_RAVEN:
1794 gfx_v9_0_init_lbpw(adev);
1795 break;
1796 case CHIP_VEGA20:
1797 gfx_v9_4_init_lbpw(adev);
1798 break;
1799 default:
1800 break;
1801 }
1802
1803 return 0;
1804 }
1805
1806 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1807 {
1808 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1809 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1810 }
1811
1812 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1813 {
1814 int r;
1815 u32 *hpd;
1816 const __le32 *fw_data;
1817 unsigned fw_size;
1818 u32 *fw;
1819 size_t mec_hpd_size;
1820
1821 const struct gfx_firmware_header_v1_0 *mec_hdr;
1822
1823 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1824
1825 /* take ownership of the relevant compute queues */
1826 amdgpu_gfx_compute_queue_acquire(adev);
1827 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1828
1829 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1830 AMDGPU_GEM_DOMAIN_VRAM,
1831 &adev->gfx.mec.hpd_eop_obj,
1832 &adev->gfx.mec.hpd_eop_gpu_addr,
1833 (void **)&hpd);
1834 if (r) {
1835 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1836 gfx_v9_0_mec_fini(adev);
1837 return r;
1838 }
1839
1840 memset(hpd, 0, adev->gfx.mec.hpd_eop_obj->tbo.mem.size);
1841
1842 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1843 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1844
1845 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1846
1847 fw_data = (const __le32 *)
1848 (adev->gfx.mec_fw->data +
1849 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1850 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
1851
1852 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1853 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1854 &adev->gfx.mec.mec_fw_obj,
1855 &adev->gfx.mec.mec_fw_gpu_addr,
1856 (void **)&fw);
1857 if (r) {
1858 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1859 gfx_v9_0_mec_fini(adev);
1860 return r;
1861 }
1862
1863 memcpy(fw, fw_data, fw_size);
1864
1865 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1866 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1867
1868 return 0;
1869 }
1870
1871 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1872 {
1873 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
1874 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1875 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1876 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1877 (SQ_IND_INDEX__FORCE_READ_MASK));
1878 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1879 }
1880
1881 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1882 uint32_t wave, uint32_t thread,
1883 uint32_t regno, uint32_t num, uint32_t *out)
1884 {
1885 WREG32_SOC15(GC, 0, mmSQ_IND_INDEX,
1886 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1887 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1888 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
1889 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
1890 (SQ_IND_INDEX__FORCE_READ_MASK) |
1891 (SQ_IND_INDEX__AUTO_INCR_MASK));
1892 while (num--)
1893 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1894 }
1895
1896 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
1897 {
1898 /* type 1 wave data */
1899 dst[(*no_fields)++] = 1;
1900 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
1901 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
1902 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
1903 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
1904 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
1905 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
1906 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
1907 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
1908 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
1909 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
1910 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
1911 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
1912 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
1913 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
1914 }
1915
1916 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
1917 uint32_t wave, uint32_t start,
1918 uint32_t size, uint32_t *dst)
1919 {
1920 wave_read_regs(
1921 adev, simd, wave, 0,
1922 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
1923 }
1924
1925 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
1926 uint32_t wave, uint32_t thread,
1927 uint32_t start, uint32_t size,
1928 uint32_t *dst)
1929 {
1930 wave_read_regs(
1931 adev, simd, wave, thread,
1932 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
1933 }
1934
1935 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
1936 u32 me, u32 pipe, u32 q, u32 vm)
1937 {
1938 soc15_grbm_select(adev, me, pipe, q, vm);
1939 }
1940
1941 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
1942 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
1943 .select_se_sh = &gfx_v9_0_select_se_sh,
1944 .read_wave_data = &gfx_v9_0_read_wave_data,
1945 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
1946 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
1947 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
1948 .ras_error_inject = &gfx_v9_0_ras_error_inject,
1949 .query_ras_error_count = &gfx_v9_0_query_ras_error_count
1950 };
1951
1952 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
1953 {
1954 u32 gb_addr_config;
1955 int err;
1956
1957 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
1958
1959 switch (adev->asic_type) {
1960 case CHIP_VEGA10:
1961 adev->gfx.config.max_hw_contexts = 8;
1962 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1963 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1964 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1965 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1966 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
1967 break;
1968 case CHIP_VEGA12:
1969 adev->gfx.config.max_hw_contexts = 8;
1970 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1971 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1972 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1973 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1974 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
1975 DRM_INFO("fix gfx.config for vega12\n");
1976 break;
1977 case CHIP_VEGA20:
1978 adev->gfx.config.max_hw_contexts = 8;
1979 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1980 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1981 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1982 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1983 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
1984 gb_addr_config &= ~0xf3e777ff;
1985 gb_addr_config |= 0x22014042;
1986 /* check vbios table if gpu info is not available */
1987 err = amdgpu_atomfirmware_get_gfx_info(adev);
1988 if (err)
1989 return err;
1990 break;
1991 case CHIP_RAVEN:
1992 adev->gfx.config.max_hw_contexts = 8;
1993 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1994 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1995 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1996 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
1997 if (adev->rev_id >= 8)
1998 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
1999 else
2000 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2001 break;
2002 case CHIP_ARCTURUS:
2003 adev->gfx.config.max_hw_contexts = 8;
2004 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2005 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2006 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2007 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2008 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2009 gb_addr_config &= ~0xf3e777ff;
2010 gb_addr_config |= 0x22014042;
2011 break;
2012 case CHIP_RENOIR:
2013 adev->gfx.config.max_hw_contexts = 8;
2014 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2015 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2016 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2017 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2018 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2019 gb_addr_config &= ~0xf3e777ff;
2020 gb_addr_config |= 0x22010042;
2021 break;
2022 default:
2023 BUG();
2024 break;
2025 }
2026
2027 adev->gfx.config.gb_addr_config = gb_addr_config;
2028
2029 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2030 REG_GET_FIELD(
2031 adev->gfx.config.gb_addr_config,
2032 GB_ADDR_CONFIG,
2033 NUM_PIPES);
2034
2035 adev->gfx.config.max_tile_pipes =
2036 adev->gfx.config.gb_addr_config_fields.num_pipes;
2037
2038 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2039 REG_GET_FIELD(
2040 adev->gfx.config.gb_addr_config,
2041 GB_ADDR_CONFIG,
2042 NUM_BANKS);
2043 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2044 REG_GET_FIELD(
2045 adev->gfx.config.gb_addr_config,
2046 GB_ADDR_CONFIG,
2047 MAX_COMPRESSED_FRAGS);
2048 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2049 REG_GET_FIELD(
2050 adev->gfx.config.gb_addr_config,
2051 GB_ADDR_CONFIG,
2052 NUM_RB_PER_SE);
2053 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2054 REG_GET_FIELD(
2055 adev->gfx.config.gb_addr_config,
2056 GB_ADDR_CONFIG,
2057 NUM_SHADER_ENGINES);
2058 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2059 REG_GET_FIELD(
2060 adev->gfx.config.gb_addr_config,
2061 GB_ADDR_CONFIG,
2062 PIPE_INTERLEAVE_SIZE));
2063
2064 return 0;
2065 }
2066
2067 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2068 int mec, int pipe, int queue)
2069 {
2070 int r;
2071 unsigned irq_type;
2072 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2073
2074 ring = &adev->gfx.compute_ring[ring_id];
2075
2076 /* mec0 is me1 */
2077 ring->me = mec + 1;
2078 ring->pipe = pipe;
2079 ring->queue = queue;
2080
2081 ring->ring_obj = NULL;
2082 ring->use_doorbell = true;
2083 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2084 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2085 + (ring_id * GFX9_MEC_HPD_SIZE);
2086 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2087
2088 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2089 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2090 + ring->pipe;
2091
2092 /* type-2 packets are deprecated on MEC, use type-3 instead */
2093 r = amdgpu_ring_init(adev, ring, 1024,
2094 &adev->gfx.eop_irq, irq_type);
2095 if (r)
2096 return r;
2097
2098
2099 return 0;
2100 }
2101
2102 static int gfx_v9_0_sw_init(void *handle)
2103 {
2104 int i, j, k, r, ring_id;
2105 struct amdgpu_ring *ring;
2106 struct amdgpu_kiq *kiq;
2107 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2108
2109 switch (adev->asic_type) {
2110 case CHIP_VEGA10:
2111 case CHIP_VEGA12:
2112 case CHIP_VEGA20:
2113 case CHIP_RAVEN:
2114 case CHIP_ARCTURUS:
2115 case CHIP_RENOIR:
2116 adev->gfx.mec.num_mec = 2;
2117 break;
2118 default:
2119 adev->gfx.mec.num_mec = 1;
2120 break;
2121 }
2122
2123 adev->gfx.mec.num_pipe_per_mec = 4;
2124 adev->gfx.mec.num_queue_per_pipe = 8;
2125
2126 /* EOP Event */
2127 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2128 if (r)
2129 return r;
2130
2131 /* Privileged reg */
2132 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2133 &adev->gfx.priv_reg_irq);
2134 if (r)
2135 return r;
2136
2137 /* Privileged inst */
2138 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2139 &adev->gfx.priv_inst_irq);
2140 if (r)
2141 return r;
2142
2143 /* ECC error */
2144 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2145 &adev->gfx.cp_ecc_error_irq);
2146 if (r)
2147 return r;
2148
2149 /* FUE error */
2150 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2151 &adev->gfx.cp_ecc_error_irq);
2152 if (r)
2153 return r;
2154
2155 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2156
2157 gfx_v9_0_scratch_init(adev);
2158
2159 r = gfx_v9_0_init_microcode(adev);
2160 if (r) {
2161 DRM_ERROR("Failed to load gfx firmware!\n");
2162 return r;
2163 }
2164
2165 r = adev->gfx.rlc.funcs->init(adev);
2166 if (r) {
2167 DRM_ERROR("Failed to init rlc BOs!\n");
2168 return r;
2169 }
2170
2171 r = gfx_v9_0_mec_init(adev);
2172 if (r) {
2173 DRM_ERROR("Failed to init MEC BOs!\n");
2174 return r;
2175 }
2176
2177 /* set up the gfx ring */
2178 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2179 ring = &adev->gfx.gfx_ring[i];
2180 ring->ring_obj = NULL;
2181 if (!i)
2182 sprintf(ring->name, "gfx");
2183 else
2184 sprintf(ring->name, "gfx_%d", i);
2185 ring->use_doorbell = true;
2186 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2187 r = amdgpu_ring_init(adev, ring, 1024,
2188 &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
2189 if (r)
2190 return r;
2191 }
2192
2193 /* set up the compute queues - allocate horizontally across pipes */
2194 ring_id = 0;
2195 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2196 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2197 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2198 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2199 continue;
2200
2201 r = gfx_v9_0_compute_ring_init(adev,
2202 ring_id,
2203 i, k, j);
2204 if (r)
2205 return r;
2206
2207 ring_id++;
2208 }
2209 }
2210 }
2211
2212 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2213 if (r) {
2214 DRM_ERROR("Failed to init KIQ BOs!\n");
2215 return r;
2216 }
2217
2218 kiq = &adev->gfx.kiq;
2219 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2220 if (r)
2221 return r;
2222
2223 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2224 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2225 if (r)
2226 return r;
2227
2228 adev->gfx.ce_ram_size = 0x8000;
2229
2230 r = gfx_v9_0_gpu_early_init(adev);
2231 if (r)
2232 return r;
2233
2234 return 0;
2235 }
2236
2237
2238 static int gfx_v9_0_sw_fini(void *handle)
2239 {
2240 int i;
2241 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2242
2243 amdgpu_gfx_ras_fini(adev);
2244
2245 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2246 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2247 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2248 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2249
2250 amdgpu_gfx_mqd_sw_fini(adev);
2251 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2252 amdgpu_gfx_kiq_fini(adev);
2253
2254 gfx_v9_0_mec_fini(adev);
2255 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2256 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) {
2257 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2258 &adev->gfx.rlc.cp_table_gpu_addr,
2259 (void **)&adev->gfx.rlc.cp_table_ptr);
2260 }
2261 gfx_v9_0_free_microcode(adev);
2262
2263 return 0;
2264 }
2265
2266
2267 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2268 {
2269 /* TODO */
2270 }
2271
2272 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance)
2273 {
2274 u32 data;
2275
2276 if (instance == 0xffffffff)
2277 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2278 else
2279 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2280
2281 if (se_num == 0xffffffff)
2282 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2283 else
2284 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2285
2286 if (sh_num == 0xffffffff)
2287 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2288 else
2289 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2290
2291 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2292 }
2293
2294 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2295 {
2296 u32 data, mask;
2297
2298 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2299 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2300
2301 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2302 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2303
2304 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2305 adev->gfx.config.max_sh_per_se);
2306
2307 return (~data) & mask;
2308 }
2309
2310 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2311 {
2312 int i, j;
2313 u32 data;
2314 u32 active_rbs = 0;
2315 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2316 adev->gfx.config.max_sh_per_se;
2317
2318 mutex_lock(&adev->grbm_idx_mutex);
2319 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2320 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2321 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2322 data = gfx_v9_0_get_rb_active_bitmap(adev);
2323 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2324 rb_bitmap_width_per_sh);
2325 }
2326 }
2327 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2328 mutex_unlock(&adev->grbm_idx_mutex);
2329
2330 adev->gfx.config.backend_enable_mask = active_rbs;
2331 adev->gfx.config.num_rbs = hweight32(active_rbs);
2332 }
2333
2334 #define DEFAULT_SH_MEM_BASES (0x6000)
2335 #define FIRST_COMPUTE_VMID (8)
2336 #define LAST_COMPUTE_VMID (16)
2337 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2338 {
2339 int i;
2340 uint32_t sh_mem_config;
2341 uint32_t sh_mem_bases;
2342
2343 /*
2344 * Configure apertures:
2345 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2346 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2347 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2348 */
2349 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2350
2351 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2352 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2353 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2354
2355 mutex_lock(&adev->srbm_mutex);
2356 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
2357 soc15_grbm_select(adev, 0, 0, 0, i);
2358 /* CP and shaders */
2359 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2360 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2361 }
2362 soc15_grbm_select(adev, 0, 0, 0, 0);
2363 mutex_unlock(&adev->srbm_mutex);
2364
2365 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2366 acccess. These should be enabled by FW for target VMIDs. */
2367 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
2368 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2369 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2370 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2371 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2372 }
2373 }
2374
2375 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2376 {
2377 int vmid;
2378
2379 /*
2380 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2381 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2382 * the driver can enable them for graphics. VMID0 should maintain
2383 * access so that HWS firmware can save/restore entries.
2384 */
2385 for (vmid = 1; vmid < 16; vmid++) {
2386 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2387 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2388 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2389 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2390 }
2391 }
2392
2393 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2394 {
2395 u32 tmp;
2396 int i;
2397
2398 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2399
2400 gfx_v9_0_tiling_mode_table_init(adev);
2401
2402 gfx_v9_0_setup_rb(adev);
2403 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2404 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2405
2406 /* XXX SH_MEM regs */
2407 /* where to put LDS, scratch, GPUVM in FSA64 space */
2408 mutex_lock(&adev->srbm_mutex);
2409 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2410 soc15_grbm_select(adev, 0, 0, 0, i);
2411 /* CP and shaders */
2412 if (i == 0) {
2413 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2414 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2415 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2416 !!amdgpu_noretry);
2417 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2418 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2419 } else {
2420 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2421 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2422 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2423 !!amdgpu_noretry);
2424 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2425 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2426 (adev->gmc.private_aperture_start >> 48));
2427 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2428 (adev->gmc.shared_aperture_start >> 48));
2429 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2430 }
2431 }
2432 soc15_grbm_select(adev, 0, 0, 0, 0);
2433
2434 mutex_unlock(&adev->srbm_mutex);
2435
2436 gfx_v9_0_init_compute_vmid(adev);
2437 gfx_v9_0_init_gds_vmid(adev);
2438 }
2439
2440 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2441 {
2442 u32 i, j, k;
2443 u32 mask;
2444
2445 mutex_lock(&adev->grbm_idx_mutex);
2446 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2447 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2448 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2449 for (k = 0; k < adev->usec_timeout; k++) {
2450 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2451 break;
2452 udelay(1);
2453 }
2454 if (k == adev->usec_timeout) {
2455 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2456 0xffffffff, 0xffffffff);
2457 mutex_unlock(&adev->grbm_idx_mutex);
2458 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2459 i, j);
2460 return;
2461 }
2462 }
2463 }
2464 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2465 mutex_unlock(&adev->grbm_idx_mutex);
2466
2467 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2468 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2469 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2470 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2471 for (k = 0; k < adev->usec_timeout; k++) {
2472 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2473 break;
2474 udelay(1);
2475 }
2476 }
2477
2478 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2479 bool enable)
2480 {
2481 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2482
2483 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2484 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2485 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2486 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2487
2488 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2489 }
2490
2491 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2492 {
2493 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2494 /* csib */
2495 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2496 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2497 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2498 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2499 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2500 adev->gfx.rlc.clear_state_size);
2501 }
2502
2503 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2504 int indirect_offset,
2505 int list_size,
2506 int *unique_indirect_regs,
2507 int unique_indirect_reg_count,
2508 int *indirect_start_offsets,
2509 int *indirect_start_offsets_count,
2510 int max_start_offsets_count)
2511 {
2512 int idx;
2513
2514 for (; indirect_offset < list_size; indirect_offset++) {
2515 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2516 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2517 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2518
2519 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2520 indirect_offset += 2;
2521
2522 /* look for the matching indice */
2523 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2524 if (unique_indirect_regs[idx] ==
2525 register_list_format[indirect_offset] ||
2526 !unique_indirect_regs[idx])
2527 break;
2528 }
2529
2530 BUG_ON(idx >= unique_indirect_reg_count);
2531
2532 if (!unique_indirect_regs[idx])
2533 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2534
2535 indirect_offset++;
2536 }
2537 }
2538 }
2539
2540 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2541 {
2542 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2543 int unique_indirect_reg_count = 0;
2544
2545 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2546 int indirect_start_offsets_count = 0;
2547
2548 int list_size = 0;
2549 int i = 0, j = 0;
2550 u32 tmp = 0;
2551
2552 u32 *register_list_format =
2553 kmemdup(adev->gfx.rlc.register_list_format,
2554 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2555 if (!register_list_format)
2556 return -ENOMEM;
2557
2558 /* setup unique_indirect_regs array and indirect_start_offsets array */
2559 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2560 gfx_v9_1_parse_ind_reg_list(register_list_format,
2561 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2562 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2563 unique_indirect_regs,
2564 unique_indirect_reg_count,
2565 indirect_start_offsets,
2566 &indirect_start_offsets_count,
2567 ARRAY_SIZE(indirect_start_offsets));
2568
2569 /* enable auto inc in case it is disabled */
2570 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2571 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2572 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2573
2574 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2575 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2576 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2577 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2578 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2579 adev->gfx.rlc.register_restore[i]);
2580
2581 /* load indirect register */
2582 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2583 adev->gfx.rlc.reg_list_format_start);
2584
2585 /* direct register portion */
2586 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2587 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2588 register_list_format[i]);
2589
2590 /* indirect register portion */
2591 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2592 if (register_list_format[i] == 0xFFFFFFFF) {
2593 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2594 continue;
2595 }
2596
2597 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2598 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2599
2600 for (j = 0; j < unique_indirect_reg_count; j++) {
2601 if (register_list_format[i] == unique_indirect_regs[j]) {
2602 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2603 break;
2604 }
2605 }
2606
2607 BUG_ON(j >= unique_indirect_reg_count);
2608
2609 i++;
2610 }
2611
2612 /* set save/restore list size */
2613 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2614 list_size = list_size >> 1;
2615 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2616 adev->gfx.rlc.reg_restore_list_size);
2617 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2618
2619 /* write the starting offsets to RLC scratch ram */
2620 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2621 adev->gfx.rlc.starting_offsets_start);
2622 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2623 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2624 indirect_start_offsets[i]);
2625
2626 /* load unique indirect regs*/
2627 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2628 if (unique_indirect_regs[i] != 0) {
2629 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2630 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2631 unique_indirect_regs[i] & 0x3FFFF);
2632
2633 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2634 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2635 unique_indirect_regs[i] >> 20);
2636 }
2637 }
2638
2639 kfree(register_list_format);
2640 return 0;
2641 }
2642
2643 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2644 {
2645 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2646 }
2647
2648 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2649 bool enable)
2650 {
2651 uint32_t data = 0;
2652 uint32_t default_data = 0;
2653
2654 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2655 if (enable == true) {
2656 /* enable GFXIP control over CGPG */
2657 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2658 if(default_data != data)
2659 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2660
2661 /* update status */
2662 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2663 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2664 if(default_data != data)
2665 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2666 } else {
2667 /* restore GFXIP control over GCPG */
2668 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2669 if(default_data != data)
2670 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2671 }
2672 }
2673
2674 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2675 {
2676 uint32_t data = 0;
2677
2678 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2679 AMD_PG_SUPPORT_GFX_SMG |
2680 AMD_PG_SUPPORT_GFX_DMG)) {
2681 /* init IDLE_POLL_COUNT = 60 */
2682 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2683 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2684 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2685 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2686
2687 /* init RLC PG Delay */
2688 data = 0;
2689 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2690 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2691 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2692 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2693 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2694
2695 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2696 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2697 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2698 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2699
2700 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2701 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2702 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2703 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2704
2705 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2706 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2707
2708 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2709 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2710 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2711
2712 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2713 }
2714 }
2715
2716 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2717 bool enable)
2718 {
2719 uint32_t data = 0;
2720 uint32_t default_data = 0;
2721
2722 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2723 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2724 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2725 enable ? 1 : 0);
2726 if (default_data != data)
2727 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2728 }
2729
2730 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2731 bool enable)
2732 {
2733 uint32_t data = 0;
2734 uint32_t default_data = 0;
2735
2736 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2737 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2738 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2739 enable ? 1 : 0);
2740 if(default_data != data)
2741 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2742 }
2743
2744 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2745 bool enable)
2746 {
2747 uint32_t data = 0;
2748 uint32_t default_data = 0;
2749
2750 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2751 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2752 CP_PG_DISABLE,
2753 enable ? 0 : 1);
2754 if(default_data != data)
2755 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2756 }
2757
2758 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2759 bool enable)
2760 {
2761 uint32_t data, default_data;
2762
2763 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2764 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2765 GFX_POWER_GATING_ENABLE,
2766 enable ? 1 : 0);
2767 if(default_data != data)
2768 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2769 }
2770
2771 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2772 bool enable)
2773 {
2774 uint32_t data, default_data;
2775
2776 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2777 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2778 GFX_PIPELINE_PG_ENABLE,
2779 enable ? 1 : 0);
2780 if(default_data != data)
2781 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2782
2783 if (!enable)
2784 /* read any GFX register to wake up GFX */
2785 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2786 }
2787
2788 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2789 bool enable)
2790 {
2791 uint32_t data, default_data;
2792
2793 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2794 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2795 STATIC_PER_CU_PG_ENABLE,
2796 enable ? 1 : 0);
2797 if(default_data != data)
2798 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2799 }
2800
2801 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2802 bool enable)
2803 {
2804 uint32_t data, default_data;
2805
2806 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2807 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2808 DYN_PER_CU_PG_ENABLE,
2809 enable ? 1 : 0);
2810 if(default_data != data)
2811 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2812 }
2813
2814 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2815 {
2816 gfx_v9_0_init_csb(adev);
2817
2818 /*
2819 * Rlc save restore list is workable since v2_1.
2820 * And it's needed by gfxoff feature.
2821 */
2822 if (adev->gfx.rlc.is_rlc_v2_1) {
2823 if (adev->asic_type == CHIP_VEGA12 ||
2824 (adev->asic_type == CHIP_RAVEN &&
2825 adev->rev_id >= 8))
2826 gfx_v9_1_init_rlc_save_restore_list(adev);
2827 gfx_v9_0_enable_save_restore_machine(adev);
2828 }
2829
2830 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2831 AMD_PG_SUPPORT_GFX_SMG |
2832 AMD_PG_SUPPORT_GFX_DMG |
2833 AMD_PG_SUPPORT_CP |
2834 AMD_PG_SUPPORT_GDS |
2835 AMD_PG_SUPPORT_RLC_SMU_HS)) {
2836 WREG32(mmRLC_JUMP_TABLE_RESTORE,
2837 adev->gfx.rlc.cp_table_gpu_addr >> 8);
2838 gfx_v9_0_init_gfx_power_gating(adev);
2839 }
2840 }
2841
2842 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2843 {
2844 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2845 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
2846 gfx_v9_0_wait_for_rlc_serdes(adev);
2847 }
2848
2849 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2850 {
2851 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2852 udelay(50);
2853 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
2854 udelay(50);
2855 }
2856
2857 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
2858 {
2859 #ifdef AMDGPU_RLC_DEBUG_RETRY
2860 u32 rlc_ucode_ver;
2861 #endif
2862
2863 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
2864 udelay(50);
2865
2866 /* carrizo do enable cp interrupt after cp inited */
2867 if (!(adev->flags & AMD_IS_APU)) {
2868 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
2869 udelay(50);
2870 }
2871
2872 #ifdef AMDGPU_RLC_DEBUG_RETRY
2873 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
2874 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
2875 if(rlc_ucode_ver == 0x108) {
2876 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
2877 rlc_ucode_ver, adev->gfx.rlc_fw_version);
2878 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
2879 * default is 0x9C4 to create a 100us interval */
2880 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
2881 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
2882 * to disable the page fault retry interrupts, default is
2883 * 0x100 (256) */
2884 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
2885 }
2886 #endif
2887 }
2888
2889 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
2890 {
2891 const struct rlc_firmware_header_v2_0 *hdr;
2892 const __le32 *fw_data;
2893 unsigned i, fw_size;
2894
2895 if (!adev->gfx.rlc_fw)
2896 return -EINVAL;
2897
2898 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
2899 amdgpu_ucode_print_rlc_hdr(&hdr->header);
2900
2901 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
2902 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2903 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
2904
2905 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
2906 RLCG_UCODE_LOADING_START_ADDRESS);
2907 for (i = 0; i < fw_size; i++)
2908 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
2909 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
2910
2911 return 0;
2912 }
2913
2914 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
2915 {
2916 int r;
2917
2918 if (amdgpu_sriov_vf(adev)) {
2919 gfx_v9_0_init_csb(adev);
2920 return 0;
2921 }
2922
2923 adev->gfx.rlc.funcs->stop(adev);
2924
2925 /* disable CG */
2926 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
2927
2928 gfx_v9_0_init_pg(adev);
2929
2930 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
2931 /* legacy rlc firmware loading */
2932 r = gfx_v9_0_rlc_load_microcode(adev);
2933 if (r)
2934 return r;
2935 }
2936
2937 switch (adev->asic_type) {
2938 case CHIP_RAVEN:
2939 if (amdgpu_lbpw == 0)
2940 gfx_v9_0_enable_lbpw(adev, false);
2941 else
2942 gfx_v9_0_enable_lbpw(adev, true);
2943 break;
2944 case CHIP_VEGA20:
2945 if (amdgpu_lbpw > 0)
2946 gfx_v9_0_enable_lbpw(adev, true);
2947 else
2948 gfx_v9_0_enable_lbpw(adev, false);
2949 break;
2950 default:
2951 break;
2952 }
2953
2954 adev->gfx.rlc.funcs->start(adev);
2955
2956 return 0;
2957 }
2958
2959 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
2960 {
2961 int i;
2962 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
2963
2964 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
2965 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
2966 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
2967 if (!enable) {
2968 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2969 adev->gfx.gfx_ring[i].sched.ready = false;
2970 }
2971 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
2972 udelay(50);
2973 }
2974
2975 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
2976 {
2977 const struct gfx_firmware_header_v1_0 *pfp_hdr;
2978 const struct gfx_firmware_header_v1_0 *ce_hdr;
2979 const struct gfx_firmware_header_v1_0 *me_hdr;
2980 const __le32 *fw_data;
2981 unsigned i, fw_size;
2982
2983 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
2984 return -EINVAL;
2985
2986 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
2987 adev->gfx.pfp_fw->data;
2988 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
2989 adev->gfx.ce_fw->data;
2990 me_hdr = (const struct gfx_firmware_header_v1_0 *)
2991 adev->gfx.me_fw->data;
2992
2993 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
2994 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
2995 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
2996
2997 gfx_v9_0_cp_gfx_enable(adev, false);
2998
2999 /* PFP */
3000 fw_data = (const __le32 *)
3001 (adev->gfx.pfp_fw->data +
3002 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3003 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3004 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3005 for (i = 0; i < fw_size; i++)
3006 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3007 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3008
3009 /* CE */
3010 fw_data = (const __le32 *)
3011 (adev->gfx.ce_fw->data +
3012 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3013 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3014 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3015 for (i = 0; i < fw_size; i++)
3016 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3017 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3018
3019 /* ME */
3020 fw_data = (const __le32 *)
3021 (adev->gfx.me_fw->data +
3022 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3023 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3024 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3025 for (i = 0; i < fw_size; i++)
3026 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3027 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3028
3029 return 0;
3030 }
3031
3032 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3033 {
3034 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3035 const struct cs_section_def *sect = NULL;
3036 const struct cs_extent_def *ext = NULL;
3037 int r, i, tmp;
3038
3039 /* init the CP */
3040 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3041 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3042
3043 gfx_v9_0_cp_gfx_enable(adev, true);
3044
3045 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3046 if (r) {
3047 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3048 return r;
3049 }
3050
3051 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3052 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3053
3054 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3055 amdgpu_ring_write(ring, 0x80000000);
3056 amdgpu_ring_write(ring, 0x80000000);
3057
3058 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3059 for (ext = sect->section; ext->extent != NULL; ++ext) {
3060 if (sect->id == SECT_CONTEXT) {
3061 amdgpu_ring_write(ring,
3062 PACKET3(PACKET3_SET_CONTEXT_REG,
3063 ext->reg_count));
3064 amdgpu_ring_write(ring,
3065 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3066 for (i = 0; i < ext->reg_count; i++)
3067 amdgpu_ring_write(ring, ext->extent[i]);
3068 }
3069 }
3070 }
3071
3072 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3073 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3074
3075 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3076 amdgpu_ring_write(ring, 0);
3077
3078 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3079 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3080 amdgpu_ring_write(ring, 0x8000);
3081 amdgpu_ring_write(ring, 0x8000);
3082
3083 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3084 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3085 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3086 amdgpu_ring_write(ring, tmp);
3087 amdgpu_ring_write(ring, 0);
3088
3089 amdgpu_ring_commit(ring);
3090
3091 return 0;
3092 }
3093
3094 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3095 {
3096 struct amdgpu_ring *ring;
3097 u32 tmp;
3098 u32 rb_bufsz;
3099 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3100
3101 /* Set the write pointer delay */
3102 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3103
3104 /* set the RB to use vmid 0 */
3105 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3106
3107 /* Set ring buffer size */
3108 ring = &adev->gfx.gfx_ring[0];
3109 rb_bufsz = order_base_2(ring->ring_size / 8);
3110 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3111 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3112 #ifdef __BIG_ENDIAN
3113 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3114 #endif
3115 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3116
3117 /* Initialize the ring buffer's write pointers */
3118 ring->wptr = 0;
3119 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3120 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3121
3122 /* set the wb address wether it's enabled or not */
3123 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3124 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3125 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3126
3127 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3128 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3129 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3130
3131 mdelay(1);
3132 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3133
3134 rb_addr = ring->gpu_addr >> 8;
3135 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3136 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3137
3138 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3139 if (ring->use_doorbell) {
3140 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3141 DOORBELL_OFFSET, ring->doorbell_index);
3142 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3143 DOORBELL_EN, 1);
3144 } else {
3145 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3146 }
3147 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3148
3149 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3150 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3151 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3152
3153 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3154 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3155
3156
3157 /* start the ring */
3158 gfx_v9_0_cp_gfx_start(adev);
3159 ring->sched.ready = true;
3160
3161 return 0;
3162 }
3163
3164 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3165 {
3166 int i;
3167
3168 if (enable) {
3169 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3170 } else {
3171 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3172 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3173 for (i = 0; i < adev->gfx.num_compute_rings; i++)
3174 adev->gfx.compute_ring[i].sched.ready = false;
3175 adev->gfx.kiq.ring.sched.ready = false;
3176 }
3177 udelay(50);
3178 }
3179
3180 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3181 {
3182 const struct gfx_firmware_header_v1_0 *mec_hdr;
3183 const __le32 *fw_data;
3184 unsigned i;
3185 u32 tmp;
3186
3187 if (!adev->gfx.mec_fw)
3188 return -EINVAL;
3189
3190 gfx_v9_0_cp_compute_enable(adev, false);
3191
3192 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3193 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3194
3195 fw_data = (const __le32 *)
3196 (adev->gfx.mec_fw->data +
3197 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3198 tmp = 0;
3199 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3200 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3201 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3202
3203 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3204 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3205 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3206 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3207
3208 /* MEC1 */
3209 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3210 mec_hdr->jt_offset);
3211 for (i = 0; i < mec_hdr->jt_size; i++)
3212 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3213 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3214
3215 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3216 adev->gfx.mec_fw_version);
3217 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3218
3219 return 0;
3220 }
3221
3222 /* KIQ functions */
3223 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3224 {
3225 uint32_t tmp;
3226 struct amdgpu_device *adev = ring->adev;
3227
3228 /* tell RLC which is KIQ queue */
3229 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3230 tmp &= 0xffffff00;
3231 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3232 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3233 tmp |= 0x80;
3234 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3235 }
3236
3237 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3238 {
3239 struct amdgpu_device *adev = ring->adev;
3240 struct v9_mqd *mqd = ring->mqd_ptr;
3241 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3242 uint32_t tmp;
3243
3244 mqd->header = 0xC0310800;
3245 mqd->compute_pipelinestat_enable = 0x00000001;
3246 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3247 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3248 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3249 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3250 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3251 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3252 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3253 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3254 mqd->compute_misc_reserved = 0x00000003;
3255
3256 mqd->dynamic_cu_mask_addr_lo =
3257 lower_32_bits(ring->mqd_gpu_addr
3258 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3259 mqd->dynamic_cu_mask_addr_hi =
3260 upper_32_bits(ring->mqd_gpu_addr
3261 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3262
3263 eop_base_addr = ring->eop_gpu_addr >> 8;
3264 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3265 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3266
3267 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3268 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3269 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3270 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3271
3272 mqd->cp_hqd_eop_control = tmp;
3273
3274 /* enable doorbell? */
3275 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3276
3277 if (ring->use_doorbell) {
3278 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3279 DOORBELL_OFFSET, ring->doorbell_index);
3280 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3281 DOORBELL_EN, 1);
3282 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3283 DOORBELL_SOURCE, 0);
3284 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3285 DOORBELL_HIT, 0);
3286 } else {
3287 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3288 DOORBELL_EN, 0);
3289 }
3290
3291 mqd->cp_hqd_pq_doorbell_control = tmp;
3292
3293 /* disable the queue if it's active */
3294 ring->wptr = 0;
3295 mqd->cp_hqd_dequeue_request = 0;
3296 mqd->cp_hqd_pq_rptr = 0;
3297 mqd->cp_hqd_pq_wptr_lo = 0;
3298 mqd->cp_hqd_pq_wptr_hi = 0;
3299
3300 /* set the pointer to the MQD */
3301 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3302 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3303
3304 /* set MQD vmid to 0 */
3305 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3306 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3307 mqd->cp_mqd_control = tmp;
3308
3309 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3310 hqd_gpu_addr = ring->gpu_addr >> 8;
3311 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3312 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3313
3314 /* set up the HQD, this is similar to CP_RB0_CNTL */
3315 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3316 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3317 (order_base_2(ring->ring_size / 4) - 1));
3318 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3319 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3320 #ifdef __BIG_ENDIAN
3321 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3322 #endif
3323 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3324 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3325 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3326 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3327 mqd->cp_hqd_pq_control = tmp;
3328
3329 /* set the wb address whether it's enabled or not */
3330 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3331 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3332 mqd->cp_hqd_pq_rptr_report_addr_hi =
3333 upper_32_bits(wb_gpu_addr) & 0xffff;
3334
3335 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3336 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3337 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3338 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3339
3340 tmp = 0;
3341 /* enable the doorbell if requested */
3342 if (ring->use_doorbell) {
3343 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3344 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3345 DOORBELL_OFFSET, ring->doorbell_index);
3346
3347 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3348 DOORBELL_EN, 1);
3349 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3350 DOORBELL_SOURCE, 0);
3351 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3352 DOORBELL_HIT, 0);
3353 }
3354
3355 mqd->cp_hqd_pq_doorbell_control = tmp;
3356
3357 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3358 ring->wptr = 0;
3359 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3360
3361 /* set the vmid for the queue */
3362 mqd->cp_hqd_vmid = 0;
3363
3364 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3365 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3366 mqd->cp_hqd_persistent_state = tmp;
3367
3368 /* set MIN_IB_AVAIL_SIZE */
3369 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3370 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3371 mqd->cp_hqd_ib_control = tmp;
3372
3373 /* map_queues packet doesn't need activate the queue,
3374 * so only kiq need set this field.
3375 */
3376 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3377 mqd->cp_hqd_active = 1;
3378
3379 return 0;
3380 }
3381
3382 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3383 {
3384 struct amdgpu_device *adev = ring->adev;
3385 struct v9_mqd *mqd = ring->mqd_ptr;
3386 int j;
3387
3388 /* disable wptr polling */
3389 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3390
3391 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3392 mqd->cp_hqd_eop_base_addr_lo);
3393 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3394 mqd->cp_hqd_eop_base_addr_hi);
3395
3396 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3397 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3398 mqd->cp_hqd_eop_control);
3399
3400 /* enable doorbell? */
3401 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3402 mqd->cp_hqd_pq_doorbell_control);
3403
3404 /* disable the queue if it's active */
3405 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3406 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3407 for (j = 0; j < adev->usec_timeout; j++) {
3408 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3409 break;
3410 udelay(1);
3411 }
3412 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3413 mqd->cp_hqd_dequeue_request);
3414 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3415 mqd->cp_hqd_pq_rptr);
3416 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3417 mqd->cp_hqd_pq_wptr_lo);
3418 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3419 mqd->cp_hqd_pq_wptr_hi);
3420 }
3421
3422 /* set the pointer to the MQD */
3423 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3424 mqd->cp_mqd_base_addr_lo);
3425 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3426 mqd->cp_mqd_base_addr_hi);
3427
3428 /* set MQD vmid to 0 */
3429 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3430 mqd->cp_mqd_control);
3431
3432 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3433 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3434 mqd->cp_hqd_pq_base_lo);
3435 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3436 mqd->cp_hqd_pq_base_hi);
3437
3438 /* set up the HQD, this is similar to CP_RB0_CNTL */
3439 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3440 mqd->cp_hqd_pq_control);
3441
3442 /* set the wb address whether it's enabled or not */
3443 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3444 mqd->cp_hqd_pq_rptr_report_addr_lo);
3445 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3446 mqd->cp_hqd_pq_rptr_report_addr_hi);
3447
3448 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3449 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3450 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3451 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3452 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3453
3454 /* enable the doorbell if requested */
3455 if (ring->use_doorbell) {
3456 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3457 (adev->doorbell_index.kiq * 2) << 2);
3458 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3459 (adev->doorbell_index.userqueue_end * 2) << 2);
3460 }
3461
3462 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3463 mqd->cp_hqd_pq_doorbell_control);
3464
3465 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3466 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3467 mqd->cp_hqd_pq_wptr_lo);
3468 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3469 mqd->cp_hqd_pq_wptr_hi);
3470
3471 /* set the vmid for the queue */
3472 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3473
3474 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3475 mqd->cp_hqd_persistent_state);
3476
3477 /* activate the queue */
3478 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3479 mqd->cp_hqd_active);
3480
3481 if (ring->use_doorbell)
3482 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3483
3484 return 0;
3485 }
3486
3487 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3488 {
3489 struct amdgpu_device *adev = ring->adev;
3490 int j;
3491
3492 /* disable the queue if it's active */
3493 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3494
3495 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3496
3497 for (j = 0; j < adev->usec_timeout; j++) {
3498 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3499 break;
3500 udelay(1);
3501 }
3502
3503 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3504 DRM_DEBUG("KIQ dequeue request failed.\n");
3505
3506 /* Manual disable if dequeue request times out */
3507 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3508 }
3509
3510 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3511 0);
3512 }
3513
3514 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3515 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3516 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3517 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3518 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3519 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3520 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3521 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3522
3523 return 0;
3524 }
3525
3526 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3527 {
3528 struct amdgpu_device *adev = ring->adev;
3529 struct v9_mqd *mqd = ring->mqd_ptr;
3530 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3531
3532 gfx_v9_0_kiq_setting(ring);
3533
3534 if (adev->in_gpu_reset) { /* for GPU_RESET case */
3535 /* reset MQD to a clean status */
3536 if (adev->gfx.mec.mqd_backup[mqd_idx])
3537 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3538
3539 /* reset ring buffer */
3540 ring->wptr = 0;
3541 amdgpu_ring_clear_ring(ring);
3542
3543 mutex_lock(&adev->srbm_mutex);
3544 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3545 gfx_v9_0_kiq_init_register(ring);
3546 soc15_grbm_select(adev, 0, 0, 0, 0);
3547 mutex_unlock(&adev->srbm_mutex);
3548 } else {
3549 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3550 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3551 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3552 mutex_lock(&adev->srbm_mutex);
3553 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3554 gfx_v9_0_mqd_init(ring);
3555 gfx_v9_0_kiq_init_register(ring);
3556 soc15_grbm_select(adev, 0, 0, 0, 0);
3557 mutex_unlock(&adev->srbm_mutex);
3558
3559 if (adev->gfx.mec.mqd_backup[mqd_idx])
3560 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3561 }
3562
3563 return 0;
3564 }
3565
3566 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3567 {
3568 struct amdgpu_device *adev = ring->adev;
3569 struct v9_mqd *mqd = ring->mqd_ptr;
3570 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3571
3572 if (!adev->in_gpu_reset && !adev->in_suspend) {
3573 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3574 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3575 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3576 mutex_lock(&adev->srbm_mutex);
3577 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3578 gfx_v9_0_mqd_init(ring);
3579 soc15_grbm_select(adev, 0, 0, 0, 0);
3580 mutex_unlock(&adev->srbm_mutex);
3581
3582 if (adev->gfx.mec.mqd_backup[mqd_idx])
3583 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3584 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
3585 /* reset MQD to a clean status */
3586 if (adev->gfx.mec.mqd_backup[mqd_idx])
3587 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3588
3589 /* reset ring buffer */
3590 ring->wptr = 0;
3591 amdgpu_ring_clear_ring(ring);
3592 } else {
3593 amdgpu_ring_clear_ring(ring);
3594 }
3595
3596 return 0;
3597 }
3598
3599 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3600 {
3601 struct amdgpu_ring *ring;
3602 int r;
3603
3604 ring = &adev->gfx.kiq.ring;
3605
3606 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3607 if (unlikely(r != 0))
3608 return r;
3609
3610 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3611 if (unlikely(r != 0))
3612 return r;
3613
3614 gfx_v9_0_kiq_init_queue(ring);
3615 amdgpu_bo_kunmap(ring->mqd_obj);
3616 ring->mqd_ptr = NULL;
3617 amdgpu_bo_unreserve(ring->mqd_obj);
3618 ring->sched.ready = true;
3619 return 0;
3620 }
3621
3622 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3623 {
3624 struct amdgpu_ring *ring = NULL;
3625 int r = 0, i;
3626
3627 gfx_v9_0_cp_compute_enable(adev, true);
3628
3629 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3630 ring = &adev->gfx.compute_ring[i];
3631
3632 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3633 if (unlikely(r != 0))
3634 goto done;
3635 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3636 if (!r) {
3637 r = gfx_v9_0_kcq_init_queue(ring);
3638 amdgpu_bo_kunmap(ring->mqd_obj);
3639 ring->mqd_ptr = NULL;
3640 }
3641 amdgpu_bo_unreserve(ring->mqd_obj);
3642 if (r)
3643 goto done;
3644 }
3645
3646 r = amdgpu_gfx_enable_kcq(adev);
3647 done:
3648 return r;
3649 }
3650
3651 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3652 {
3653 int r, i;
3654 struct amdgpu_ring *ring;
3655
3656 if (!(adev->flags & AMD_IS_APU))
3657 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3658
3659 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3660 if (adev->asic_type != CHIP_ARCTURUS) {
3661 /* legacy firmware loading */
3662 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3663 if (r)
3664 return r;
3665 }
3666
3667 r = gfx_v9_0_cp_compute_load_microcode(adev);
3668 if (r)
3669 return r;
3670 }
3671
3672 r = gfx_v9_0_kiq_resume(adev);
3673 if (r)
3674 return r;
3675
3676 if (adev->asic_type != CHIP_ARCTURUS) {
3677 r = gfx_v9_0_cp_gfx_resume(adev);
3678 if (r)
3679 return r;
3680 }
3681
3682 r = gfx_v9_0_kcq_resume(adev);
3683 if (r)
3684 return r;
3685
3686 if (adev->asic_type != CHIP_ARCTURUS) {
3687 ring = &adev->gfx.gfx_ring[0];
3688 r = amdgpu_ring_test_helper(ring);
3689 if (r)
3690 return r;
3691 }
3692
3693 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3694 ring = &adev->gfx.compute_ring[i];
3695 amdgpu_ring_test_helper(ring);
3696 }
3697
3698 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3699
3700 return 0;
3701 }
3702
3703 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3704 {
3705 u32 tmp;
3706
3707 if (adev->asic_type != CHIP_ARCTURUS)
3708 return;
3709
3710 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3711 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3712 adev->df.hash_status.hash_64k);
3713 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3714 adev->df.hash_status.hash_2m);
3715 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3716 adev->df.hash_status.hash_1g);
3717 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3718 }
3719
3720 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3721 {
3722 if (adev->asic_type != CHIP_ARCTURUS)
3723 gfx_v9_0_cp_gfx_enable(adev, enable);
3724 gfx_v9_0_cp_compute_enable(adev, enable);
3725 }
3726
3727 static int gfx_v9_0_hw_init(void *handle)
3728 {
3729 int r;
3730 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3731
3732 if (!amdgpu_sriov_vf(adev))
3733 gfx_v9_0_init_golden_registers(adev);
3734
3735 gfx_v9_0_constants_init(adev);
3736
3737 gfx_v9_0_init_tcp_config(adev);
3738
3739 r = adev->gfx.rlc.funcs->resume(adev);
3740 if (r)
3741 return r;
3742
3743 r = gfx_v9_0_cp_resume(adev);
3744 if (r)
3745 return r;
3746
3747 return r;
3748 }
3749
3750 static int gfx_v9_0_hw_fini(void *handle)
3751 {
3752 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3753
3754 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3755 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3756 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3757
3758 /* DF freeze and kcq disable will fail */
3759 if (!amdgpu_ras_intr_triggered())
3760 /* disable KCQ to avoid CPC touch memory not valid anymore */
3761 amdgpu_gfx_disable_kcq(adev);
3762
3763 if (amdgpu_sriov_vf(adev)) {
3764 gfx_v9_0_cp_gfx_enable(adev, false);
3765 /* must disable polling for SRIOV when hw finished, otherwise
3766 * CPC engine may still keep fetching WB address which is already
3767 * invalid after sw finished and trigger DMAR reading error in
3768 * hypervisor side.
3769 */
3770 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3771 return 0;
3772 }
3773
3774 /* Use deinitialize sequence from CAIL when unbinding device from driver,
3775 * otherwise KIQ is hanging when binding back
3776 */
3777 if (!adev->in_gpu_reset && !adev->in_suspend) {
3778 mutex_lock(&adev->srbm_mutex);
3779 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3780 adev->gfx.kiq.ring.pipe,
3781 adev->gfx.kiq.ring.queue, 0);
3782 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3783 soc15_grbm_select(adev, 0, 0, 0, 0);
3784 mutex_unlock(&adev->srbm_mutex);
3785 }
3786
3787 gfx_v9_0_cp_enable(adev, false);
3788 adev->gfx.rlc.funcs->stop(adev);
3789
3790 return 0;
3791 }
3792
3793 static int gfx_v9_0_suspend(void *handle)
3794 {
3795 return gfx_v9_0_hw_fini(handle);
3796 }
3797
3798 static int gfx_v9_0_resume(void *handle)
3799 {
3800 return gfx_v9_0_hw_init(handle);
3801 }
3802
3803 static bool gfx_v9_0_is_idle(void *handle)
3804 {
3805 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3806
3807 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3808 GRBM_STATUS, GUI_ACTIVE))
3809 return false;
3810 else
3811 return true;
3812 }
3813
3814 static int gfx_v9_0_wait_for_idle(void *handle)
3815 {
3816 unsigned i;
3817 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3818
3819 for (i = 0; i < adev->usec_timeout; i++) {
3820 if (gfx_v9_0_is_idle(handle))
3821 return 0;
3822 udelay(1);
3823 }
3824 return -ETIMEDOUT;
3825 }
3826
3827 static int gfx_v9_0_soft_reset(void *handle)
3828 {
3829 u32 grbm_soft_reset = 0;
3830 u32 tmp;
3831 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3832
3833 /* GRBM_STATUS */
3834 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
3835 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
3836 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
3837 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
3838 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
3839 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3840 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
3841 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3842 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3843 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3844 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
3845 }
3846
3847 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
3848 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3849 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
3850 }
3851
3852 /* GRBM_STATUS2 */
3853 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
3854 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
3855 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
3856 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
3857
3858
3859 if (grbm_soft_reset) {
3860 /* stop the rlc */
3861 adev->gfx.rlc.funcs->stop(adev);
3862
3863 if (adev->asic_type != CHIP_ARCTURUS)
3864 /* Disable GFX parsing/prefetching */
3865 gfx_v9_0_cp_gfx_enable(adev, false);
3866
3867 /* Disable MEC parsing/prefetching */
3868 gfx_v9_0_cp_compute_enable(adev, false);
3869
3870 if (grbm_soft_reset) {
3871 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3872 tmp |= grbm_soft_reset;
3873 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3874 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3875 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3876
3877 udelay(50);
3878
3879 tmp &= ~grbm_soft_reset;
3880 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
3881 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
3882 }
3883
3884 /* Wait a little for things to settle down */
3885 udelay(50);
3886 }
3887 return 0;
3888 }
3889
3890 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
3891 {
3892 uint64_t clock;
3893
3894 mutex_lock(&adev->gfx.gpu_clock_mutex);
3895 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
3896 uint32_t tmp, lsb, msb, i = 0;
3897 do {
3898 if (i != 0)
3899 udelay(1);
3900 tmp = RREG32_SOC15(GC, 0, mmRLC_REFCLOCK_TIMESTAMP_MSB);
3901 lsb = RREG32_SOC15(GC, 0, mmRLC_REFCLOCK_TIMESTAMP_LSB);
3902 msb = RREG32_SOC15(GC, 0, mmRLC_REFCLOCK_TIMESTAMP_MSB);
3903 i++;
3904 } while (unlikely(tmp != msb) && (i < adev->usec_timeout));
3905 clock = (uint64_t)lsb | ((uint64_t)msb << 32ULL);
3906 } else {
3907 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
3908 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
3909 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
3910 }
3911 mutex_unlock(&adev->gfx.gpu_clock_mutex);
3912 return clock;
3913 }
3914
3915 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
3916 uint32_t vmid,
3917 uint32_t gds_base, uint32_t gds_size,
3918 uint32_t gws_base, uint32_t gws_size,
3919 uint32_t oa_base, uint32_t oa_size)
3920 {
3921 struct amdgpu_device *adev = ring->adev;
3922
3923 /* GDS Base */
3924 gfx_v9_0_write_data_to_reg(ring, 0, false,
3925 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
3926 gds_base);
3927
3928 /* GDS Size */
3929 gfx_v9_0_write_data_to_reg(ring, 0, false,
3930 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
3931 gds_size);
3932
3933 /* GWS */
3934 gfx_v9_0_write_data_to_reg(ring, 0, false,
3935 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
3936 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
3937
3938 /* OA */
3939 gfx_v9_0_write_data_to_reg(ring, 0, false,
3940 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
3941 (1 << (oa_size + oa_base)) - (1 << oa_base));
3942 }
3943
3944 static const u32 vgpr_init_compute_shader[] =
3945 {
3946 0xb07c0000, 0xbe8000ff,
3947 0x000000f8, 0xbf110800,
3948 0x7e000280, 0x7e020280,
3949 0x7e040280, 0x7e060280,
3950 0x7e080280, 0x7e0a0280,
3951 0x7e0c0280, 0x7e0e0280,
3952 0x80808800, 0xbe803200,
3953 0xbf84fff5, 0xbf9c0000,
3954 0xd28c0001, 0x0001007f,
3955 0xd28d0001, 0x0002027e,
3956 0x10020288, 0xb8810904,
3957 0xb7814000, 0xd1196a01,
3958 0x00000301, 0xbe800087,
3959 0xbefc00c1, 0xd89c4000,
3960 0x00020201, 0xd89cc080,
3961 0x00040401, 0x320202ff,
3962 0x00000800, 0x80808100,
3963 0xbf84fff8, 0x7e020280,
3964 0xbf810000, 0x00000000,
3965 };
3966
3967 static const u32 sgpr_init_compute_shader[] =
3968 {
3969 0xb07c0000, 0xbe8000ff,
3970 0x0000005f, 0xbee50080,
3971 0xbe812c65, 0xbe822c65,
3972 0xbe832c65, 0xbe842c65,
3973 0xbe852c65, 0xb77c0005,
3974 0x80808500, 0xbf84fff8,
3975 0xbe800080, 0xbf810000,
3976 };
3977
3978 /* When below register arrays changed, please update gpr_reg_size,
3979 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
3980 to cover all gfx9 ASICs */
3981 static const struct soc15_reg_entry vgpr_init_regs[] = {
3982 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
3983 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
3984 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
3985 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
3986 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
3987 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
3988 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
3989 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
3990 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
3991 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
3992 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
3993 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
3994 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
3995 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
3996 };
3997
3998 static const struct soc15_reg_entry sgpr1_init_regs[] = {
3999 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4000 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4001 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4002 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4003 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4004 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4005 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4006 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4007 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4008 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4009 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4010 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4011 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4012 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4013 };
4014
4015 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4016 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4017 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4018 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4019 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4020 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4021 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4022 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4023 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4024 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4025 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4026 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4027 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4028 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4029 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4030 };
4031
4032 static const struct soc15_reg_entry sec_ded_counter_registers[] = {
4033 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4034 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4035 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4036 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4037 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4038 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4039 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4040 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4041 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4042 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4043 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4044 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4045 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4046 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4047 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4048 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4049 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4050 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4051 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4052 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4053 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4054 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4055 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4056 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4057 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4058 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4059 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4060 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4061 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4062 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4063 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4064 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4065 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4066 { SOC15_REG_ENTRY(HDP, 0, mmHDP_EDC_CNT), 0, 1, 1},
4067 };
4068
4069 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4070 {
4071 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4072 int i, r;
4073
4074 /* only support when RAS is enabled */
4075 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4076 return 0;
4077
4078 r = amdgpu_ring_alloc(ring, 7);
4079 if (r) {
4080 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4081 ring->name, r);
4082 return r;
4083 }
4084
4085 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4086 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4087
4088 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4089 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4090 PACKET3_DMA_DATA_DST_SEL(1) |
4091 PACKET3_DMA_DATA_SRC_SEL(2) |
4092 PACKET3_DMA_DATA_ENGINE(0)));
4093 amdgpu_ring_write(ring, 0);
4094 amdgpu_ring_write(ring, 0);
4095 amdgpu_ring_write(ring, 0);
4096 amdgpu_ring_write(ring, 0);
4097 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4098 adev->gds.gds_size);
4099
4100 amdgpu_ring_commit(ring);
4101
4102 for (i = 0; i < adev->usec_timeout; i++) {
4103 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4104 break;
4105 udelay(1);
4106 }
4107
4108 if (i >= adev->usec_timeout)
4109 r = -ETIMEDOUT;
4110
4111 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4112
4113 return r;
4114 }
4115
4116 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4117 {
4118 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4119 struct amdgpu_ib ib;
4120 struct dma_fence *f = NULL;
4121 int r, i, j, k;
4122 unsigned total_size, vgpr_offset, sgpr_offset;
4123 u64 gpu_addr;
4124
4125 int compute_dim_x = adev->gfx.config.max_shader_engines *
4126 adev->gfx.config.max_cu_per_sh *
4127 adev->gfx.config.max_sh_per_se;
4128 int sgpr_work_group_size = 5;
4129 int gpr_reg_size = compute_dim_x / 16 + 6;
4130
4131 /* only support when RAS is enabled */
4132 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4133 return 0;
4134
4135 /* bail if the compute ring is not ready */
4136 if (!ring->sched.ready)
4137 return 0;
4138
4139 total_size =
4140 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4141 total_size +=
4142 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4143 total_size +=
4144 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4145 total_size = ALIGN(total_size, 256);
4146 vgpr_offset = total_size;
4147 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
4148 sgpr_offset = total_size;
4149 total_size += sizeof(sgpr_init_compute_shader);
4150
4151 /* allocate an indirect buffer to put the commands in */
4152 memset(&ib, 0, sizeof(ib));
4153 r = amdgpu_ib_get(adev, NULL, total_size, &ib);
4154 if (r) {
4155 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4156 return r;
4157 }
4158
4159 /* load the compute shaders */
4160 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
4161 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
4162
4163 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4164 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4165
4166 /* init the ib length to 0 */
4167 ib.length_dw = 0;
4168
4169 /* VGPR */
4170 /* write the register state for the compute dispatch */
4171 for (i = 0; i < gpr_reg_size; i++) {
4172 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4173 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs[i])
4174 - PACKET3_SET_SH_REG_START;
4175 ib.ptr[ib.length_dw++] = vgpr_init_regs[i].reg_value;
4176 }
4177 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4178 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4179 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4180 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4181 - PACKET3_SET_SH_REG_START;
4182 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4183 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4184
4185 /* write dispatch packet */
4186 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4187 ib.ptr[ib.length_dw++] = compute_dim_x; /* x */
4188 ib.ptr[ib.length_dw++] = 1; /* y */
4189 ib.ptr[ib.length_dw++] = 1; /* z */
4190 ib.ptr[ib.length_dw++] =
4191 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4192
4193 /* write CS partial flush packet */
4194 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4195 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4196
4197 /* SGPR1 */
4198 /* write the register state for the compute dispatch */
4199 for (i = 0; i < gpr_reg_size; i++) {
4200 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4201 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4202 - PACKET3_SET_SH_REG_START;
4203 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4204 }
4205 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4206 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4207 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4208 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4209 - PACKET3_SET_SH_REG_START;
4210 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4211 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4212
4213 /* write dispatch packet */
4214 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4215 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4216 ib.ptr[ib.length_dw++] = 1; /* y */
4217 ib.ptr[ib.length_dw++] = 1; /* z */
4218 ib.ptr[ib.length_dw++] =
4219 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4220
4221 /* write CS partial flush packet */
4222 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4223 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4224
4225 /* SGPR2 */
4226 /* write the register state for the compute dispatch */
4227 for (i = 0; i < gpr_reg_size; i++) {
4228 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4229 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4230 - PACKET3_SET_SH_REG_START;
4231 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4232 }
4233 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4234 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4235 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4236 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4237 - PACKET3_SET_SH_REG_START;
4238 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4239 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4240
4241 /* write dispatch packet */
4242 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4243 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4244 ib.ptr[ib.length_dw++] = 1; /* y */
4245 ib.ptr[ib.length_dw++] = 1; /* z */
4246 ib.ptr[ib.length_dw++] =
4247 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4248
4249 /* write CS partial flush packet */
4250 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4251 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4252
4253 /* shedule the ib on the ring */
4254 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4255 if (r) {
4256 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4257 goto fail;
4258 }
4259
4260 /* wait for the GPU to finish processing the IB */
4261 r = dma_fence_wait(f, false);
4262 if (r) {
4263 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4264 goto fail;
4265 }
4266
4267 /* read back registers to clear the counters */
4268 mutex_lock(&adev->grbm_idx_mutex);
4269 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) {
4270 for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) {
4271 for (k = 0; k < sec_ded_counter_registers[i].instance; k++) {
4272 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
4273 RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i]));
4274 }
4275 }
4276 }
4277 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
4278 mutex_unlock(&adev->grbm_idx_mutex);
4279
4280 fail:
4281 amdgpu_ib_free(adev, &ib, NULL);
4282 dma_fence_put(f);
4283
4284 return r;
4285 }
4286
4287 static int gfx_v9_0_early_init(void *handle)
4288 {
4289 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4290
4291 if (adev->asic_type == CHIP_ARCTURUS)
4292 adev->gfx.num_gfx_rings = 0;
4293 else
4294 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4295 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
4296 gfx_v9_0_set_kiq_pm4_funcs(adev);
4297 gfx_v9_0_set_ring_funcs(adev);
4298 gfx_v9_0_set_irq_funcs(adev);
4299 gfx_v9_0_set_gds_init(adev);
4300 gfx_v9_0_set_rlc_funcs(adev);
4301
4302 return 0;
4303 }
4304
4305 static int gfx_v9_0_ecc_late_init(void *handle)
4306 {
4307 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4308 int r;
4309
4310 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4311 if (r)
4312 return r;
4313
4314 /* requires IBs so do in late init after IB pool is initialized */
4315 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4316 if (r)
4317 return r;
4318
4319 r = amdgpu_gfx_ras_late_init(adev);
4320 if (r)
4321 return r;
4322
4323 return 0;
4324 }
4325
4326 static int gfx_v9_0_late_init(void *handle)
4327 {
4328 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4329 int r;
4330
4331 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4332 if (r)
4333 return r;
4334
4335 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4336 if (r)
4337 return r;
4338
4339 r = gfx_v9_0_ecc_late_init(handle);
4340 if (r)
4341 return r;
4342
4343 return 0;
4344 }
4345
4346 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4347 {
4348 uint32_t rlc_setting;
4349
4350 /* if RLC is not enabled, do nothing */
4351 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4352 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4353 return false;
4354
4355 return true;
4356 }
4357
4358 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4359 {
4360 uint32_t data;
4361 unsigned i;
4362
4363 data = RLC_SAFE_MODE__CMD_MASK;
4364 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4365 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4366
4367 /* wait for RLC_SAFE_MODE */
4368 for (i = 0; i < adev->usec_timeout; i++) {
4369 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4370 break;
4371 udelay(1);
4372 }
4373 }
4374
4375 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4376 {
4377 uint32_t data;
4378
4379 data = RLC_SAFE_MODE__CMD_MASK;
4380 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4381 }
4382
4383 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4384 bool enable)
4385 {
4386 amdgpu_gfx_rlc_enter_safe_mode(adev);
4387
4388 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4389 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4390 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4391 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4392 } else {
4393 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4394 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4395 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4396 }
4397
4398 amdgpu_gfx_rlc_exit_safe_mode(adev);
4399 }
4400
4401 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4402 bool enable)
4403 {
4404 /* TODO: double check if we need to perform under safe mode */
4405 /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4406
4407 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4408 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4409 else
4410 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4411
4412 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4413 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4414 else
4415 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4416
4417 /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4418 }
4419
4420 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4421 bool enable)
4422 {
4423 uint32_t data, def;
4424
4425 amdgpu_gfx_rlc_enter_safe_mode(adev);
4426
4427 /* It is disabled by HW by default */
4428 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4429 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4430 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4431
4432 if (adev->asic_type != CHIP_VEGA12)
4433 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4434
4435 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4436 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4437 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4438
4439 /* only for Vega10 & Raven1 */
4440 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4441
4442 if (def != data)
4443 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4444
4445 /* MGLS is a global flag to control all MGLS in GFX */
4446 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4447 /* 2 - RLC memory Light sleep */
4448 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4449 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4450 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4451 if (def != data)
4452 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4453 }
4454 /* 3 - CP memory Light sleep */
4455 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4456 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4457 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4458 if (def != data)
4459 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4460 }
4461 }
4462 } else {
4463 /* 1 - MGCG_OVERRIDE */
4464 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4465
4466 if (adev->asic_type != CHIP_VEGA12)
4467 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4468
4469 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4470 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4471 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4472 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4473
4474 if (def != data)
4475 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4476
4477 /* 2 - disable MGLS in RLC */
4478 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4479 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4480 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4481 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4482 }
4483
4484 /* 3 - disable MGLS in CP */
4485 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4486 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4487 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4488 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4489 }
4490 }
4491
4492 amdgpu_gfx_rlc_exit_safe_mode(adev);
4493 }
4494
4495 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4496 bool enable)
4497 {
4498 uint32_t data, def;
4499
4500 if (adev->asic_type == CHIP_ARCTURUS)
4501 return;
4502
4503 amdgpu_gfx_rlc_enter_safe_mode(adev);
4504
4505 /* Enable 3D CGCG/CGLS */
4506 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4507 /* write cmd to clear cgcg/cgls ov */
4508 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4509 /* unset CGCG override */
4510 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4511 /* update CGCG and CGLS override bits */
4512 if (def != data)
4513 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4514
4515 /* enable 3Dcgcg FSM(0x0000363f) */
4516 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4517
4518 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4519 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4520 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4521 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4522 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4523 if (def != data)
4524 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4525
4526 /* set IDLE_POLL_COUNT(0x00900100) */
4527 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4528 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4529 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4530 if (def != data)
4531 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4532 } else {
4533 /* Disable CGCG/CGLS */
4534 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4535 /* disable cgcg, cgls should be disabled */
4536 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4537 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4538 /* disable cgcg and cgls in FSM */
4539 if (def != data)
4540 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4541 }
4542
4543 amdgpu_gfx_rlc_exit_safe_mode(adev);
4544 }
4545
4546 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4547 bool enable)
4548 {
4549 uint32_t def, data;
4550
4551 amdgpu_gfx_rlc_enter_safe_mode(adev);
4552
4553 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4554 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4555 /* unset CGCG override */
4556 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4557 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4558 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4559 else
4560 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4561 /* update CGCG and CGLS override bits */
4562 if (def != data)
4563 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4564
4565 /* enable cgcg FSM(0x0000363F) */
4566 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4567
4568 if (adev->asic_type == CHIP_ARCTURUS)
4569 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4570 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4571 else
4572 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4573 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4574 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4575 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4576 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4577 if (def != data)
4578 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4579
4580 /* set IDLE_POLL_COUNT(0x00900100) */
4581 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4582 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4583 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4584 if (def != data)
4585 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4586 } else {
4587 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4588 /* reset CGCG/CGLS bits */
4589 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4590 /* disable cgcg and cgls in FSM */
4591 if (def != data)
4592 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4593 }
4594
4595 amdgpu_gfx_rlc_exit_safe_mode(adev);
4596 }
4597
4598 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4599 bool enable)
4600 {
4601 if (enable) {
4602 /* CGCG/CGLS should be enabled after MGCG/MGLS
4603 * === MGCG + MGLS ===
4604 */
4605 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4606 /* === CGCG /CGLS for GFX 3D Only === */
4607 gfx_v9_0_update_3d_clock_gating(adev, enable);
4608 /* === CGCG + CGLS === */
4609 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4610 } else {
4611 /* CGCG/CGLS should be disabled before MGCG/MGLS
4612 * === CGCG + CGLS ===
4613 */
4614 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4615 /* === CGCG /CGLS for GFX 3D Only === */
4616 gfx_v9_0_update_3d_clock_gating(adev, enable);
4617 /* === MGCG + MGLS === */
4618 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4619 }
4620 return 0;
4621 }
4622
4623 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
4624 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
4625 .set_safe_mode = gfx_v9_0_set_safe_mode,
4626 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
4627 .init = gfx_v9_0_rlc_init,
4628 .get_csb_size = gfx_v9_0_get_csb_size,
4629 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
4630 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
4631 .resume = gfx_v9_0_rlc_resume,
4632 .stop = gfx_v9_0_rlc_stop,
4633 .reset = gfx_v9_0_rlc_reset,
4634 .start = gfx_v9_0_rlc_start
4635 };
4636
4637 static int gfx_v9_0_set_powergating_state(void *handle,
4638 enum amd_powergating_state state)
4639 {
4640 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4641 bool enable = (state == AMD_PG_STATE_GATE) ? true : false;
4642
4643 switch (adev->asic_type) {
4644 case CHIP_RAVEN:
4645 case CHIP_RENOIR:
4646 if (!enable) {
4647 amdgpu_gfx_off_ctrl(adev, false);
4648 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
4649 }
4650 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
4651 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
4652 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
4653 } else {
4654 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
4655 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
4656 }
4657
4658 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
4659 gfx_v9_0_enable_cp_power_gating(adev, true);
4660 else
4661 gfx_v9_0_enable_cp_power_gating(adev, false);
4662
4663 /* update gfx cgpg state */
4664 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
4665
4666 /* update mgcg state */
4667 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
4668
4669 if (enable)
4670 amdgpu_gfx_off_ctrl(adev, true);
4671 break;
4672 case CHIP_VEGA12:
4673 if (!enable) {
4674 amdgpu_gfx_off_ctrl(adev, false);
4675 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
4676 } else {
4677 amdgpu_gfx_off_ctrl(adev, true);
4678 }
4679 break;
4680 default:
4681 break;
4682 }
4683
4684 return 0;
4685 }
4686
4687 static int gfx_v9_0_set_clockgating_state(void *handle,
4688 enum amd_clockgating_state state)
4689 {
4690 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4691
4692 if (amdgpu_sriov_vf(adev))
4693 return 0;
4694
4695 switch (adev->asic_type) {
4696 case CHIP_VEGA10:
4697 case CHIP_VEGA12:
4698 case CHIP_VEGA20:
4699 case CHIP_RAVEN:
4700 case CHIP_ARCTURUS:
4701 case CHIP_RENOIR:
4702 gfx_v9_0_update_gfx_clock_gating(adev,
4703 state == AMD_CG_STATE_GATE ? true : false);
4704 break;
4705 default:
4706 break;
4707 }
4708 return 0;
4709 }
4710
4711 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
4712 {
4713 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4714 int data;
4715
4716 if (amdgpu_sriov_vf(adev))
4717 *flags = 0;
4718
4719 /* AMD_CG_SUPPORT_GFX_MGCG */
4720 data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4721 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
4722 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
4723
4724 /* AMD_CG_SUPPORT_GFX_CGCG */
4725 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4726 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
4727 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
4728
4729 /* AMD_CG_SUPPORT_GFX_CGLS */
4730 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
4731 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
4732
4733 /* AMD_CG_SUPPORT_GFX_RLC_LS */
4734 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4735 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
4736 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
4737
4738 /* AMD_CG_SUPPORT_GFX_CP_LS */
4739 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4740 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
4741 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
4742
4743 if (adev->asic_type != CHIP_ARCTURUS) {
4744 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
4745 data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4746 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
4747 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
4748
4749 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
4750 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
4751 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
4752 }
4753 }
4754
4755 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
4756 {
4757 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
4758 }
4759
4760 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
4761 {
4762 struct amdgpu_device *adev = ring->adev;
4763 u64 wptr;
4764
4765 /* XXX check if swapping is necessary on BE */
4766 if (ring->use_doorbell) {
4767 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
4768 } else {
4769 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
4770 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
4771 }
4772
4773 return wptr;
4774 }
4775
4776 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
4777 {
4778 struct amdgpu_device *adev = ring->adev;
4779
4780 if (ring->use_doorbell) {
4781 /* XXX check if swapping is necessary on BE */
4782 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
4783 WDOORBELL64(ring->doorbell_index, ring->wptr);
4784 } else {
4785 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4786 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
4787 }
4788 }
4789
4790 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
4791 {
4792 struct amdgpu_device *adev = ring->adev;
4793 u32 ref_and_mask, reg_mem_engine;
4794 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
4795
4796 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4797 switch (ring->me) {
4798 case 1:
4799 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
4800 break;
4801 case 2:
4802 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
4803 break;
4804 default:
4805 return;
4806 }
4807 reg_mem_engine = 0;
4808 } else {
4809 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
4810 reg_mem_engine = 1; /* pfp */
4811 }
4812
4813 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
4814 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
4815 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
4816 ref_and_mask, ref_and_mask, 0x20);
4817 }
4818
4819 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
4820 struct amdgpu_job *job,
4821 struct amdgpu_ib *ib,
4822 uint32_t flags)
4823 {
4824 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4825 u32 header, control = 0;
4826
4827 if (ib->flags & AMDGPU_IB_FLAG_CE)
4828 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4829 else
4830 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4831
4832 control |= ib->length_dw | (vmid << 24);
4833
4834 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
4835 control |= INDIRECT_BUFFER_PRE_ENB(1);
4836
4837 if (!(ib->flags & AMDGPU_IB_FLAG_CE))
4838 gfx_v9_0_ring_emit_de_meta(ring);
4839 }
4840
4841 amdgpu_ring_write(ring, header);
4842 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
4843 amdgpu_ring_write(ring,
4844 #ifdef __BIG_ENDIAN
4845 (2 << 0) |
4846 #endif
4847 lower_32_bits(ib->gpu_addr));
4848 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4849 amdgpu_ring_write(ring, control);
4850 }
4851
4852 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
4853 struct amdgpu_job *job,
4854 struct amdgpu_ib *ib,
4855 uint32_t flags)
4856 {
4857 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
4858 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
4859
4860 /* Currently, there is a high possibility to get wave ID mismatch
4861 * between ME and GDS, leading to a hw deadlock, because ME generates
4862 * different wave IDs than the GDS expects. This situation happens
4863 * randomly when at least 5 compute pipes use GDS ordered append.
4864 * The wave IDs generated by ME are also wrong after suspend/resume.
4865 * Those are probably bugs somewhere else in the kernel driver.
4866 *
4867 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
4868 * GDS to 0 for this ring (me/pipe).
4869 */
4870 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
4871 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
4872 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
4873 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
4874 }
4875
4876 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
4877 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
4878 amdgpu_ring_write(ring,
4879 #ifdef __BIG_ENDIAN
4880 (2 << 0) |
4881 #endif
4882 lower_32_bits(ib->gpu_addr));
4883 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
4884 amdgpu_ring_write(ring, control);
4885 }
4886
4887 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
4888 u64 seq, unsigned flags)
4889 {
4890 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
4891 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
4892 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
4893
4894 /* RELEASE_MEM - flush caches, send int */
4895 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
4896 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
4897 EOP_TC_NC_ACTION_EN) :
4898 (EOP_TCL1_ACTION_EN |
4899 EOP_TC_ACTION_EN |
4900 EOP_TC_WB_ACTION_EN |
4901 EOP_TC_MD_ACTION_EN)) |
4902 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
4903 EVENT_INDEX(5)));
4904 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
4905
4906 /*
4907 * the address should be Qword aligned if 64bit write, Dword
4908 * aligned if only send 32bit data low (discard data high)
4909 */
4910 if (write64bit)
4911 BUG_ON(addr & 0x7);
4912 else
4913 BUG_ON(addr & 0x3);
4914 amdgpu_ring_write(ring, lower_32_bits(addr));
4915 amdgpu_ring_write(ring, upper_32_bits(addr));
4916 amdgpu_ring_write(ring, lower_32_bits(seq));
4917 amdgpu_ring_write(ring, upper_32_bits(seq));
4918 amdgpu_ring_write(ring, 0);
4919 }
4920
4921 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
4922 {
4923 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
4924 uint32_t seq = ring->fence_drv.sync_seq;
4925 uint64_t addr = ring->fence_drv.gpu_addr;
4926
4927 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
4928 lower_32_bits(addr), upper_32_bits(addr),
4929 seq, 0xffffffff, 4);
4930 }
4931
4932 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
4933 unsigned vmid, uint64_t pd_addr)
4934 {
4935 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
4936
4937 /* compute doesn't have PFP */
4938 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
4939 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4940 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4941 amdgpu_ring_write(ring, 0x0);
4942 }
4943 }
4944
4945 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
4946 {
4947 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
4948 }
4949
4950 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
4951 {
4952 u64 wptr;
4953
4954 /* XXX check if swapping is necessary on BE */
4955 if (ring->use_doorbell)
4956 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
4957 else
4958 BUG();
4959 return wptr;
4960 }
4961
4962 static void gfx_v9_0_ring_set_pipe_percent(struct amdgpu_ring *ring,
4963 bool acquire)
4964 {
4965 struct amdgpu_device *adev = ring->adev;
4966 int pipe_num, tmp, reg;
4967 int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1;
4968
4969 pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;
4970
4971 /* first me only has 2 entries, GFX and HP3D */
4972 if (ring->me > 0)
4973 pipe_num -= 2;
4974
4975 reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX) + pipe_num;
4976 tmp = RREG32(reg);
4977 tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent);
4978 WREG32(reg, tmp);
4979 }
4980
4981 static void gfx_v9_0_pipe_reserve_resources(struct amdgpu_device *adev,
4982 struct amdgpu_ring *ring,
4983 bool acquire)
4984 {
4985 int i, pipe;
4986 bool reserve;
4987 struct amdgpu_ring *iring;
4988
4989 mutex_lock(&adev->gfx.pipe_reserve_mutex);
4990 pipe = amdgpu_gfx_mec_queue_to_bit(adev, ring->me, ring->pipe, 0);
4991 if (acquire)
4992 set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
4993 else
4994 clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);
4995
4996 if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {
4997 /* Clear all reservations - everyone reacquires all resources */
4998 for (i = 0; i < adev->gfx.num_gfx_rings; ++i)
4999 gfx_v9_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],
5000 true);
5001
5002 for (i = 0; i < adev->gfx.num_compute_rings; ++i)
5003 gfx_v9_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],
5004 true);
5005 } else {
5006 /* Lower all pipes without a current reservation */
5007 for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
5008 iring = &adev->gfx.gfx_ring[i];
5009 pipe = amdgpu_gfx_mec_queue_to_bit(adev,
5010 iring->me,
5011 iring->pipe,
5012 0);
5013 reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
5014 gfx_v9_0_ring_set_pipe_percent(iring, reserve);
5015 }
5016
5017 for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
5018 iring = &adev->gfx.compute_ring[i];
5019 pipe = amdgpu_gfx_mec_queue_to_bit(adev,
5020 iring->me,
5021 iring->pipe,
5022 0);
5023 reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
5024 gfx_v9_0_ring_set_pipe_percent(iring, reserve);
5025 }
5026 }
5027
5028 mutex_unlock(&adev->gfx.pipe_reserve_mutex);
5029 }
5030
5031 static void gfx_v9_0_hqd_set_priority(struct amdgpu_device *adev,
5032 struct amdgpu_ring *ring,
5033 bool acquire)
5034 {
5035 uint32_t pipe_priority = acquire ? 0x2 : 0x0;
5036 uint32_t queue_priority = acquire ? 0xf : 0x0;
5037
5038 mutex_lock(&adev->srbm_mutex);
5039 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5040
5041 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PIPE_PRIORITY, pipe_priority);
5042 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_QUEUE_PRIORITY, queue_priority);
5043
5044 soc15_grbm_select(adev, 0, 0, 0, 0);
5045 mutex_unlock(&adev->srbm_mutex);
5046 }
5047
5048 static void gfx_v9_0_ring_set_priority_compute(struct amdgpu_ring *ring,
5049 enum drm_sched_priority priority)
5050 {
5051 struct amdgpu_device *adev = ring->adev;
5052 bool acquire = priority == DRM_SCHED_PRIORITY_HIGH_HW;
5053
5054 if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
5055 return;
5056
5057 gfx_v9_0_hqd_set_priority(adev, ring, acquire);
5058 gfx_v9_0_pipe_reserve_resources(adev, ring, acquire);
5059 }
5060
5061 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5062 {
5063 struct amdgpu_device *adev = ring->adev;
5064
5065 /* XXX check if swapping is necessary on BE */
5066 if (ring->use_doorbell) {
5067 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5068 WDOORBELL64(ring->doorbell_index, ring->wptr);
5069 } else{
5070 BUG(); /* only DOORBELL method supported on gfx9 now */
5071 }
5072 }
5073
5074 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5075 u64 seq, unsigned int flags)
5076 {
5077 struct amdgpu_device *adev = ring->adev;
5078
5079 /* we only allocate 32bit for each seq wb address */
5080 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5081
5082 /* write fence seq to the "addr" */
5083 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5084 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5085 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5086 amdgpu_ring_write(ring, lower_32_bits(addr));
5087 amdgpu_ring_write(ring, upper_32_bits(addr));
5088 amdgpu_ring_write(ring, lower_32_bits(seq));
5089
5090 if (flags & AMDGPU_FENCE_FLAG_INT) {
5091 /* set register to trigger INT */
5092 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5093 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5094 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5095 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5096 amdgpu_ring_write(ring, 0);
5097 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5098 }
5099 }
5100
5101 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5102 {
5103 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5104 amdgpu_ring_write(ring, 0);
5105 }
5106
5107 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5108 {
5109 struct v9_ce_ib_state ce_payload = {0};
5110 uint64_t csa_addr;
5111 int cnt;
5112
5113 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5114 csa_addr = amdgpu_csa_vaddr(ring->adev);
5115
5116 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5117 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5118 WRITE_DATA_DST_SEL(8) |
5119 WR_CONFIRM) |
5120 WRITE_DATA_CACHE_POLICY(0));
5121 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5122 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5123 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5124 }
5125
5126 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5127 {
5128 struct v9_de_ib_state de_payload = {0};
5129 uint64_t csa_addr, gds_addr;
5130 int cnt;
5131
5132 csa_addr = amdgpu_csa_vaddr(ring->adev);
5133 gds_addr = csa_addr + 4096;
5134 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5135 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5136
5137 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5138 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5139 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5140 WRITE_DATA_DST_SEL(8) |
5141 WR_CONFIRM) |
5142 WRITE_DATA_CACHE_POLICY(0));
5143 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5144 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5145 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5146 }
5147
5148 static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
5149 {
5150 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5151 amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
5152 }
5153
5154 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5155 {
5156 uint32_t dw2 = 0;
5157
5158 if (amdgpu_sriov_vf(ring->adev))
5159 gfx_v9_0_ring_emit_ce_meta(ring);
5160
5161 gfx_v9_0_ring_emit_tmz(ring, true);
5162
5163 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5164 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5165 /* set load_global_config & load_global_uconfig */
5166 dw2 |= 0x8001;
5167 /* set load_cs_sh_regs */
5168 dw2 |= 0x01000000;
5169 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5170 dw2 |= 0x10002;
5171
5172 /* set load_ce_ram if preamble presented */
5173 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5174 dw2 |= 0x10000000;
5175 } else {
5176 /* still load_ce_ram if this is the first time preamble presented
5177 * although there is no context switch happens.
5178 */
5179 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5180 dw2 |= 0x10000000;
5181 }
5182
5183 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5184 amdgpu_ring_write(ring, dw2);
5185 amdgpu_ring_write(ring, 0);
5186 }
5187
5188 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5189 {
5190 unsigned ret;
5191 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5192 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5193 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5194 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5195 ret = ring->wptr & ring->buf_mask;
5196 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5197 return ret;
5198 }
5199
5200 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5201 {
5202 unsigned cur;
5203 BUG_ON(offset > ring->buf_mask);
5204 BUG_ON(ring->ring[offset] != 0x55aa55aa);
5205
5206 cur = (ring->wptr & ring->buf_mask) - 1;
5207 if (likely(cur > offset))
5208 ring->ring[offset] = cur - offset;
5209 else
5210 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5211 }
5212
5213 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
5214 {
5215 struct amdgpu_device *adev = ring->adev;
5216
5217 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5218 amdgpu_ring_write(ring, 0 | /* src: register*/
5219 (5 << 8) | /* dst: memory */
5220 (1 << 20)); /* write confirm */
5221 amdgpu_ring_write(ring, reg);
5222 amdgpu_ring_write(ring, 0);
5223 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5224 adev->virt.reg_val_offs * 4));
5225 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5226 adev->virt.reg_val_offs * 4));
5227 }
5228
5229 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5230 uint32_t val)
5231 {
5232 uint32_t cmd = 0;
5233
5234 switch (ring->funcs->type) {
5235 case AMDGPU_RING_TYPE_GFX:
5236 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5237 break;
5238 case AMDGPU_RING_TYPE_KIQ:
5239 cmd = (1 << 16); /* no inc addr */
5240 break;
5241 default:
5242 cmd = WR_CONFIRM;
5243 break;
5244 }
5245 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5246 amdgpu_ring_write(ring, cmd);
5247 amdgpu_ring_write(ring, reg);
5248 amdgpu_ring_write(ring, 0);
5249 amdgpu_ring_write(ring, val);
5250 }
5251
5252 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5253 uint32_t val, uint32_t mask)
5254 {
5255 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5256 }
5257
5258 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5259 uint32_t reg0, uint32_t reg1,
5260 uint32_t ref, uint32_t mask)
5261 {
5262 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5263 struct amdgpu_device *adev = ring->adev;
5264 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5265 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5266
5267 if (fw_version_ok)
5268 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5269 ref, mask, 0x20);
5270 else
5271 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5272 ref, mask);
5273 }
5274
5275 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5276 {
5277 struct amdgpu_device *adev = ring->adev;
5278 uint32_t value = 0;
5279
5280 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5281 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5282 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5283 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5284 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5285 }
5286
5287 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5288 enum amdgpu_interrupt_state state)
5289 {
5290 switch (state) {
5291 case AMDGPU_IRQ_STATE_DISABLE:
5292 case AMDGPU_IRQ_STATE_ENABLE:
5293 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5294 TIME_STAMP_INT_ENABLE,
5295 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5296 break;
5297 default:
5298 break;
5299 }
5300 }
5301
5302 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5303 int me, int pipe,
5304 enum amdgpu_interrupt_state state)
5305 {
5306 u32 mec_int_cntl, mec_int_cntl_reg;
5307
5308 /*
5309 * amdgpu controls only the first MEC. That's why this function only
5310 * handles the setting of interrupts for this specific MEC. All other
5311 * pipes' interrupts are set by amdkfd.
5312 */
5313
5314 if (me == 1) {
5315 switch (pipe) {
5316 case 0:
5317 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5318 break;
5319 case 1:
5320 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5321 break;
5322 case 2:
5323 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5324 break;
5325 case 3:
5326 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5327 break;
5328 default:
5329 DRM_DEBUG("invalid pipe %d\n", pipe);
5330 return;
5331 }
5332 } else {
5333 DRM_DEBUG("invalid me %d\n", me);
5334 return;
5335 }
5336
5337 switch (state) {
5338 case AMDGPU_IRQ_STATE_DISABLE:
5339 mec_int_cntl = RREG32(mec_int_cntl_reg);
5340 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5341 TIME_STAMP_INT_ENABLE, 0);
5342 WREG32(mec_int_cntl_reg, mec_int_cntl);
5343 break;
5344 case AMDGPU_IRQ_STATE_ENABLE:
5345 mec_int_cntl = RREG32(mec_int_cntl_reg);
5346 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5347 TIME_STAMP_INT_ENABLE, 1);
5348 WREG32(mec_int_cntl_reg, mec_int_cntl);
5349 break;
5350 default:
5351 break;
5352 }
5353 }
5354
5355 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5356 struct amdgpu_irq_src *source,
5357 unsigned type,
5358 enum amdgpu_interrupt_state state)
5359 {
5360 switch (state) {
5361 case AMDGPU_IRQ_STATE_DISABLE:
5362 case AMDGPU_IRQ_STATE_ENABLE:
5363 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5364 PRIV_REG_INT_ENABLE,
5365 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5366 break;
5367 default:
5368 break;
5369 }
5370
5371 return 0;
5372 }
5373
5374 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5375 struct amdgpu_irq_src *source,
5376 unsigned type,
5377 enum amdgpu_interrupt_state state)
5378 {
5379 switch (state) {
5380 case AMDGPU_IRQ_STATE_DISABLE:
5381 case AMDGPU_IRQ_STATE_ENABLE:
5382 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5383 PRIV_INSTR_INT_ENABLE,
5384 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5385 default:
5386 break;
5387 }
5388
5389 return 0;
5390 }
5391
5392 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
5393 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5394 CP_ECC_ERROR_INT_ENABLE, 1)
5395
5396 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
5397 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5398 CP_ECC_ERROR_INT_ENABLE, 0)
5399
5400 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5401 struct amdgpu_irq_src *source,
5402 unsigned type,
5403 enum amdgpu_interrupt_state state)
5404 {
5405 switch (state) {
5406 case AMDGPU_IRQ_STATE_DISABLE:
5407 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5408 CP_ECC_ERROR_INT_ENABLE, 0);
5409 DISABLE_ECC_ON_ME_PIPE(1, 0);
5410 DISABLE_ECC_ON_ME_PIPE(1, 1);
5411 DISABLE_ECC_ON_ME_PIPE(1, 2);
5412 DISABLE_ECC_ON_ME_PIPE(1, 3);
5413 break;
5414
5415 case AMDGPU_IRQ_STATE_ENABLE:
5416 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5417 CP_ECC_ERROR_INT_ENABLE, 1);
5418 ENABLE_ECC_ON_ME_PIPE(1, 0);
5419 ENABLE_ECC_ON_ME_PIPE(1, 1);
5420 ENABLE_ECC_ON_ME_PIPE(1, 2);
5421 ENABLE_ECC_ON_ME_PIPE(1, 3);
5422 break;
5423 default:
5424 break;
5425 }
5426
5427 return 0;
5428 }
5429
5430
5431 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5432 struct amdgpu_irq_src *src,
5433 unsigned type,
5434 enum amdgpu_interrupt_state state)
5435 {
5436 switch (type) {
5437 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5438 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5439 break;
5440 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5441 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5442 break;
5443 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5444 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5445 break;
5446 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5447 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5448 break;
5449 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5450 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5451 break;
5452 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5453 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5454 break;
5455 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5456 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5457 break;
5458 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5459 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5460 break;
5461 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5462 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5463 break;
5464 default:
5465 break;
5466 }
5467 return 0;
5468 }
5469
5470 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5471 struct amdgpu_irq_src *source,
5472 struct amdgpu_iv_entry *entry)
5473 {
5474 int i;
5475 u8 me_id, pipe_id, queue_id;
5476 struct amdgpu_ring *ring;
5477
5478 DRM_DEBUG("IH: CP EOP\n");
5479 me_id = (entry->ring_id & 0x0c) >> 2;
5480 pipe_id = (entry->ring_id & 0x03) >> 0;
5481 queue_id = (entry->ring_id & 0x70) >> 4;
5482
5483 switch (me_id) {
5484 case 0:
5485 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5486 break;
5487 case 1:
5488 case 2:
5489 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5490 ring = &adev->gfx.compute_ring[i];
5491 /* Per-queue interrupt is supported for MEC starting from VI.
5492 * The interrupt can only be enabled/disabled per pipe instead of per queue.
5493 */
5494 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5495 amdgpu_fence_process(ring);
5496 }
5497 break;
5498 }
5499 return 0;
5500 }
5501
5502 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5503 struct amdgpu_iv_entry *entry)
5504 {
5505 u8 me_id, pipe_id, queue_id;
5506 struct amdgpu_ring *ring;
5507 int i;
5508
5509 me_id = (entry->ring_id & 0x0c) >> 2;
5510 pipe_id = (entry->ring_id & 0x03) >> 0;
5511 queue_id = (entry->ring_id & 0x70) >> 4;
5512
5513 switch (me_id) {
5514 case 0:
5515 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5516 break;
5517 case 1:
5518 case 2:
5519 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5520 ring = &adev->gfx.compute_ring[i];
5521 if (ring->me == me_id && ring->pipe == pipe_id &&
5522 ring->queue == queue_id)
5523 drm_sched_fault(&ring->sched);
5524 }
5525 break;
5526 }
5527 }
5528
5529 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5530 struct amdgpu_irq_src *source,
5531 struct amdgpu_iv_entry *entry)
5532 {
5533 DRM_ERROR("Illegal register access in command stream\n");
5534 gfx_v9_0_fault(adev, entry);
5535 return 0;
5536 }
5537
5538 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5539 struct amdgpu_irq_src *source,
5540 struct amdgpu_iv_entry *entry)
5541 {
5542 DRM_ERROR("Illegal instruction in command stream\n");
5543 gfx_v9_0_fault(adev, entry);
5544 return 0;
5545 }
5546
5547
5548 static const struct soc15_ras_field_entry gc_ras_fields_vg20[] = {
5549 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5550 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5551 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5552 },
5553 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5554 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5555 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5556 },
5557 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5558 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5559 0, 0
5560 },
5561 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5562 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5563 0, 0
5564 },
5565 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5566 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5567 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5568 },
5569 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5570 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5571 0, 0
5572 },
5573 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5574 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5575 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5576 },
5577 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5578 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5579 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5580 },
5581 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5582 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5583 0, 0
5584 },
5585 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5586 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5587 0, 0
5588 },
5589 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5590 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5591 0, 0
5592 },
5593 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5594 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5595 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5596 },
5597 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5598 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5599 0, 0
5600 },
5601 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5602 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5603 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5604 },
5605 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5606 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5607 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5608 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5609 },
5610 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5611 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5612 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5613 0, 0
5614 },
5615 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
5616 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5617 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
5618 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
5619 },
5620 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
5621 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5622 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
5623 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
5624 },
5625 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
5626 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5627 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
5628 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
5629 },
5630 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
5631 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5632 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
5633 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
5634 },
5635 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
5636 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
5637 0, 0
5638 },
5639 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5640 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
5641 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
5642 },
5643 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5644 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
5645 0, 0
5646 },
5647 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5648 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
5649 0, 0
5650 },
5651 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5652 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
5653 0, 0
5654 },
5655 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5656 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
5657 0, 0
5658 },
5659 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5660 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
5661 0, 0
5662 },
5663 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5664 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
5665 0, 0
5666 },
5667 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5668 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
5669 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
5670 },
5671 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5672 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
5673 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
5674 },
5675 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5676 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
5677 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
5678 },
5679 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5680 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
5681 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
5682 },
5683 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5684 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
5685 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
5686 },
5687 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5688 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
5689 0, 0
5690 },
5691 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5692 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
5693 0, 0
5694 },
5695 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5696 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
5697 0, 0
5698 },
5699 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5700 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
5701 0, 0
5702 },
5703 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5704 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
5705 0, 0
5706 },
5707 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5708 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
5709 0, 0
5710 },
5711 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5712 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
5713 0, 0
5714 },
5715 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5716 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
5717 0, 0
5718 },
5719 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5720 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
5721 0, 0
5722 },
5723 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5724 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
5725 0, 0
5726 },
5727 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5728 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
5729 0, 0
5730 },
5731 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5732 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
5733 0, 0
5734 },
5735 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5736 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
5737 0, 0
5738 },
5739 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
5740 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
5741 0, 0
5742 },
5743 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5744 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
5745 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
5746 },
5747 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5748 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
5749 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
5750 },
5751 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5752 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
5753 0, 0
5754 },
5755 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5756 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
5757 0, 0
5758 },
5759 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5760 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
5761 0, 0
5762 },
5763 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5764 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
5765 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
5766 },
5767 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
5768 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
5769 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
5770 },
5771 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
5772 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
5773 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
5774 },
5775 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
5776 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
5777 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
5778 },
5779 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
5780 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
5781 0, 0
5782 },
5783 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5784 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
5785 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
5786 },
5787 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5788 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
5789 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
5790 },
5791 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5792 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
5793 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
5794 },
5795 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5796 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
5797 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
5798 },
5799 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5800 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
5801 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
5802 },
5803 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5804 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
5805 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
5806 },
5807 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
5808 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
5809 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
5810 },
5811 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
5812 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
5813 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
5814 },
5815 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
5816 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
5817 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
5818 },
5819 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
5820 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
5821 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
5822 },
5823 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
5824 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
5825 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
5826 },
5827 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
5828 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
5829 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
5830 },
5831 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
5832 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
5833 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
5834 },
5835 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5836 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
5837 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
5838 },
5839 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5840 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
5841 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
5842 },
5843 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5844 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
5845 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
5846 },
5847 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5848 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
5849 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
5850 },
5851 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5852 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
5853 0, 0
5854 },
5855 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5856 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
5857 0, 0
5858 },
5859 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5860 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
5861 0, 0
5862 },
5863 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5864 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
5865 0, 0
5866 },
5867 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5868 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
5869 0, 0
5870 },
5871 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
5872 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
5873 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
5874 },
5875 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5876 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
5877 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
5878 },
5879 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5880 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
5881 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
5882 },
5883 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5884 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
5885 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
5886 },
5887 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5888 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
5889 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
5890 },
5891 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5892 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
5893 0, 0
5894 },
5895 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5896 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
5897 0, 0
5898 },
5899 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5900 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
5901 0, 0
5902 },
5903 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5904 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
5905 0, 0
5906 },
5907 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
5908 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
5909 0, 0
5910 },
5911 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5912 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
5913 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
5914 },
5915 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5916 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
5917 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
5918 },
5919 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5920 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
5921 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
5922 },
5923 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5924 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
5925 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
5926 },
5927 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5928 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
5929 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
5930 },
5931 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5932 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
5933 0, 0
5934 },
5935 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5936 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
5937 0, 0
5938 },
5939 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5940 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
5941 0, 0
5942 },
5943 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5944 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
5945 0, 0
5946 },
5947 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
5948 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
5949 0, 0
5950 },
5951 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5952 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
5953 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
5954 },
5955 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5956 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
5957 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
5958 },
5959 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5960 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
5961 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
5962 },
5963 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5964 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
5965 0, 0
5966 },
5967 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5968 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
5969 0, 0
5970 },
5971 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5972 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
5973 0, 0
5974 },
5975 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5976 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
5977 0, 0
5978 },
5979 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5980 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
5981 0, 0
5982 },
5983 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
5984 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
5985 0, 0
5986 }
5987 };
5988
5989 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
5990 void *inject_if)
5991 {
5992 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
5993 int ret;
5994 struct ta_ras_trigger_error_input block_info = { 0 };
5995
5996 if (adev->asic_type != CHIP_VEGA20)
5997 return -EINVAL;
5998
5999 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6000 return -EINVAL;
6001
6002 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6003 return -EPERM;
6004
6005 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6006 info->head.type)) {
6007 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6008 ras_gfx_subblocks[info->head.sub_block_index].name,
6009 info->head.type);
6010 return -EPERM;
6011 }
6012
6013 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6014 info->head.type)) {
6015 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6016 ras_gfx_subblocks[info->head.sub_block_index].name,
6017 info->head.type);
6018 return -EPERM;
6019 }
6020
6021 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6022 block_info.sub_block_index =
6023 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6024 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6025 block_info.address = info->address;
6026 block_info.value = info->value;
6027
6028 mutex_lock(&adev->grbm_idx_mutex);
6029 ret = psp_ras_trigger_error(&adev->psp, &block_info);
6030 mutex_unlock(&adev->grbm_idx_mutex);
6031
6032 return ret;
6033 }
6034
6035 static const char *vml2_mems[] = {
6036 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6037 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6038 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6039 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6040 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6041 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6042 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6043 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6044 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6045 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6046 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6047 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6048 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6049 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6050 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6051 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6052 };
6053
6054 static const char *vml2_walker_mems[] = {
6055 "UTC_VML2_CACHE_PDE0_MEM0",
6056 "UTC_VML2_CACHE_PDE0_MEM1",
6057 "UTC_VML2_CACHE_PDE1_MEM0",
6058 "UTC_VML2_CACHE_PDE1_MEM1",
6059 "UTC_VML2_CACHE_PDE2_MEM0",
6060 "UTC_VML2_CACHE_PDE2_MEM1",
6061 "UTC_VML2_RDIF_LOG_FIFO",
6062 };
6063
6064 static const char *atc_l2_cache_2m_mems[] = {
6065 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6066 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6067 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6068 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6069 };
6070
6071 static const char *atc_l2_cache_4k_mems[] = {
6072 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6073 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6074 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6075 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6076 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6077 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6078 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6079 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6080 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6081 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6082 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6083 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6084 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6085 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6086 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6087 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6088 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6089 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6090 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6091 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6092 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6093 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6094 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6095 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6096 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6097 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6098 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6099 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6100 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6101 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6102 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6103 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6104 };
6105
6106 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6107 struct ras_err_data *err_data)
6108 {
6109 uint32_t i, data;
6110 uint32_t sec_count, ded_count;
6111
6112 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6113 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6114 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6115 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6116 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6117 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6118 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6119 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6120
6121 for (i = 0; i < 16; i++) {
6122 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6123 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6124
6125 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6126 if (sec_count) {
6127 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6128 vml2_mems[i], sec_count);
6129 err_data->ce_count += sec_count;
6130 }
6131
6132 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6133 if (ded_count) {
6134 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i,
6135 vml2_mems[i], ded_count);
6136 err_data->ue_count += ded_count;
6137 }
6138 }
6139
6140 for (i = 0; i < 7; i++) {
6141 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6142 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6143
6144 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6145 SEC_COUNT);
6146 if (sec_count) {
6147 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6148 vml2_walker_mems[i], sec_count);
6149 err_data->ce_count += sec_count;
6150 }
6151
6152 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6153 DED_COUNT);
6154 if (ded_count) {
6155 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i,
6156 vml2_walker_mems[i], ded_count);
6157 err_data->ue_count += ded_count;
6158 }
6159 }
6160
6161 for (i = 0; i < 4; i++) {
6162 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6163 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6164
6165 sec_count = (data & 0x00006000L) >> 0xd;
6166 if (sec_count) {
6167 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6168 atc_l2_cache_2m_mems[i], sec_count);
6169 err_data->ce_count += sec_count;
6170 }
6171 }
6172
6173 for (i = 0; i < 32; i++) {
6174 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6175 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6176
6177 sec_count = (data & 0x00006000L) >> 0xd;
6178 if (sec_count) {
6179 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6180 atc_l2_cache_4k_mems[i], sec_count);
6181 err_data->ce_count += sec_count;
6182 }
6183
6184 ded_count = (data & 0x00018000L) >> 0xf;
6185 if (ded_count) {
6186 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i,
6187 atc_l2_cache_4k_mems[i], ded_count);
6188 err_data->ue_count += ded_count;
6189 }
6190 }
6191
6192 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6193 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6194 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6195 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6196
6197 return 0;
6198 }
6199
6200 static int __get_ras_error_count(const struct soc15_reg_entry *reg,
6201 uint32_t se_id, uint32_t inst_id, uint32_t value,
6202 uint32_t *sec_count, uint32_t *ded_count)
6203 {
6204 uint32_t i;
6205 uint32_t sec_cnt, ded_cnt;
6206
6207 for (i = 0; i < ARRAY_SIZE(gc_ras_fields_vg20); i++) {
6208 if(gc_ras_fields_vg20[i].reg_offset != reg->reg_offset ||
6209 gc_ras_fields_vg20[i].seg != reg->seg ||
6210 gc_ras_fields_vg20[i].inst != reg->inst)
6211 continue;
6212
6213 sec_cnt = (value &
6214 gc_ras_fields_vg20[i].sec_count_mask) >>
6215 gc_ras_fields_vg20[i].sec_count_shift;
6216 if (sec_cnt) {
6217 DRM_INFO("GFX SubBlock %s, Instance[%d][%d], SEC %d\n",
6218 gc_ras_fields_vg20[i].name,
6219 se_id, inst_id,
6220 sec_cnt);
6221 *sec_count += sec_cnt;
6222 }
6223
6224 ded_cnt = (value &
6225 gc_ras_fields_vg20[i].ded_count_mask) >>
6226 gc_ras_fields_vg20[i].ded_count_shift;
6227 if (ded_cnt) {
6228 DRM_INFO("GFX SubBlock %s, Instance[%d][%d], DED %d\n",
6229 gc_ras_fields_vg20[i].name,
6230 se_id, inst_id,
6231 ded_cnt);
6232 *ded_count += ded_cnt;
6233 }
6234 }
6235
6236 return 0;
6237 }
6238
6239 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6240 void *ras_error_status)
6241 {
6242 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6243 uint32_t sec_count = 0, ded_count = 0;
6244 uint32_t i, j, k;
6245 uint32_t reg_value;
6246
6247 if (adev->asic_type != CHIP_VEGA20)
6248 return -EINVAL;
6249
6250 err_data->ue_count = 0;
6251 err_data->ce_count = 0;
6252
6253 mutex_lock(&adev->grbm_idx_mutex);
6254
6255 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) {
6256 for (j = 0; j < sec_ded_counter_registers[i].se_num; j++) {
6257 for (k = 0; k < sec_ded_counter_registers[i].instance; k++) {
6258 gfx_v9_0_select_se_sh(adev, j, 0, k);
6259 reg_value =
6260 RREG32(SOC15_REG_ENTRY_OFFSET(sec_ded_counter_registers[i]));
6261 if (reg_value)
6262 __get_ras_error_count(&sec_ded_counter_registers[i],
6263 j, k, reg_value,
6264 &sec_count, &ded_count);
6265 }
6266 }
6267 }
6268
6269 err_data->ce_count += sec_count;
6270 err_data->ue_count += ded_count;
6271
6272 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6273 mutex_unlock(&adev->grbm_idx_mutex);
6274
6275 gfx_v9_0_query_utc_edc_status(adev, err_data);
6276
6277 return 0;
6278 }
6279
6280 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6281 .name = "gfx_v9_0",
6282 .early_init = gfx_v9_0_early_init,
6283 .late_init = gfx_v9_0_late_init,
6284 .sw_init = gfx_v9_0_sw_init,
6285 .sw_fini = gfx_v9_0_sw_fini,
6286 .hw_init = gfx_v9_0_hw_init,
6287 .hw_fini = gfx_v9_0_hw_fini,
6288 .suspend = gfx_v9_0_suspend,
6289 .resume = gfx_v9_0_resume,
6290 .is_idle = gfx_v9_0_is_idle,
6291 .wait_for_idle = gfx_v9_0_wait_for_idle,
6292 .soft_reset = gfx_v9_0_soft_reset,
6293 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6294 .set_powergating_state = gfx_v9_0_set_powergating_state,
6295 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6296 };
6297
6298 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6299 .type = AMDGPU_RING_TYPE_GFX,
6300 .align_mask = 0xff,
6301 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6302 .support_64bit_ptrs = true,
6303 .vmhub = AMDGPU_GFXHUB_0,
6304 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6305 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6306 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6307 .emit_frame_size = /* totally 242 maximum if 16 IBs */
6308 5 + /* COND_EXEC */
6309 7 + /* PIPELINE_SYNC */
6310 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6311 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6312 2 + /* VM_FLUSH */
6313 8 + /* FENCE for VM_FLUSH */
6314 20 + /* GDS switch */
6315 4 + /* double SWITCH_BUFFER,
6316 the first COND_EXEC jump to the place just
6317 prior to this double SWITCH_BUFFER */
6318 5 + /* COND_EXEC */
6319 7 + /* HDP_flush */
6320 4 + /* VGT_flush */
6321 14 + /* CE_META */
6322 31 + /* DE_META */
6323 3 + /* CNTX_CTRL */
6324 5 + /* HDP_INVL */
6325 8 + 8 + /* FENCE x2 */
6326 2, /* SWITCH_BUFFER */
6327 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6328 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6329 .emit_fence = gfx_v9_0_ring_emit_fence,
6330 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6331 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6332 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6333 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6334 .test_ring = gfx_v9_0_ring_test_ring,
6335 .test_ib = gfx_v9_0_ring_test_ib,
6336 .insert_nop = amdgpu_ring_insert_nop,
6337 .pad_ib = amdgpu_ring_generic_pad_ib,
6338 .emit_switch_buffer = gfx_v9_ring_emit_sb,
6339 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6340 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6341 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6342 .emit_tmz = gfx_v9_0_ring_emit_tmz,
6343 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6344 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6345 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6346 .soft_recovery = gfx_v9_0_ring_soft_recovery,
6347 };
6348
6349 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6350 .type = AMDGPU_RING_TYPE_COMPUTE,
6351 .align_mask = 0xff,
6352 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6353 .support_64bit_ptrs = true,
6354 .vmhub = AMDGPU_GFXHUB_0,
6355 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6356 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6357 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6358 .emit_frame_size =
6359 20 + /* gfx_v9_0_ring_emit_gds_switch */
6360 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6361 5 + /* hdp invalidate */
6362 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6363 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6364 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6365 2 + /* gfx_v9_0_ring_emit_vm_flush */
6366 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6367 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6368 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6369 .emit_fence = gfx_v9_0_ring_emit_fence,
6370 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6371 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6372 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6373 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6374 .test_ring = gfx_v9_0_ring_test_ring,
6375 .test_ib = gfx_v9_0_ring_test_ib,
6376 .insert_nop = amdgpu_ring_insert_nop,
6377 .pad_ib = amdgpu_ring_generic_pad_ib,
6378 .set_priority = gfx_v9_0_ring_set_priority_compute,
6379 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6380 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6381 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6382 };
6383
6384 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6385 .type = AMDGPU_RING_TYPE_KIQ,
6386 .align_mask = 0xff,
6387 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6388 .support_64bit_ptrs = true,
6389 .vmhub = AMDGPU_GFXHUB_0,
6390 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6391 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6392 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6393 .emit_frame_size =
6394 20 + /* gfx_v9_0_ring_emit_gds_switch */
6395 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6396 5 + /* hdp invalidate */
6397 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6398 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6399 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6400 2 + /* gfx_v9_0_ring_emit_vm_flush */
6401 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6402 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6403 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6404 .test_ring = gfx_v9_0_ring_test_ring,
6405 .insert_nop = amdgpu_ring_insert_nop,
6406 .pad_ib = amdgpu_ring_generic_pad_ib,
6407 .emit_rreg = gfx_v9_0_ring_emit_rreg,
6408 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6409 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6410 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6411 };
6412
6413 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6414 {
6415 int i;
6416
6417 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6418
6419 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6420 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6421
6422 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6423 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6424 }
6425
6426 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6427 .set = gfx_v9_0_set_eop_interrupt_state,
6428 .process = gfx_v9_0_eop_irq,
6429 };
6430
6431 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6432 .set = gfx_v9_0_set_priv_reg_fault_state,
6433 .process = gfx_v9_0_priv_reg_irq,
6434 };
6435
6436 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6437 .set = gfx_v9_0_set_priv_inst_fault_state,
6438 .process = gfx_v9_0_priv_inst_irq,
6439 };
6440
6441 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6442 .set = gfx_v9_0_set_cp_ecc_error_state,
6443 .process = amdgpu_gfx_cp_ecc_error_irq,
6444 };
6445
6446
6447 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6448 {
6449 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6450 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6451
6452 adev->gfx.priv_reg_irq.num_types = 1;
6453 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6454
6455 adev->gfx.priv_inst_irq.num_types = 1;
6456 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6457
6458 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6459 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6460 }
6461
6462 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6463 {
6464 switch (adev->asic_type) {
6465 case CHIP_VEGA10:
6466 case CHIP_VEGA12:
6467 case CHIP_VEGA20:
6468 case CHIP_RAVEN:
6469 case CHIP_ARCTURUS:
6470 case CHIP_RENOIR:
6471 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
6472 break;
6473 default:
6474 break;
6475 }
6476 }
6477
6478 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
6479 {
6480 /* init asci gds info */
6481 switch (adev->asic_type) {
6482 case CHIP_VEGA10:
6483 case CHIP_VEGA12:
6484 case CHIP_VEGA20:
6485 adev->gds.gds_size = 0x10000;
6486 break;
6487 case CHIP_RAVEN:
6488 case CHIP_ARCTURUS:
6489 adev->gds.gds_size = 0x1000;
6490 break;
6491 default:
6492 adev->gds.gds_size = 0x10000;
6493 break;
6494 }
6495
6496 switch (adev->asic_type) {
6497 case CHIP_VEGA10:
6498 case CHIP_VEGA20:
6499 adev->gds.gds_compute_max_wave_id = 0x7ff;
6500 break;
6501 case CHIP_VEGA12:
6502 adev->gds.gds_compute_max_wave_id = 0x27f;
6503 break;
6504 case CHIP_RAVEN:
6505 if (adev->rev_id >= 0x8)
6506 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
6507 else
6508 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
6509 break;
6510 case CHIP_ARCTURUS:
6511 adev->gds.gds_compute_max_wave_id = 0xfff;
6512 break;
6513 default:
6514 /* this really depends on the chip */
6515 adev->gds.gds_compute_max_wave_id = 0x7ff;
6516 break;
6517 }
6518
6519 adev->gds.gws_size = 64;
6520 adev->gds.oa_size = 16;
6521 }
6522
6523 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
6524 u32 bitmap)
6525 {
6526 u32 data;
6527
6528 if (!bitmap)
6529 return;
6530
6531 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6532 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6533
6534 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
6535 }
6536
6537 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
6538 {
6539 u32 data, mask;
6540
6541 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
6542 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
6543
6544 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6545 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6546
6547 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
6548
6549 return (~data) & mask;
6550 }
6551
6552 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
6553 struct amdgpu_cu_info *cu_info)
6554 {
6555 int i, j, k, counter, active_cu_number = 0;
6556 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
6557 unsigned disable_masks[4 * 4];
6558
6559 if (!adev || !cu_info)
6560 return -EINVAL;
6561
6562 /*
6563 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
6564 */
6565 if (adev->gfx.config.max_shader_engines *
6566 adev->gfx.config.max_sh_per_se > 16)
6567 return -EINVAL;
6568
6569 amdgpu_gfx_parse_disable_cu(disable_masks,
6570 adev->gfx.config.max_shader_engines,
6571 adev->gfx.config.max_sh_per_se);
6572
6573 mutex_lock(&adev->grbm_idx_mutex);
6574 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6575 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6576 mask = 1;
6577 ao_bitmap = 0;
6578 counter = 0;
6579 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
6580 gfx_v9_0_set_user_cu_inactive_bitmap(
6581 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
6582 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
6583
6584 /*
6585 * The bitmap(and ao_cu_bitmap) in cu_info structure is
6586 * 4x4 size array, and it's usually suitable for Vega
6587 * ASICs which has 4*2 SE/SH layout.
6588 * But for Arcturus, SE/SH layout is changed to 8*1.
6589 * To mostly reduce the impact, we make it compatible
6590 * with current bitmap array as below:
6591 * SE4,SH0 --> bitmap[0][1]
6592 * SE5,SH0 --> bitmap[1][1]
6593 * SE6,SH0 --> bitmap[2][1]
6594 * SE7,SH0 --> bitmap[3][1]
6595 */
6596 cu_info->bitmap[i % 4][j + i / 4] = bitmap;
6597
6598 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
6599 if (bitmap & mask) {
6600 if (counter < adev->gfx.config.max_cu_per_sh)
6601 ao_bitmap |= mask;
6602 counter ++;
6603 }
6604 mask <<= 1;
6605 }
6606 active_cu_number += counter;
6607 if (i < 2 && j < 2)
6608 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
6609 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
6610 }
6611 }
6612 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6613 mutex_unlock(&adev->grbm_idx_mutex);
6614
6615 cu_info->number = active_cu_number;
6616 cu_info->ao_cu_mask = ao_cu_mask;
6617 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6618
6619 return 0;
6620 }
6621
6622 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
6623 {
6624 .type = AMD_IP_BLOCK_TYPE_GFX,
6625 .major = 9,
6626 .minor = 0,
6627 .rev = 0,
6628 .funcs = &gfx_v9_0_ip_funcs,
6629 };
Linux with added FPC-III support