2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include "kfd_kernel_queue.h"
25 #include "kfd_device_queue_manager.h"
26 #include "kfd_pm4_headers_vi.h"
27 #include "kfd_pm4_opcodes.h"
29 unsigned int pm_build_pm4_header(unsigned int opcode
, size_t packet_size
)
31 union PM4_MES_TYPE_3_HEADER header
;
34 header
.opcode
= opcode
;
35 header
.count
= packet_size
/ 4 - 2;
36 header
.type
= PM4_TYPE_3
;
41 static int pm_map_process_vi(struct packet_manager
*pm
, uint32_t *buffer
,
42 struct qcm_process_device
*qpd
)
44 struct pm4_mes_map_process
*packet
;
46 packet
= (struct pm4_mes_map_process
*)buffer
;
48 memset(buffer
, 0, sizeof(struct pm4_mes_map_process
));
50 packet
->header
.u32All
= pm_build_pm4_header(IT_MAP_PROCESS
,
51 sizeof(struct pm4_mes_map_process
));
52 packet
->bitfields2
.diq_enable
= (qpd
->is_debug
) ? 1 : 0;
53 packet
->bitfields2
.process_quantum
= 10;
54 packet
->bitfields2
.pasid
= qpd
->pqm
->process
->pasid
;
55 packet
->bitfields3
.page_table_base
= qpd
->page_table_base
;
56 packet
->bitfields10
.gds_size
= qpd
->gds_size
;
57 packet
->bitfields10
.num_gws
= qpd
->num_gws
;
58 packet
->bitfields10
.num_oac
= qpd
->num_oac
;
59 packet
->bitfields10
.num_queues
= (qpd
->is_debug
) ? 0 : qpd
->queue_count
;
61 packet
->sh_mem_config
= qpd
->sh_mem_config
;
62 packet
->sh_mem_bases
= qpd
->sh_mem_bases
;
63 packet
->sh_mem_ape1_base
= qpd
->sh_mem_ape1_base
;
64 packet
->sh_mem_ape1_limit
= qpd
->sh_mem_ape1_limit
;
66 packet
->sh_hidden_private_base_vmid
= qpd
->sh_hidden_private_base
;
68 packet
->gds_addr_lo
= lower_32_bits(qpd
->gds_context_area
);
69 packet
->gds_addr_hi
= upper_32_bits(qpd
->gds_context_area
);
74 static int pm_runlist_vi(struct packet_manager
*pm
, uint32_t *buffer
,
75 uint64_t ib
, size_t ib_size_in_dwords
, bool chain
)
77 struct pm4_mes_runlist
*packet
;
78 int concurrent_proc_cnt
= 0;
79 struct kfd_dev
*kfd
= pm
->dqm
->dev
;
84 /* Determine the number of processes to map together to HW:
85 * it can not exceed the number of VMIDs available to the
86 * scheduler, and it is determined by the smaller of the number
87 * of processes in the runlist and kfd module parameter
89 * Note: the arbitration between the number of VMIDs and
90 * hws_max_conc_proc has been done in
91 * kgd2kfd_device_init().
93 concurrent_proc_cnt
= min(pm
->dqm
->processes_count
,
94 kfd
->max_proc_per_quantum
);
96 packet
= (struct pm4_mes_runlist
*)buffer
;
98 memset(buffer
, 0, sizeof(struct pm4_mes_runlist
));
99 packet
->header
.u32All
= pm_build_pm4_header(IT_RUN_LIST
,
100 sizeof(struct pm4_mes_runlist
));
102 packet
->bitfields4
.ib_size
= ib_size_in_dwords
;
103 packet
->bitfields4
.chain
= chain
? 1 : 0;
104 packet
->bitfields4
.offload_polling
= 0;
105 packet
->bitfields4
.valid
= 1;
106 packet
->bitfields4
.process_cnt
= concurrent_proc_cnt
;
107 packet
->ordinal2
= lower_32_bits(ib
);
108 packet
->bitfields3
.ib_base_hi
= upper_32_bits(ib
);
113 int pm_set_resources_vi(struct packet_manager
*pm
, uint32_t *buffer
,
114 struct scheduling_resources
*res
)
116 struct pm4_mes_set_resources
*packet
;
118 packet
= (struct pm4_mes_set_resources
*)buffer
;
119 memset(buffer
, 0, sizeof(struct pm4_mes_set_resources
));
121 packet
->header
.u32All
= pm_build_pm4_header(IT_SET_RESOURCES
,
122 sizeof(struct pm4_mes_set_resources
));
124 packet
->bitfields2
.queue_type
=
125 queue_type__mes_set_resources__hsa_interface_queue_hiq
;
126 packet
->bitfields2
.vmid_mask
= res
->vmid_mask
;
127 packet
->bitfields2
.unmap_latency
= KFD_UNMAP_LATENCY_MS
/ 100;
128 packet
->bitfields7
.oac_mask
= res
->oac_mask
;
129 packet
->bitfields8
.gds_heap_base
= res
->gds_heap_base
;
130 packet
->bitfields8
.gds_heap_size
= res
->gds_heap_size
;
132 packet
->gws_mask_lo
= lower_32_bits(res
->gws_mask
);
133 packet
->gws_mask_hi
= upper_32_bits(res
->gws_mask
);
135 packet
->queue_mask_lo
= lower_32_bits(res
->queue_mask
);
136 packet
->queue_mask_hi
= upper_32_bits(res
->queue_mask
);
141 static int pm_map_queues_vi(struct packet_manager
*pm
, uint32_t *buffer
,
142 struct queue
*q
, bool is_static
)
144 struct pm4_mes_map_queues
*packet
;
145 bool use_static
= is_static
;
147 packet
= (struct pm4_mes_map_queues
*)buffer
;
148 memset(buffer
, 0, sizeof(struct pm4_mes_map_queues
));
150 packet
->header
.u32All
= pm_build_pm4_header(IT_MAP_QUEUES
,
151 sizeof(struct pm4_mes_map_queues
));
152 packet
->bitfields2
.num_queues
= 1;
153 packet
->bitfields2
.queue_sel
=
154 queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi
;
156 packet
->bitfields2
.engine_sel
=
157 engine_sel__mes_map_queues__compute_vi
;
158 packet
->bitfields2
.queue_type
=
159 queue_type__mes_map_queues__normal_compute_vi
;
161 switch (q
->properties
.type
) {
162 case KFD_QUEUE_TYPE_COMPUTE
:
164 packet
->bitfields2
.queue_type
=
165 queue_type__mes_map_queues__normal_latency_static_queue_vi
;
167 case KFD_QUEUE_TYPE_DIQ
:
168 packet
->bitfields2
.queue_type
=
169 queue_type__mes_map_queues__debug_interface_queue_vi
;
171 case KFD_QUEUE_TYPE_SDMA
:
172 case KFD_QUEUE_TYPE_SDMA_XGMI
:
173 packet
->bitfields2
.engine_sel
= q
->properties
.sdma_engine_id
+
174 engine_sel__mes_map_queues__sdma0_vi
;
175 use_static
= false; /* no static queues under SDMA */
178 WARN(1, "queue type %d", q
->properties
.type
);
181 packet
->bitfields3
.doorbell_offset
=
182 q
->properties
.doorbell_off
;
184 packet
->mqd_addr_lo
=
185 lower_32_bits(q
->gart_mqd_addr
);
187 packet
->mqd_addr_hi
=
188 upper_32_bits(q
->gart_mqd_addr
);
190 packet
->wptr_addr_lo
=
191 lower_32_bits((uint64_t)q
->properties
.write_ptr
);
193 packet
->wptr_addr_hi
=
194 upper_32_bits((uint64_t)q
->properties
.write_ptr
);
199 static int pm_unmap_queues_vi(struct packet_manager
*pm
, uint32_t *buffer
,
200 enum kfd_queue_type type
,
201 enum kfd_unmap_queues_filter filter
,
202 uint32_t filter_param
, bool reset
,
203 unsigned int sdma_engine
)
205 struct pm4_mes_unmap_queues
*packet
;
207 packet
= (struct pm4_mes_unmap_queues
*)buffer
;
208 memset(buffer
, 0, sizeof(struct pm4_mes_unmap_queues
));
210 packet
->header
.u32All
= pm_build_pm4_header(IT_UNMAP_QUEUES
,
211 sizeof(struct pm4_mes_unmap_queues
));
213 case KFD_QUEUE_TYPE_COMPUTE
:
214 case KFD_QUEUE_TYPE_DIQ
:
215 packet
->bitfields2
.engine_sel
=
216 engine_sel__mes_unmap_queues__compute
;
218 case KFD_QUEUE_TYPE_SDMA
:
219 case KFD_QUEUE_TYPE_SDMA_XGMI
:
220 packet
->bitfields2
.engine_sel
=
221 engine_sel__mes_unmap_queues__sdma0
+ sdma_engine
;
224 WARN(1, "queue type %d", type
);
229 packet
->bitfields2
.action
=
230 action__mes_unmap_queues__reset_queues
;
232 packet
->bitfields2
.action
=
233 action__mes_unmap_queues__preempt_queues
;
236 case KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE
:
237 packet
->bitfields2
.queue_sel
=
238 queue_sel__mes_unmap_queues__perform_request_on_specified_queues
;
239 packet
->bitfields2
.num_queues
= 1;
240 packet
->bitfields3b
.doorbell_offset0
= filter_param
;
242 case KFD_UNMAP_QUEUES_FILTER_BY_PASID
:
243 packet
->bitfields2
.queue_sel
=
244 queue_sel__mes_unmap_queues__perform_request_on_pasid_queues
;
245 packet
->bitfields3a
.pasid
= filter_param
;
247 case KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES
:
248 packet
->bitfields2
.queue_sel
=
249 queue_sel__mes_unmap_queues__unmap_all_queues
;
251 case KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES
:
252 /* in this case, we do not preempt static queues */
253 packet
->bitfields2
.queue_sel
=
254 queue_sel__mes_unmap_queues__unmap_all_non_static_queues
;
257 WARN(1, "filter %d", filter
);
265 static int pm_query_status_vi(struct packet_manager
*pm
, uint32_t *buffer
,
266 uint64_t fence_address
, uint32_t fence_value
)
268 struct pm4_mes_query_status
*packet
;
270 packet
= (struct pm4_mes_query_status
*)buffer
;
271 memset(buffer
, 0, sizeof(struct pm4_mes_query_status
));
273 packet
->header
.u32All
= pm_build_pm4_header(IT_QUERY_STATUS
,
274 sizeof(struct pm4_mes_query_status
));
276 packet
->bitfields2
.context_id
= 0;
277 packet
->bitfields2
.interrupt_sel
=
278 interrupt_sel__mes_query_status__completion_status
;
279 packet
->bitfields2
.command
=
280 command__mes_query_status__fence_only_after_write_ack
;
282 packet
->addr_hi
= upper_32_bits((uint64_t)fence_address
);
283 packet
->addr_lo
= lower_32_bits((uint64_t)fence_address
);
284 packet
->data_hi
= upper_32_bits((uint64_t)fence_value
);
285 packet
->data_lo
= lower_32_bits((uint64_t)fence_value
);
290 static int pm_release_mem_vi(uint64_t gpu_addr
, uint32_t *buffer
)
292 struct pm4_mec_release_mem
*packet
;
294 packet
= (struct pm4_mec_release_mem
*)buffer
;
295 memset(buffer
, 0, sizeof(*packet
));
297 packet
->header
.u32All
= pm_build_pm4_header(IT_RELEASE_MEM
,
300 packet
->bitfields2
.event_type
= CACHE_FLUSH_AND_INV_TS_EVENT
;
301 packet
->bitfields2
.event_index
= event_index___release_mem__end_of_pipe
;
302 packet
->bitfields2
.tcl1_action_ena
= 1;
303 packet
->bitfields2
.tc_action_ena
= 1;
304 packet
->bitfields2
.cache_policy
= cache_policy___release_mem__lru
;
305 packet
->bitfields2
.atc
= 0;
307 packet
->bitfields3
.data_sel
= data_sel___release_mem__send_32_bit_low
;
308 packet
->bitfields3
.int_sel
=
309 int_sel___release_mem__send_interrupt_after_write_confirm
;
311 packet
->bitfields4
.address_lo_32b
= (gpu_addr
& 0xffffffff) >> 2;
312 packet
->address_hi
= upper_32_bits(gpu_addr
);
319 const struct packet_manager_funcs kfd_vi_pm_funcs
= {
320 .map_process
= pm_map_process_vi
,
321 .runlist
= pm_runlist_vi
,
322 .set_resources
= pm_set_resources_vi
,
323 .map_queues
= pm_map_queues_vi
,
324 .unmap_queues
= pm_unmap_queues_vi
,
325 .query_status
= pm_query_status_vi
,
326 .release_mem
= pm_release_mem_vi
,
327 .map_process_size
= sizeof(struct pm4_mes_map_process
),
328 .runlist_size
= sizeof(struct pm4_mes_runlist
),
329 .set_resources_size
= sizeof(struct pm4_mes_set_resources
),
330 .map_queues_size
= sizeof(struct pm4_mes_map_queues
),
331 .unmap_queues_size
= sizeof(struct pm4_mes_unmap_queues
),
332 .query_status_size
= sizeof(struct pm4_mes_query_status
),
333 .release_mem_size
= sizeof(struct pm4_mec_release_mem
)