?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / drivers / gpu / drm / amd / amdkfd / kfd_device_queue_manager.c
blobe686ce2bf3b3cd032ff7cb90df15924d66295523
1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/ratelimit.h>
25 #include <linux/printk.h>
26 #include <linux/slab.h>
27 #include <linux/list.h>
28 #include <linux/types.h>
29 #include <linux/bitops.h>
30 #include <linux/sched.h>
31 #include "kfd_priv.h"
32 #include "kfd_device_queue_manager.h"
33 #include "kfd_mqd_manager.h"
34 #include "cik_regs.h"
35 #include "kfd_kernel_queue.h"
36 #include "amdgpu_amdkfd.h"
37
38 /* Size of the per-pipe EOP queue */
39 #define CIK_HPD_EOP_BYTES_LOG2 11
40 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
41
42 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
43 u32 pasid, unsigned int vmid);
44
45 static int execute_queues_cpsch(struct device_queue_manager *dqm,
46 enum kfd_unmap_queues_filter filter,
47 uint32_t filter_param);
48 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
49 enum kfd_unmap_queues_filter filter,
50 uint32_t filter_param);
51
52 static int map_queues_cpsch(struct device_queue_manager *dqm);
53
54 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
55 struct queue *q);
56
57 static inline void deallocate_hqd(struct device_queue_manager *dqm,
58 struct queue *q);
59 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q);
60 static int allocate_sdma_queue(struct device_queue_manager *dqm,
61 struct queue *q);
62 static void kfd_process_hw_exception(struct work_struct *work);
63
64 static inline
65 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
66 {
67 if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI)
68 return KFD_MQD_TYPE_SDMA;
69 return KFD_MQD_TYPE_CP;
70 }
71
72 static bool is_pipe_enabled(struct device_queue_manager *dqm, int mec, int pipe)
73 {
74 int i;
75 int pipe_offset = (mec * dqm->dev->shared_resources.num_pipe_per_mec
76 + pipe) * dqm->dev->shared_resources.num_queue_per_pipe;
77
78 /* queue is available for KFD usage if bit is 1 */
79 for (i = 0; i < dqm->dev->shared_resources.num_queue_per_pipe; ++i)
80 if (test_bit(pipe_offset + i,
81 dqm->dev->shared_resources.cp_queue_bitmap))
82 return true;
83 return false;
84 }
85
86 unsigned int get_cp_queues_num(struct device_queue_manager *dqm)
87 {
88 return bitmap_weight(dqm->dev->shared_resources.cp_queue_bitmap,
89 KGD_MAX_QUEUES);
90 }
91
92 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm)
93 {
94 return dqm->dev->shared_resources.num_queue_per_pipe;
95 }
96
97 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm)
98 {
99 return dqm->dev->shared_resources.num_pipe_per_mec;
100 }
101
102 static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
103 {
104 return dqm->dev->device_info->num_sdma_engines;
105 }
106
107 static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm)
108 {
109 return dqm->dev->device_info->num_xgmi_sdma_engines;
110 }
111
112 static unsigned int get_num_all_sdma_engines(struct device_queue_manager *dqm)
113 {
114 return get_num_sdma_engines(dqm) + get_num_xgmi_sdma_engines(dqm);
115 }
116
117 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
118 {
119 return dqm->dev->device_info->num_sdma_engines
120 * dqm->dev->device_info->num_sdma_queues_per_engine;
121 }
122
123 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm)
124 {
125 return dqm->dev->device_info->num_xgmi_sdma_engines
126 * dqm->dev->device_info->num_sdma_queues_per_engine;
127 }
128
129 void program_sh_mem_settings(struct device_queue_manager *dqm,
130 struct qcm_process_device *qpd)
131 {
132 return dqm->dev->kfd2kgd->program_sh_mem_settings(
133 dqm->dev->kgd, qpd->vmid,
134 qpd->sh_mem_config,
135 qpd->sh_mem_ape1_base,
136 qpd->sh_mem_ape1_limit,
137 qpd->sh_mem_bases);
138 }
139
140 static void increment_queue_count(struct device_queue_manager *dqm,
141 enum kfd_queue_type type)
142 {
143 dqm->active_queue_count++;
144 if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
145 dqm->active_cp_queue_count++;
146 }
147
148 static void decrement_queue_count(struct device_queue_manager *dqm,
149 enum kfd_queue_type type)
150 {
151 dqm->active_queue_count--;
152 if (type == KFD_QUEUE_TYPE_COMPUTE || type == KFD_QUEUE_TYPE_DIQ)
153 dqm->active_cp_queue_count--;
154 }
155
156 static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q)
157 {
158 struct kfd_dev *dev = qpd->dqm->dev;
159
160 if (!KFD_IS_SOC15(dev->device_info->asic_family)) {
161 /* On pre-SOC15 chips we need to use the queue ID to
162 * preserve the user mode ABI.
163 */
164 q->doorbell_id = q->properties.queue_id;
165 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
166 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
167 /* For SDMA queues on SOC15 with 8-byte doorbell, use static
168 * doorbell assignments based on the engine and queue id.
169 * The doobell index distance between RLC (2*i) and (2*i+1)
170 * for a SDMA engine is 512.
171 */
172 uint32_t *idx_offset =
173 dev->shared_resources.sdma_doorbell_idx;
174
175 q->doorbell_id = idx_offset[q->properties.sdma_engine_id]
176 + (q->properties.sdma_queue_id & 1)
177 * KFD_QUEUE_DOORBELL_MIRROR_OFFSET
178 + (q->properties.sdma_queue_id >> 1);
179 } else {
180 /* For CP queues on SOC15 reserve a free doorbell ID */
181 unsigned int found;
182
183 found = find_first_zero_bit(qpd->doorbell_bitmap,
184 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
185 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
186 pr_debug("No doorbells available");
187 return -EBUSY;
188 }
189 set_bit(found, qpd->doorbell_bitmap);
190 q->doorbell_id = found;
191 }
192
193 q->properties.doorbell_off =
194 kfd_get_doorbell_dw_offset_in_bar(dev, qpd_to_pdd(qpd),
195 q->doorbell_id);
196 return 0;
197 }
198
199 static void deallocate_doorbell(struct qcm_process_device *qpd,
200 struct queue *q)
201 {
202 unsigned int old;
203 struct kfd_dev *dev = qpd->dqm->dev;
204
205 if (!KFD_IS_SOC15(dev->device_info->asic_family) ||
206 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
207 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
208 return;
209
210 old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap);
211 WARN_ON(!old);
212 }
213
214 static int allocate_vmid(struct device_queue_manager *dqm,
215 struct qcm_process_device *qpd,
216 struct queue *q)
217 {
218 int allocated_vmid = -1, i;
219
220 for (i = dqm->dev->vm_info.first_vmid_kfd;
221 i <= dqm->dev->vm_info.last_vmid_kfd; i++) {
222 if (!dqm->vmid_pasid[i]) {
223 allocated_vmid = i;
224 break;
225 }
226 }
227
228 if (allocated_vmid < 0) {
229 pr_err("no more vmid to allocate\n");
230 return -ENOSPC;
231 }
232
233 pr_debug("vmid allocated: %d\n", allocated_vmid);
234
235 dqm->vmid_pasid[allocated_vmid] = q->process->pasid;
236
237 set_pasid_vmid_mapping(dqm, q->process->pasid, allocated_vmid);
238
239 qpd->vmid = allocated_vmid;
240 q->properties.vmid = allocated_vmid;
241
242 program_sh_mem_settings(dqm, qpd);
243
244 /* qpd->page_table_base is set earlier when register_process()
245 * is called, i.e. when the first queue is created.
246 */
247 dqm->dev->kfd2kgd->set_vm_context_page_table_base(dqm->dev->kgd,
248 qpd->vmid,
249 qpd->page_table_base);
250 /* invalidate the VM context after pasid and vmid mapping is set up */
251 kfd_flush_tlb(qpd_to_pdd(qpd));
252
253 if (dqm->dev->kfd2kgd->set_scratch_backing_va)
254 dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->kgd,
255 qpd->sh_hidden_private_base, qpd->vmid);
256
257 return 0;
258 }
259
260 static int flush_texture_cache_nocpsch(struct kfd_dev *kdev,
261 struct qcm_process_device *qpd)
262 {
263 const struct packet_manager_funcs *pmf = qpd->dqm->packets.pmf;
264 int ret;
265
266 if (!qpd->ib_kaddr)
267 return -ENOMEM;
268
269 ret = pmf->release_mem(qpd->ib_base, (uint32_t *)qpd->ib_kaddr);
270 if (ret)
271 return ret;
272
273 return amdgpu_amdkfd_submit_ib(kdev->kgd, KGD_ENGINE_MEC1, qpd->vmid,
274 qpd->ib_base, (uint32_t *)qpd->ib_kaddr,
275 pmf->release_mem_size / sizeof(uint32_t));
276 }
277
278 static void deallocate_vmid(struct device_queue_manager *dqm,
279 struct qcm_process_device *qpd,
280 struct queue *q)
281 {
282 /* On GFX v7, CP doesn't flush TC at dequeue */
283 if (q->device->device_info->asic_family == CHIP_HAWAII)
284 if (flush_texture_cache_nocpsch(q->device, qpd))
285 pr_err("Failed to flush TC\n");
286
287 kfd_flush_tlb(qpd_to_pdd(qpd));
288
289 /* Release the vmid mapping */
290 set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
291 dqm->vmid_pasid[qpd->vmid] = 0;
292
293 qpd->vmid = 0;
294 q->properties.vmid = 0;
295 }
296
297 static int create_queue_nocpsch(struct device_queue_manager *dqm,
298 struct queue *q,
299 struct qcm_process_device *qpd)
300 {
301 struct mqd_manager *mqd_mgr;
302 int retval;
303
304 dqm_lock(dqm);
305
306 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
307 pr_warn("Can't create new usermode queue because %d queues were already created\n",
308 dqm->total_queue_count);
309 retval = -EPERM;
310 goto out_unlock;
311 }
312
313 if (list_empty(&qpd->queues_list)) {
314 retval = allocate_vmid(dqm, qpd, q);
315 if (retval)
316 goto out_unlock;
317 }
318 q->properties.vmid = qpd->vmid;
319 /*
320 * Eviction state logic: mark all queues as evicted, even ones
321 * not currently active. Restoring inactive queues later only
322 * updates the is_evicted flag but is a no-op otherwise.
323 */
324 q->properties.is_evicted = !!qpd->evicted;
325
326 q->properties.tba_addr = qpd->tba_addr;
327 q->properties.tma_addr = qpd->tma_addr;
328
329 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
330 q->properties.type)];
331 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
332 retval = allocate_hqd(dqm, q);
333 if (retval)
334 goto deallocate_vmid;
335 pr_debug("Loading mqd to hqd on pipe %d, queue %d\n",
336 q->pipe, q->queue);
337 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
338 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
339 retval = allocate_sdma_queue(dqm, q);
340 if (retval)
341 goto deallocate_vmid;
342 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
343 }
344
345 retval = allocate_doorbell(qpd, q);
346 if (retval)
347 goto out_deallocate_hqd;
348
349 /* Temporarily release dqm lock to avoid a circular lock dependency */
350 dqm_unlock(dqm);
351 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
352 dqm_lock(dqm);
353
354 if (!q->mqd_mem_obj) {
355 retval = -ENOMEM;
356 goto out_deallocate_doorbell;
357 }
358 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
359 &q->gart_mqd_addr, &q->properties);
360 if (q->properties.is_active) {
361 if (!dqm->sched_running) {
362 WARN_ONCE(1, "Load non-HWS mqd while stopped\n");
363 goto add_queue_to_list;
364 }
365
366 if (WARN(q->process->mm != current->mm,
367 "should only run in user thread"))
368 retval = -EFAULT;
369 else
370 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
371 q->queue, &q->properties, current->mm);
372 if (retval)
373 goto out_free_mqd;
374 }
375
376 add_queue_to_list:
377 list_add(&q->list, &qpd->queues_list);
378 qpd->queue_count++;
379 if (q->properties.is_active)
380 increment_queue_count(dqm, q->properties.type);
381
382 /*
383 * Unconditionally increment this counter, regardless of the queue's
384 * type or whether the queue is active.
385 */
386 dqm->total_queue_count++;
387 pr_debug("Total of %d queues are accountable so far\n",
388 dqm->total_queue_count);
389 goto out_unlock;
390
391 out_free_mqd:
392 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
393 out_deallocate_doorbell:
394 deallocate_doorbell(qpd, q);
395 out_deallocate_hqd:
396 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
397 deallocate_hqd(dqm, q);
398 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
399 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
400 deallocate_sdma_queue(dqm, q);
401 deallocate_vmid:
402 if (list_empty(&qpd->queues_list))
403 deallocate_vmid(dqm, qpd, q);
404 out_unlock:
405 dqm_unlock(dqm);
406 return retval;
407 }
408
409 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
410 {
411 bool set;
412 int pipe, bit, i;
413
414 set = false;
415
416 for (pipe = dqm->next_pipe_to_allocate, i = 0;
417 i < get_pipes_per_mec(dqm);
418 pipe = ((pipe + 1) % get_pipes_per_mec(dqm)), ++i) {
419
420 if (!is_pipe_enabled(dqm, 0, pipe))
421 continue;
422
423 if (dqm->allocated_queues[pipe] != 0) {
424 bit = ffs(dqm->allocated_queues[pipe]) - 1;
425 dqm->allocated_queues[pipe] &= ~(1 << bit);
426 q->pipe = pipe;
427 q->queue = bit;
428 set = true;
429 break;
430 }
431 }
432
433 if (!set)
434 return -EBUSY;
435
436 pr_debug("hqd slot - pipe %d, queue %d\n", q->pipe, q->queue);
437 /* horizontal hqd allocation */
438 dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_per_mec(dqm);
439
440 return 0;
441 }
442
443 static inline void deallocate_hqd(struct device_queue_manager *dqm,
444 struct queue *q)
445 {
446 dqm->allocated_queues[q->pipe] |= (1 << q->queue);
447 }
448
449 /* Access to DQM has to be locked before calling destroy_queue_nocpsch_locked
450 * to avoid asynchronized access
451 */
452 static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm,
453 struct qcm_process_device *qpd,
454 struct queue *q)
455 {
456 int retval;
457 struct mqd_manager *mqd_mgr;
458
459 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
460 q->properties.type)];
461
462 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
463 deallocate_hqd(dqm, q);
464 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
465 deallocate_sdma_queue(dqm, q);
466 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
467 deallocate_sdma_queue(dqm, q);
468 else {
469 pr_debug("q->properties.type %d is invalid\n",
470 q->properties.type);
471 return -EINVAL;
472 }
473 dqm->total_queue_count--;
474
475 deallocate_doorbell(qpd, q);
476
477 if (!dqm->sched_running) {
478 WARN_ONCE(1, "Destroy non-HWS queue while stopped\n");
479 return 0;
480 }
481
482 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
483 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
484 KFD_UNMAP_LATENCY_MS,
485 q->pipe, q->queue);
486 if (retval == -ETIME)
487 qpd->reset_wavefronts = true;
488
489
490 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
491
492 list_del(&q->list);
493 if (list_empty(&qpd->queues_list)) {
494 if (qpd->reset_wavefronts) {
495 pr_warn("Resetting wave fronts (nocpsch) on dev %p\n",
496 dqm->dev);
497 /* dbgdev_wave_reset_wavefronts has to be called before
498 * deallocate_vmid(), i.e. when vmid is still in use.
499 */
500 dbgdev_wave_reset_wavefronts(dqm->dev,
501 qpd->pqm->process);
502 qpd->reset_wavefronts = false;
503 }
504
505 deallocate_vmid(dqm, qpd, q);
506 }
507 qpd->queue_count--;
508 if (q->properties.is_active) {
509 decrement_queue_count(dqm, q->properties.type);
510 if (q->properties.is_gws) {
511 dqm->gws_queue_count--;
512 qpd->mapped_gws_queue = false;
513 }
514 }
515
516 return retval;
517 }
518
519 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
520 struct qcm_process_device *qpd,
521 struct queue *q)
522 {
523 int retval;
524 uint64_t sdma_val = 0;
525 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
526
527 /* Get the SDMA queue stats */
528 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
529 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
530 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
531 &sdma_val);
532 if (retval)
533 pr_err("Failed to read SDMA queue counter for queue: %d\n",
534 q->properties.queue_id);
535 }
536
537 dqm_lock(dqm);
538 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
539 if (!retval)
540 pdd->sdma_past_activity_counter += sdma_val;
541 dqm_unlock(dqm);
542
543 return retval;
544 }
545
546 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
547 {
548 int retval = 0;
549 struct mqd_manager *mqd_mgr;
550 struct kfd_process_device *pdd;
551 bool prev_active = false;
552
553 dqm_lock(dqm);
554 pdd = kfd_get_process_device_data(q->device, q->process);
555 if (!pdd) {
556 retval = -ENODEV;
557 goto out_unlock;
558 }
559 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
560 q->properties.type)];
561
562 /* Save previous activity state for counters */
563 prev_active = q->properties.is_active;
564
565 /* Make sure the queue is unmapped before updating the MQD */
566 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) {
567 retval = unmap_queues_cpsch(dqm,
568 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
569 if (retval) {
570 pr_err("unmap queue failed\n");
571 goto out_unlock;
572 }
573 } else if (prev_active &&
574 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
575 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
576 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
577
578 if (!dqm->sched_running) {
579 WARN_ONCE(1, "Update non-HWS queue while stopped\n");
580 goto out_unlock;
581 }
582
583 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
584 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
585 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
586 if (retval) {
587 pr_err("destroy mqd failed\n");
588 goto out_unlock;
589 }
590 }
591
592 mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
593
594 /*
595 * check active state vs. the previous state and modify
596 * counter accordingly. map_queues_cpsch uses the
597 * dqm->active_queue_count to determine whether a new runlist must be
598 * uploaded.
599 */
600 if (q->properties.is_active && !prev_active)
601 increment_queue_count(dqm, q->properties.type);
602 else if (!q->properties.is_active && prev_active)
603 decrement_queue_count(dqm, q->properties.type);
604
605 if (q->gws && !q->properties.is_gws) {
606 if (q->properties.is_active) {
607 dqm->gws_queue_count++;
608 pdd->qpd.mapped_gws_queue = true;
609 }
610 q->properties.is_gws = true;
611 } else if (!q->gws && q->properties.is_gws) {
612 if (q->properties.is_active) {
613 dqm->gws_queue_count--;
614 pdd->qpd.mapped_gws_queue = false;
615 }
616 q->properties.is_gws = false;
617 }
618
619 if (dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS)
620 retval = map_queues_cpsch(dqm);
621 else if (q->properties.is_active &&
622 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
623 q->properties.type == KFD_QUEUE_TYPE_SDMA ||
624 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
625 if (WARN(q->process->mm != current->mm,
626 "should only run in user thread"))
627 retval = -EFAULT;
628 else
629 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
630 q->pipe, q->queue,
631 &q->properties, current->mm);
632 }
633
634 out_unlock:
635 dqm_unlock(dqm);
636 return retval;
637 }
638
639 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
640 struct qcm_process_device *qpd)
641 {
642 struct queue *q;
643 struct mqd_manager *mqd_mgr;
644 struct kfd_process_device *pdd;
645 int retval, ret = 0;
646
647 dqm_lock(dqm);
648 if (qpd->evicted++ > 0) /* already evicted, do nothing */
649 goto out;
650
651 pdd = qpd_to_pdd(qpd);
652 pr_debug_ratelimited("Evicting PASID 0x%x queues\n",
653 pdd->process->pasid);
654
655 pdd->last_evict_timestamp = get_jiffies_64();
656 /* Mark all queues as evicted. Deactivate all active queues on
657 * the qpd.
658 */
659 list_for_each_entry(q, &qpd->queues_list, list) {
660 q->properties.is_evicted = true;
661 if (!q->properties.is_active)
662 continue;
663
664 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
665 q->properties.type)];
666 q->properties.is_active = false;
667 decrement_queue_count(dqm, q->properties.type);
668 if (q->properties.is_gws) {
669 dqm->gws_queue_count--;
670 qpd->mapped_gws_queue = false;
671 }
672
673 if (WARN_ONCE(!dqm->sched_running, "Evict when stopped\n"))
674 continue;
675
676 retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
677 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
678 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
679 if (retval && !ret)
680 /* Return the first error, but keep going to
681 * maintain a consistent eviction state
682 */
683 ret = retval;
684 }
685
686 out:
687 dqm_unlock(dqm);
688 return ret;
689 }
690
691 static int evict_process_queues_cpsch(struct device_queue_manager *dqm,
692 struct qcm_process_device *qpd)
693 {
694 struct queue *q;
695 struct kfd_process_device *pdd;
696 int retval = 0;
697
698 dqm_lock(dqm);
699 if (qpd->evicted++ > 0) /* already evicted, do nothing */
700 goto out;
701
702 pdd = qpd_to_pdd(qpd);
703 pr_debug_ratelimited("Evicting PASID 0x%x queues\n",
704 pdd->process->pasid);
705
706 /* Mark all queues as evicted. Deactivate all active queues on
707 * the qpd.
708 */
709 list_for_each_entry(q, &qpd->queues_list, list) {
710 q->properties.is_evicted = true;
711 if (!q->properties.is_active)
712 continue;
713
714 q->properties.is_active = false;
715 decrement_queue_count(dqm, q->properties.type);
716 }
717 pdd->last_evict_timestamp = get_jiffies_64();
718 retval = execute_queues_cpsch(dqm,
719 qpd->is_debug ?
720 KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES :
721 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
722
723 out:
724 dqm_unlock(dqm);
725 return retval;
726 }
727
728 static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
729 struct qcm_process_device *qpd)
730 {
731 struct mm_struct *mm = NULL;
732 struct queue *q;
733 struct mqd_manager *mqd_mgr;
734 struct kfd_process_device *pdd;
735 uint64_t pd_base;
736 uint64_t eviction_duration;
737 int retval, ret = 0;
738
739 pdd = qpd_to_pdd(qpd);
740 /* Retrieve PD base */
741 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
742
743 dqm_lock(dqm);
744 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
745 goto out;
746 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
747 qpd->evicted--;
748 goto out;
749 }
750
751 pr_debug_ratelimited("Restoring PASID 0x%x queues\n",
752 pdd->process->pasid);
753
754 /* Update PD Base in QPD */
755 qpd->page_table_base = pd_base;
756 pr_debug("Updated PD address to 0x%llx\n", pd_base);
757
758 if (!list_empty(&qpd->queues_list)) {
759 dqm->dev->kfd2kgd->set_vm_context_page_table_base(
760 dqm->dev->kgd,
761 qpd->vmid,
762 qpd->page_table_base);
763 kfd_flush_tlb(pdd);
764 }
765
766 /* Take a safe reference to the mm_struct, which may otherwise
767 * disappear even while the kfd_process is still referenced.
768 */
769 mm = get_task_mm(pdd->process->lead_thread);
770 if (!mm) {
771 ret = -EFAULT;
772 goto out;
773 }
774
775 /* Remove the eviction flags. Activate queues that are not
776 * inactive for other reasons.
777 */
778 list_for_each_entry(q, &qpd->queues_list, list) {
779 q->properties.is_evicted = false;
780 if (!QUEUE_IS_ACTIVE(q->properties))
781 continue;
782
783 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
784 q->properties.type)];
785 q->properties.is_active = true;
786 increment_queue_count(dqm, q->properties.type);
787 if (q->properties.is_gws) {
788 dqm->gws_queue_count++;
789 qpd->mapped_gws_queue = true;
790 }
791
792 if (WARN_ONCE(!dqm->sched_running, "Restore when stopped\n"))
793 continue;
794
795 retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
796 q->queue, &q->properties, mm);
797 if (retval && !ret)
798 /* Return the first error, but keep going to
799 * maintain a consistent eviction state
800 */
801 ret = retval;
802 }
803 qpd->evicted = 0;
804 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
805 atomic64_add(eviction_duration, &pdd->evict_duration_counter);
806 out:
807 if (mm)
808 mmput(mm);
809 dqm_unlock(dqm);
810 return ret;
811 }
812
813 static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
814 struct qcm_process_device *qpd)
815 {
816 struct queue *q;
817 struct kfd_process_device *pdd;
818 uint64_t pd_base;
819 uint64_t eviction_duration;
820 int retval = 0;
821
822 pdd = qpd_to_pdd(qpd);
823 /* Retrieve PD base */
824 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
825
826 dqm_lock(dqm);
827 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
828 goto out;
829 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
830 qpd->evicted--;
831 goto out;
832 }
833
834 pr_debug_ratelimited("Restoring PASID 0x%x queues\n",
835 pdd->process->pasid);
836
837 /* Update PD Base in QPD */
838 qpd->page_table_base = pd_base;
839 pr_debug("Updated PD address to 0x%llx\n", pd_base);
840
841 /* activate all active queues on the qpd */
842 list_for_each_entry(q, &qpd->queues_list, list) {
843 q->properties.is_evicted = false;
844 if (!QUEUE_IS_ACTIVE(q->properties))
845 continue;
846
847 q->properties.is_active = true;
848 increment_queue_count(dqm, q->properties.type);
849 }
850 retval = execute_queues_cpsch(dqm,
851 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
852 qpd->evicted = 0;
853 eviction_duration = get_jiffies_64() - pdd->last_evict_timestamp;
854 atomic64_add(eviction_duration, &pdd->evict_duration_counter);
855 out:
856 dqm_unlock(dqm);
857 return retval;
858 }
859
860 static int register_process(struct device_queue_manager *dqm,
861 struct qcm_process_device *qpd)
862 {
863 struct device_process_node *n;
864 struct kfd_process_device *pdd;
865 uint64_t pd_base;
866 int retval;
867
868 n = kzalloc(sizeof(*n), GFP_KERNEL);
869 if (!n)
870 return -ENOMEM;
871
872 n->qpd = qpd;
873
874 pdd = qpd_to_pdd(qpd);
875 /* Retrieve PD base */
876 pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
877
878 dqm_lock(dqm);
879 list_add(&n->list, &dqm->queues);
880
881 /* Update PD Base in QPD */
882 qpd->page_table_base = pd_base;
883 pr_debug("Updated PD address to 0x%llx\n", pd_base);
884
885 retval = dqm->asic_ops.update_qpd(dqm, qpd);
886
887 dqm->processes_count++;
888
889 dqm_unlock(dqm);
890
891 /* Outside the DQM lock because under the DQM lock we can't do
892 * reclaim or take other locks that others hold while reclaiming.
893 */
894 kfd_inc_compute_active(dqm->dev);
895
896 return retval;
897 }
898
899 static int unregister_process(struct device_queue_manager *dqm,
900 struct qcm_process_device *qpd)
901 {
902 int retval;
903 struct device_process_node *cur, *next;
904
905 pr_debug("qpd->queues_list is %s\n",
906 list_empty(&qpd->queues_list) ? "empty" : "not empty");
907
908 retval = 0;
909 dqm_lock(dqm);
910
911 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
912 if (qpd == cur->qpd) {
913 list_del(&cur->list);
914 kfree(cur);
915 dqm->processes_count--;
916 goto out;
917 }
918 }
919 /* qpd not found in dqm list */
920 retval = 1;
921 out:
922 dqm_unlock(dqm);
923
924 /* Outside the DQM lock because under the DQM lock we can't do
925 * reclaim or take other locks that others hold while reclaiming.
926 */
927 if (!retval)
928 kfd_dec_compute_active(dqm->dev);
929
930 return retval;
931 }
932
933 static int
934 set_pasid_vmid_mapping(struct device_queue_manager *dqm, u32 pasid,
935 unsigned int vmid)
936 {
937 return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
938 dqm->dev->kgd, pasid, vmid);
939 }
940
941 static void init_interrupts(struct device_queue_manager *dqm)
942 {
943 unsigned int i;
944
945 for (i = 0 ; i < get_pipes_per_mec(dqm) ; i++)
946 if (is_pipe_enabled(dqm, 0, i))
947 dqm->dev->kfd2kgd->init_interrupts(dqm->dev->kgd, i);
948 }
949
950 static int initialize_nocpsch(struct device_queue_manager *dqm)
951 {
952 int pipe, queue;
953
954 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
955
956 dqm->allocated_queues = kcalloc(get_pipes_per_mec(dqm),
957 sizeof(unsigned int), GFP_KERNEL);
958 if (!dqm->allocated_queues)
959 return -ENOMEM;
960
961 mutex_init(&dqm->lock_hidden);
962 INIT_LIST_HEAD(&dqm->queues);
963 dqm->active_queue_count = dqm->next_pipe_to_allocate = 0;
964 dqm->active_cp_queue_count = 0;
965 dqm->gws_queue_count = 0;
966
967 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
968 int pipe_offset = pipe * get_queues_per_pipe(dqm);
969
970 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++)
971 if (test_bit(pipe_offset + queue,
972 dqm->dev->shared_resources.cp_queue_bitmap))
973 dqm->allocated_queues[pipe] |= 1 << queue;
974 }
975
976 memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
977
978 dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm));
979 dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm));
980
981 return 0;
982 }
983
984 static void uninitialize(struct device_queue_manager *dqm)
985 {
986 int i;
987
988 WARN_ON(dqm->active_queue_count > 0 || dqm->processes_count > 0);
989
990 kfree(dqm->allocated_queues);
991 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
992 kfree(dqm->mqd_mgrs[i]);
993 mutex_destroy(&dqm->lock_hidden);
994 }
995
996 static int start_nocpsch(struct device_queue_manager *dqm)
997 {
998 pr_info("SW scheduler is used");
999 init_interrupts(dqm);
1000
1001 if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
1002 return pm_init(&dqm->packets, dqm);
1003 dqm->sched_running = true;
1004
1005 return 0;
1006 }
1007
1008 static int stop_nocpsch(struct device_queue_manager *dqm)
1009 {
1010 if (dqm->dev->device_info->asic_family == CHIP_HAWAII)
1011 pm_uninit(&dqm->packets, false);
1012 dqm->sched_running = false;
1013
1014 return 0;
1015 }
1016
1017 static void pre_reset(struct device_queue_manager *dqm)
1018 {
1019 dqm_lock(dqm);
1020 dqm->is_resetting = true;
1021 dqm_unlock(dqm);
1022 }
1023
1024 static int allocate_sdma_queue(struct device_queue_manager *dqm,
1025 struct queue *q)
1026 {
1027 int bit;
1028
1029 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1030 if (dqm->sdma_bitmap == 0) {
1031 pr_err("No more SDMA queue to allocate\n");
1032 return -ENOMEM;
1033 }
1034
1035 bit = __ffs64(dqm->sdma_bitmap);
1036 dqm->sdma_bitmap &= ~(1ULL << bit);
1037 q->sdma_id = bit;
1038 q->properties.sdma_engine_id = q->sdma_id %
1039 get_num_sdma_engines(dqm);
1040 q->properties.sdma_queue_id = q->sdma_id /
1041 get_num_sdma_engines(dqm);
1042 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1043 if (dqm->xgmi_sdma_bitmap == 0) {
1044 pr_err("No more XGMI SDMA queue to allocate\n");
1045 return -ENOMEM;
1046 }
1047 bit = __ffs64(dqm->xgmi_sdma_bitmap);
1048 dqm->xgmi_sdma_bitmap &= ~(1ULL << bit);
1049 q->sdma_id = bit;
1050 /* sdma_engine_id is sdma id including
1051 * both PCIe-optimized SDMAs and XGMI-
1052 * optimized SDMAs. The calculation below
1053 * assumes the first N engines are always
1054 * PCIe-optimized ones
1055 */
1056 q->properties.sdma_engine_id = get_num_sdma_engines(dqm) +
1057 q->sdma_id % get_num_xgmi_sdma_engines(dqm);
1058 q->properties.sdma_queue_id = q->sdma_id /
1059 get_num_xgmi_sdma_engines(dqm);
1060 }
1061
1062 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
1063 pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id);
1064
1065 return 0;
1066 }
1067
1068 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
1069 struct queue *q)
1070 {
1071 if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
1072 if (q->sdma_id >= get_num_sdma_queues(dqm))
1073 return;
1074 dqm->sdma_bitmap |= (1ULL << q->sdma_id);
1075 } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1076 if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm))
1077 return;
1078 dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id);
1079 }
1080 }
1081
1082 /*
1083 * Device Queue Manager implementation for cp scheduler
1084 */
1085
1086 static int set_sched_resources(struct device_queue_manager *dqm)
1087 {
1088 int i, mec;
1089 struct scheduling_resources res;
1090
1091 res.vmid_mask = dqm->dev->shared_resources.compute_vmid_bitmap;
1092
1093 res.queue_mask = 0;
1094 for (i = 0; i < KGD_MAX_QUEUES; ++i) {
1095 mec = (i / dqm->dev->shared_resources.num_queue_per_pipe)
1096 / dqm->dev->shared_resources.num_pipe_per_mec;
1097
1098 if (!test_bit(i, dqm->dev->shared_resources.cp_queue_bitmap))
1099 continue;
1100
1101 /* only acquire queues from the first MEC */
1102 if (mec > 0)
1103 continue;
1104
1105 /* This situation may be hit in the future if a new HW
1106 * generation exposes more than 64 queues. If so, the
1107 * definition of res.queue_mask needs updating
1108 */
1109 if (WARN_ON(i >= (sizeof(res.queue_mask)*8))) {
1110 pr_err("Invalid queue enabled by amdgpu: %d\n", i);
1111 break;
1112 }
1113
1114 res.queue_mask |= 1ull
1115 << amdgpu_queue_mask_bit_to_set_resource_bit(
1116 (struct amdgpu_device *)dqm->dev->kgd, i);
1117 }
1118 res.gws_mask = ~0ull;
1119 res.oac_mask = res.gds_heap_base = res.gds_heap_size = 0;
1120
1121 pr_debug("Scheduling resources:\n"
1122 "vmid mask: 0x%8X\n"
1123 "queue mask: 0x%8llX\n",
1124 res.vmid_mask, res.queue_mask);
1125
1126 return pm_send_set_resources(&dqm->packets, &res);
1127 }
1128
1129 static int initialize_cpsch(struct device_queue_manager *dqm)
1130 {
1131 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
1132
1133 mutex_init(&dqm->lock_hidden);
1134 INIT_LIST_HEAD(&dqm->queues);
1135 dqm->active_queue_count = dqm->processes_count = 0;
1136 dqm->active_cp_queue_count = 0;
1137 dqm->gws_queue_count = 0;
1138 dqm->active_runlist = false;
1139 dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm));
1140 dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm));
1141
1142 INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
1143
1144 return 0;
1145 }
1146
1147 static int start_cpsch(struct device_queue_manager *dqm)
1148 {
1149 int retval;
1150
1151 retval = 0;
1152
1153 retval = pm_init(&dqm->packets, dqm);
1154 if (retval)
1155 goto fail_packet_manager_init;
1156
1157 retval = set_sched_resources(dqm);
1158 if (retval)
1159 goto fail_set_sched_resources;
1160
1161 pr_debug("Allocating fence memory\n");
1162
1163 /* allocate fence memory on the gart */
1164 retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
1165 &dqm->fence_mem);
1166
1167 if (retval)
1168 goto fail_allocate_vidmem;
1169
1170 dqm->fence_addr = dqm->fence_mem->cpu_ptr;
1171 dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
1172
1173 init_interrupts(dqm);
1174
1175 dqm_lock(dqm);
1176 /* clear hang status when driver try to start the hw scheduler */
1177 dqm->is_hws_hang = false;
1178 dqm->is_resetting = false;
1179 dqm->sched_running = true;
1180 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1181 dqm_unlock(dqm);
1182
1183 return 0;
1184 fail_allocate_vidmem:
1185 fail_set_sched_resources:
1186 pm_uninit(&dqm->packets, false);
1187 fail_packet_manager_init:
1188 return retval;
1189 }
1190
1191 static int stop_cpsch(struct device_queue_manager *dqm)
1192 {
1193 bool hanging;
1194
1195 dqm_lock(dqm);
1196 if (!dqm->is_hws_hang)
1197 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1198 hanging = dqm->is_hws_hang || dqm->is_resetting;
1199 dqm->sched_running = false;
1200 dqm_unlock(dqm);
1201
1202 pm_release_ib(&dqm->packets);
1203
1204 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
1205 pm_uninit(&dqm->packets, hanging);
1206
1207 return 0;
1208 }
1209
1210 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
1211 struct kernel_queue *kq,
1212 struct qcm_process_device *qpd)
1213 {
1214 dqm_lock(dqm);
1215 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1216 pr_warn("Can't create new kernel queue because %d queues were already created\n",
1217 dqm->total_queue_count);
1218 dqm_unlock(dqm);
1219 return -EPERM;
1220 }
1221
1222 /*
1223 * Unconditionally increment this counter, regardless of the queue's
1224 * type or whether the queue is active.
1225 */
1226 dqm->total_queue_count++;
1227 pr_debug("Total of %d queues are accountable so far\n",
1228 dqm->total_queue_count);
1229
1230 list_add(&kq->list, &qpd->priv_queue_list);
1231 increment_queue_count(dqm, kq->queue->properties.type);
1232 qpd->is_debug = true;
1233 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1234 dqm_unlock(dqm);
1235
1236 return 0;
1237 }
1238
1239 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
1240 struct kernel_queue *kq,
1241 struct qcm_process_device *qpd)
1242 {
1243 dqm_lock(dqm);
1244 list_del(&kq->list);
1245 decrement_queue_count(dqm, kq->queue->properties.type);
1246 qpd->is_debug = false;
1247 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1248 /*
1249 * Unconditionally decrement this counter, regardless of the queue's
1250 * type.
1251 */
1252 dqm->total_queue_count--;
1253 pr_debug("Total of %d queues are accountable so far\n",
1254 dqm->total_queue_count);
1255 dqm_unlock(dqm);
1256 }
1257
1258 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
1259 struct qcm_process_device *qpd)
1260 {
1261 int retval;
1262 struct mqd_manager *mqd_mgr;
1263
1264 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
1265 pr_warn("Can't create new usermode queue because %d queues were already created\n",
1266 dqm->total_queue_count);
1267 retval = -EPERM;
1268 goto out;
1269 }
1270
1271 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1272 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1273 dqm_lock(dqm);
1274 retval = allocate_sdma_queue(dqm, q);
1275 dqm_unlock(dqm);
1276 if (retval)
1277 goto out;
1278 }
1279
1280 retval = allocate_doorbell(qpd, q);
1281 if (retval)
1282 goto out_deallocate_sdma_queue;
1283
1284 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1285 q->properties.type)];
1286
1287 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1288 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1289 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
1290 q->properties.tba_addr = qpd->tba_addr;
1291 q->properties.tma_addr = qpd->tma_addr;
1292 q->mqd_mem_obj = mqd_mgr->allocate_mqd(mqd_mgr->dev, &q->properties);
1293 if (!q->mqd_mem_obj) {
1294 retval = -ENOMEM;
1295 goto out_deallocate_doorbell;
1296 }
1297
1298 dqm_lock(dqm);
1299 /*
1300 * Eviction state logic: mark all queues as evicted, even ones
1301 * not currently active. Restoring inactive queues later only
1302 * updates the is_evicted flag but is a no-op otherwise.
1303 */
1304 q->properties.is_evicted = !!qpd->evicted;
1305 mqd_mgr->init_mqd(mqd_mgr, &q->mqd, q->mqd_mem_obj,
1306 &q->gart_mqd_addr, &q->properties);
1307
1308 list_add(&q->list, &qpd->queues_list);
1309 qpd->queue_count++;
1310
1311 if (q->properties.is_active) {
1312 increment_queue_count(dqm, q->properties.type);
1313
1314 execute_queues_cpsch(dqm,
1315 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1316 }
1317
1318 /*
1319 * Unconditionally increment this counter, regardless of the queue's
1320 * type or whether the queue is active.
1321 */
1322 dqm->total_queue_count++;
1323
1324 pr_debug("Total of %d queues are accountable so far\n",
1325 dqm->total_queue_count);
1326
1327 dqm_unlock(dqm);
1328 return retval;
1329
1330 out_deallocate_doorbell:
1331 deallocate_doorbell(qpd, q);
1332 out_deallocate_sdma_queue:
1333 if (q->properties.type == KFD_QUEUE_TYPE_SDMA ||
1334 q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) {
1335 dqm_lock(dqm);
1336 deallocate_sdma_queue(dqm, q);
1337 dqm_unlock(dqm);
1338 }
1339 out:
1340 return retval;
1341 }
1342
1343 int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
1344 unsigned int fence_value,
1345 unsigned int timeout_ms)
1346 {
1347 unsigned long end_jiffies = msecs_to_jiffies(timeout_ms) + jiffies;
1348
1349 while (*fence_addr != fence_value) {
1350 if (time_after(jiffies, end_jiffies)) {
1351 pr_err("qcm fence wait loop timeout expired\n");
1352 /* In HWS case, this is used to halt the driver thread
1353 * in order not to mess up CP states before doing
1354 * scandumps for FW debugging.
1355 */
1356 while (halt_if_hws_hang)
1357 schedule();
1358
1359 return -ETIME;
1360 }
1361 schedule();
1362 }
1363
1364 return 0;
1365 }
1366
1367 /* dqm->lock mutex has to be locked before calling this function */
1368 static int map_queues_cpsch(struct device_queue_manager *dqm)
1369 {
1370 int retval;
1371
1372 if (!dqm->sched_running)
1373 return 0;
1374 if (dqm->active_queue_count <= 0 || dqm->processes_count <= 0)
1375 return 0;
1376 if (dqm->active_runlist)
1377 return 0;
1378
1379 retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1380 pr_debug("%s sent runlist\n", __func__);
1381 if (retval) {
1382 pr_err("failed to execute runlist\n");
1383 return retval;
1384 }
1385 dqm->active_runlist = true;
1386
1387 return retval;
1388 }
1389
1390 /* dqm->lock mutex has to be locked before calling this function */
1391 static int unmap_queues_cpsch(struct device_queue_manager *dqm,
1392 enum kfd_unmap_queues_filter filter,
1393 uint32_t filter_param)
1394 {
1395 int retval = 0;
1396
1397 if (!dqm->sched_running)
1398 return 0;
1399 if (dqm->is_hws_hang)
1400 return -EIO;
1401 if (!dqm->active_runlist)
1402 return retval;
1403
1404 retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
1405 filter, filter_param, false, 0);
1406 if (retval)
1407 return retval;
1408
1409 *dqm->fence_addr = KFD_FENCE_INIT;
1410 pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
1411 KFD_FENCE_COMPLETED);
1412 /* should be timed out */
1413 retval = amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
1414 queue_preemption_timeout_ms);
1415 if (retval) {
1416 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
1417 dqm->is_hws_hang = true;
1418 /* It's possible we're detecting a HWS hang in the
1419 * middle of a GPU reset. No need to schedule another
1420 * reset in this case.
1421 */
1422 if (!dqm->is_resetting)
1423 schedule_work(&dqm->hw_exception_work);
1424 return retval;
1425 }
1426
1427 pm_release_ib(&dqm->packets);
1428 dqm->active_runlist = false;
1429
1430 return retval;
1431 }
1432
1433 /* dqm->lock mutex has to be locked before calling this function */
1434 static int execute_queues_cpsch(struct device_queue_manager *dqm,
1435 enum kfd_unmap_queues_filter filter,
1436 uint32_t filter_param)
1437 {
1438 int retval;
1439
1440 if (dqm->is_hws_hang)
1441 return -EIO;
1442 retval = unmap_queues_cpsch(dqm, filter, filter_param);
1443 if (retval)
1444 return retval;
1445
1446 return map_queues_cpsch(dqm);
1447 }
1448
1449 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1450 struct qcm_process_device *qpd,
1451 struct queue *q)
1452 {
1453 int retval;
1454 struct mqd_manager *mqd_mgr;
1455 uint64_t sdma_val = 0;
1456 struct kfd_process_device *pdd = qpd_to_pdd(qpd);
1457
1458 /* Get the SDMA queue stats */
1459 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1460 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1461 retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr,
1462 &sdma_val);
1463 if (retval)
1464 pr_err("Failed to read SDMA queue counter for queue: %d\n",
1465 q->properties.queue_id);
1466 }
1467
1468 retval = 0;
1469
1470 /* remove queue from list to prevent rescheduling after preemption */
1471 dqm_lock(dqm);
1472
1473 if (qpd->is_debug) {
1474 /*
1475 * error, currently we do not allow to destroy a queue
1476 * of a currently debugged process
1477 */
1478 retval = -EBUSY;
1479 goto failed_try_destroy_debugged_queue;
1480
1481 }
1482
1483 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1484 q->properties.type)];
1485
1486 deallocate_doorbell(qpd, q);
1487
1488 if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) ||
1489 (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) {
1490 deallocate_sdma_queue(dqm, q);
1491 pdd->sdma_past_activity_counter += sdma_val;
1492 }
1493
1494 list_del(&q->list);
1495 qpd->queue_count--;
1496 if (q->properties.is_active) {
1497 decrement_queue_count(dqm, q->properties.type);
1498 retval = execute_queues_cpsch(dqm,
1499 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1500 if (retval == -ETIME)
1501 qpd->reset_wavefronts = true;
1502 if (q->properties.is_gws) {
1503 dqm->gws_queue_count--;
1504 qpd->mapped_gws_queue = false;
1505 }
1506 }
1507
1508 /*
1509 * Unconditionally decrement this counter, regardless of the queue's
1510 * type
1511 */
1512 dqm->total_queue_count--;
1513 pr_debug("Total of %d queues are accountable so far\n",
1514 dqm->total_queue_count);
1515
1516 dqm_unlock(dqm);
1517
1518 /* Do free_mqd after dqm_unlock(dqm) to avoid circular locking */
1519 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
1520
1521 return retval;
1522
1523 failed_try_destroy_debugged_queue:
1524
1525 dqm_unlock(dqm);
1526 return retval;
1527 }
1528
1529 /*
1530 * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1531 * stay in user mode.
1532 */
1533 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1534 /* APE1 limit is inclusive and 64K aligned. */
1535 #define APE1_LIMIT_ALIGNMENT 0xFFFF
1536
1537 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1538 struct qcm_process_device *qpd,
1539 enum cache_policy default_policy,
1540 enum cache_policy alternate_policy,
1541 void __user *alternate_aperture_base,
1542 uint64_t alternate_aperture_size)
1543 {
1544 bool retval = true;
1545
1546 if (!dqm->asic_ops.set_cache_memory_policy)
1547 return retval;
1548
1549 dqm_lock(dqm);
1550
1551 if (alternate_aperture_size == 0) {
1552 /* base > limit disables APE1 */
1553 qpd->sh_mem_ape1_base = 1;
1554 qpd->sh_mem_ape1_limit = 0;
1555 } else {
1556 /*
1557 * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1558 * SH_MEM_APE1_BASE[31:0], 0x0000 }
1559 * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1560 * SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1561 * Verify that the base and size parameters can be
1562 * represented in this format and convert them.
1563 * Additionally restrict APE1 to user-mode addresses.
1564 */
1565
1566 uint64_t base = (uintptr_t)alternate_aperture_base;
1567 uint64_t limit = base + alternate_aperture_size - 1;
1568
1569 if (limit <= base || (base & APE1_FIXED_BITS_MASK) != 0 ||
1570 (limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT) {
1571 retval = false;
1572 goto out;
1573 }
1574
1575 qpd->sh_mem_ape1_base = base >> 16;
1576 qpd->sh_mem_ape1_limit = limit >> 16;
1577 }
1578
1579 retval = dqm->asic_ops.set_cache_memory_policy(
1580 dqm,
1581 qpd,
1582 default_policy,
1583 alternate_policy,
1584 alternate_aperture_base,
1585 alternate_aperture_size);
1586
1587 if ((dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1588 program_sh_mem_settings(dqm, qpd);
1589
1590 pr_debug("sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1591 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1592 qpd->sh_mem_ape1_limit);
1593
1594 out:
1595 dqm_unlock(dqm);
1596 return retval;
1597 }
1598
1599 static int set_trap_handler(struct device_queue_manager *dqm,
1600 struct qcm_process_device *qpd,
1601 uint64_t tba_addr,
1602 uint64_t tma_addr)
1603 {
1604 uint64_t *tma;
1605
1606 if (dqm->dev->cwsr_enabled) {
1607 /* Jump from CWSR trap handler to user trap */
1608 tma = (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1609 tma[0] = tba_addr;
1610 tma[1] = tma_addr;
1611 } else {
1612 qpd->tba_addr = tba_addr;
1613 qpd->tma_addr = tma_addr;
1614 }
1615
1616 return 0;
1617 }
1618
1619 static int process_termination_nocpsch(struct device_queue_manager *dqm,
1620 struct qcm_process_device *qpd)
1621 {
1622 struct queue *q, *next;
1623 struct device_process_node *cur, *next_dpn;
1624 int retval = 0;
1625 bool found = false;
1626
1627 dqm_lock(dqm);
1628
1629 /* Clear all user mode queues */
1630 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1631 int ret;
1632
1633 ret = destroy_queue_nocpsch_locked(dqm, qpd, q);
1634 if (ret)
1635 retval = ret;
1636 }
1637
1638 /* Unregister process */
1639 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1640 if (qpd == cur->qpd) {
1641 list_del(&cur->list);
1642 kfree(cur);
1643 dqm->processes_count--;
1644 found = true;
1645 break;
1646 }
1647 }
1648
1649 dqm_unlock(dqm);
1650
1651 /* Outside the DQM lock because under the DQM lock we can't do
1652 * reclaim or take other locks that others hold while reclaiming.
1653 */
1654 if (found)
1655 kfd_dec_compute_active(dqm->dev);
1656
1657 return retval;
1658 }
1659
1660 static int get_wave_state(struct device_queue_manager *dqm,
1661 struct queue *q,
1662 void __user *ctl_stack,
1663 u32 *ctl_stack_used_size,
1664 u32 *save_area_used_size)
1665 {
1666 struct mqd_manager *mqd_mgr;
1667 int r;
1668
1669 dqm_lock(dqm);
1670
1671 if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
1672 q->properties.is_active || !q->device->cwsr_enabled) {
1673 r = -EINVAL;
1674 goto dqm_unlock;
1675 }
1676
1677 mqd_mgr = dqm->mqd_mgrs[KFD_MQD_TYPE_CP];
1678
1679 if (!mqd_mgr->get_wave_state) {
1680 r = -EINVAL;
1681 goto dqm_unlock;
1682 }
1683
1684 r = mqd_mgr->get_wave_state(mqd_mgr, q->mqd, ctl_stack,
1685 ctl_stack_used_size, save_area_used_size);
1686
1687 dqm_unlock:
1688 dqm_unlock(dqm);
1689 return r;
1690 }
1691
1692 static int process_termination_cpsch(struct device_queue_manager *dqm,
1693 struct qcm_process_device *qpd)
1694 {
1695 int retval;
1696 struct queue *q, *next;
1697 struct kernel_queue *kq, *kq_next;
1698 struct mqd_manager *mqd_mgr;
1699 struct device_process_node *cur, *next_dpn;
1700 enum kfd_unmap_queues_filter filter =
1701 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
1702 bool found = false;
1703
1704 retval = 0;
1705
1706 dqm_lock(dqm);
1707
1708 /* Clean all kernel queues */
1709 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
1710 list_del(&kq->list);
1711 decrement_queue_count(dqm, kq->queue->properties.type);
1712 qpd->is_debug = false;
1713 dqm->total_queue_count--;
1714 filter = KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES;
1715 }
1716
1717 /* Clear all user mode queues */
1718 list_for_each_entry(q, &qpd->queues_list, list) {
1719 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1720 deallocate_sdma_queue(dqm, q);
1721 else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)
1722 deallocate_sdma_queue(dqm, q);
1723
1724 if (q->properties.is_active) {
1725 decrement_queue_count(dqm, q->properties.type);
1726 if (q->properties.is_gws) {
1727 dqm->gws_queue_count--;
1728 qpd->mapped_gws_queue = false;
1729 }
1730 }
1731
1732 dqm->total_queue_count--;
1733 }
1734
1735 /* Unregister process */
1736 list_for_each_entry_safe(cur, next_dpn, &dqm->queues, list) {
1737 if (qpd == cur->qpd) {
1738 list_del(&cur->list);
1739 kfree(cur);
1740 dqm->processes_count--;
1741 found = true;
1742 break;
1743 }
1744 }
1745
1746 retval = execute_queues_cpsch(dqm, filter, 0);
1747 if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
1748 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
1749 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
1750 qpd->reset_wavefronts = false;
1751 }
1752
1753 dqm_unlock(dqm);
1754
1755 /* Outside the DQM lock because under the DQM lock we can't do
1756 * reclaim or take other locks that others hold while reclaiming.
1757 */
1758 if (found)
1759 kfd_dec_compute_active(dqm->dev);
1760
1761 /* Lastly, free mqd resources.
1762 * Do free_mqd() after dqm_unlock to avoid circular locking.
1763 */
1764 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1765 mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type(
1766 q->properties.type)];
1767 list_del(&q->list);
1768 qpd->queue_count--;
1769 mqd_mgr->free_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
1770 }
1771
1772 return retval;
1773 }
1774
1775 static int init_mqd_managers(struct device_queue_manager *dqm)
1776 {
1777 int i, j;
1778 struct mqd_manager *mqd_mgr;
1779
1780 for (i = 0; i < KFD_MQD_TYPE_MAX; i++) {
1781 mqd_mgr = dqm->asic_ops.mqd_manager_init(i, dqm->dev);
1782 if (!mqd_mgr) {
1783 pr_err("mqd manager [%d] initialization failed\n", i);
1784 goto out_free;
1785 }
1786 dqm->mqd_mgrs[i] = mqd_mgr;
1787 }
1788
1789 return 0;
1790
1791 out_free:
1792 for (j = 0; j < i; j++) {
1793 kfree(dqm->mqd_mgrs[j]);
1794 dqm->mqd_mgrs[j] = NULL;
1795 }
1796
1797 return -ENOMEM;
1798 }
1799
1800 /* Allocate one hiq mqd (HWS) and all SDMA mqd in a continuous trunk*/
1801 static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm)
1802 {
1803 int retval;
1804 struct kfd_dev *dev = dqm->dev;
1805 struct kfd_mem_obj *mem_obj = &dqm->hiq_sdma_mqd;
1806 uint32_t size = dqm->mqd_mgrs[KFD_MQD_TYPE_SDMA]->mqd_size *
1807 get_num_all_sdma_engines(dqm) *
1808 dev->device_info->num_sdma_queues_per_engine +
1809 dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size;
1810
1811 retval = amdgpu_amdkfd_alloc_gtt_mem(dev->kgd, size,
1812 &(mem_obj->gtt_mem), &(mem_obj->gpu_addr),
1813 (void *)&(mem_obj->cpu_ptr), false);
1814
1815 return retval;
1816 }
1817
1818 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1819 {
1820 struct device_queue_manager *dqm;
1821
1822 pr_debug("Loading device queue manager\n");
1823
1824 dqm = kzalloc(sizeof(*dqm), GFP_KERNEL);
1825 if (!dqm)
1826 return NULL;
1827
1828 switch (dev->device_info->asic_family) {
1829 /* HWS is not available on Hawaii. */
1830 case CHIP_HAWAII:
1831 /* HWS depends on CWSR for timely dequeue. CWSR is not
1832 * available on Tonga.
1833 *
1834 * FIXME: This argument also applies to Kaveri.
1835 */
1836 case CHIP_TONGA:
1837 dqm->sched_policy = KFD_SCHED_POLICY_NO_HWS;
1838 break;
1839 default:
1840 dqm->sched_policy = sched_policy;
1841 break;
1842 }
1843
1844 dqm->dev = dev;
1845 switch (dqm->sched_policy) {
1846 case KFD_SCHED_POLICY_HWS:
1847 case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1848 /* initialize dqm for cp scheduling */
1849 dqm->ops.create_queue = create_queue_cpsch;
1850 dqm->ops.initialize = initialize_cpsch;
1851 dqm->ops.start = start_cpsch;
1852 dqm->ops.stop = stop_cpsch;
1853 dqm->ops.pre_reset = pre_reset;
1854 dqm->ops.destroy_queue = destroy_queue_cpsch;
1855 dqm->ops.update_queue = update_queue;
1856 dqm->ops.register_process = register_process;
1857 dqm->ops.unregister_process = unregister_process;
1858 dqm->ops.uninitialize = uninitialize;
1859 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1860 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1861 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1862 dqm->ops.set_trap_handler = set_trap_handler;
1863 dqm->ops.process_termination = process_termination_cpsch;
1864 dqm->ops.evict_process_queues = evict_process_queues_cpsch;
1865 dqm->ops.restore_process_queues = restore_process_queues_cpsch;
1866 dqm->ops.get_wave_state = get_wave_state;
1867 break;
1868 case KFD_SCHED_POLICY_NO_HWS:
1869 /* initialize dqm for no cp scheduling */
1870 dqm->ops.start = start_nocpsch;
1871 dqm->ops.stop = stop_nocpsch;
1872 dqm->ops.pre_reset = pre_reset;
1873 dqm->ops.create_queue = create_queue_nocpsch;
1874 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1875 dqm->ops.update_queue = update_queue;
1876 dqm->ops.register_process = register_process;
1877 dqm->ops.unregister_process = unregister_process;
1878 dqm->ops.initialize = initialize_nocpsch;
1879 dqm->ops.uninitialize = uninitialize;
1880 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1881 dqm->ops.set_trap_handler = set_trap_handler;
1882 dqm->ops.process_termination = process_termination_nocpsch;
1883 dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
1884 dqm->ops.restore_process_queues =
1885 restore_process_queues_nocpsch;
1886 dqm->ops.get_wave_state = get_wave_state;
1887 break;
1888 default:
1889 pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
1890 goto out_free;
1891 }
1892
1893 switch (dev->device_info->asic_family) {
1894 case CHIP_CARRIZO:
1895 device_queue_manager_init_vi(&dqm->asic_ops);
1896 break;
1897
1898 case CHIP_KAVERI:
1899 device_queue_manager_init_cik(&dqm->asic_ops);
1900 break;
1901
1902 case CHIP_HAWAII:
1903 device_queue_manager_init_cik_hawaii(&dqm->asic_ops);
1904 break;
1905
1906 case CHIP_TONGA:
1907 case CHIP_FIJI:
1908 case CHIP_POLARIS10:
1909 case CHIP_POLARIS11:
1910 case CHIP_POLARIS12:
1911 case CHIP_VEGAM:
1912 device_queue_manager_init_vi_tonga(&dqm->asic_ops);
1913 break;
1914
1915 case CHIP_VEGA10:
1916 case CHIP_VEGA12:
1917 case CHIP_VEGA20:
1918 case CHIP_RAVEN:
1919 case CHIP_RENOIR:
1920 case CHIP_ARCTURUS:
1921 device_queue_manager_init_v9(&dqm->asic_ops);
1922 break;
1923 case CHIP_NAVI10:
1924 case CHIP_NAVI12:
1925 case CHIP_NAVI14:
1926 case CHIP_SIENNA_CICHLID:
1927 case CHIP_NAVY_FLOUNDER:
1928 case CHIP_VANGOGH:
1929 case CHIP_DIMGREY_CAVEFISH:
1930 device_queue_manager_init_v10_navi10(&dqm->asic_ops);
1931 break;
1932 default:
1933 WARN(1, "Unexpected ASIC family %u",
1934 dev->device_info->asic_family);
1935 goto out_free;
1936 }
1937
1938 if (init_mqd_managers(dqm))
1939 goto out_free;
1940
1941 if (allocate_hiq_sdma_mqd(dqm)) {
1942 pr_err("Failed to allocate hiq sdma mqd trunk buffer\n");
1943 goto out_free;
1944 }
1945
1946 if (!dqm->ops.initialize(dqm))
1947 return dqm;
1948
1949 out_free:
1950 kfree(dqm);
1951 return NULL;
1952 }
1953
1954 static void deallocate_hiq_sdma_mqd(struct kfd_dev *dev,
1955 struct kfd_mem_obj *mqd)
1956 {
1957 WARN(!mqd, "No hiq sdma mqd trunk to free");
1958
1959 amdgpu_amdkfd_free_gtt_mem(dev->kgd, mqd->gtt_mem);
1960 }
1961
1962 void device_queue_manager_uninit(struct device_queue_manager *dqm)
1963 {
1964 dqm->ops.uninitialize(dqm);
1965 deallocate_hiq_sdma_mqd(dqm->dev, &dqm->hiq_sdma_mqd);
1966 kfree(dqm);
1967 }
1968
1969 int kfd_process_vm_fault(struct device_queue_manager *dqm, u32 pasid)
1970 {
1971 struct kfd_process_device *pdd;
1972 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1973 int ret = 0;
1974
1975 if (!p)
1976 return -EINVAL;
1977 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
1978 pdd = kfd_get_process_device_data(dqm->dev, p);
1979 if (pdd)
1980 ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
1981 kfd_unref_process(p);
1982
1983 return ret;
1984 }
1985
1986 static void kfd_process_hw_exception(struct work_struct *work)
1987 {
1988 struct device_queue_manager *dqm = container_of(work,
1989 struct device_queue_manager, hw_exception_work);
1990 amdgpu_amdkfd_gpu_reset(dqm->dev->kgd);
1991 }
1992
1993 #if defined(CONFIG_DEBUG_FS)
1994
1995 static void seq_reg_dump(struct seq_file *m,
1996 uint32_t (*dump)[2], uint32_t n_regs)
1997 {
1998 uint32_t i, count;
1999
2000 for (i = 0, count = 0; i < n_regs; i++) {
2001 if (count == 0 ||
2002 dump[i-1][0] + sizeof(uint32_t) != dump[i][0]) {
2003 seq_printf(m, "%s %08x: %08x",
2004 i ? "\n" : "",
2005 dump[i][0], dump[i][1]);
2006 count = 7;
2007 } else {
2008 seq_printf(m, " %08x", dump[i][1]);
2009 count--;
2010 }
2011 }
2012
2013 seq_puts(m, "\n");
2014 }
2015
2016 int dqm_debugfs_hqds(struct seq_file *m, void *data)
2017 {
2018 struct device_queue_manager *dqm = data;
2019 uint32_t (*dump)[2], n_regs;
2020 int pipe, queue;
2021 int r = 0;
2022
2023 if (!dqm->sched_running) {
2024 seq_printf(m, " Device is stopped\n");
2025
2026 return 0;
2027 }
2028
2029 r = dqm->dev->kfd2kgd->hqd_dump(dqm->dev->kgd,
2030 KFD_CIK_HIQ_PIPE, KFD_CIK_HIQ_QUEUE,
2031 &dump, &n_regs);
2032 if (!r) {
2033 seq_printf(m, " HIQ on MEC %d Pipe %d Queue %d\n",
2034 KFD_CIK_HIQ_PIPE/get_pipes_per_mec(dqm)+1,
2035 KFD_CIK_HIQ_PIPE%get_pipes_per_mec(dqm),
2036 KFD_CIK_HIQ_QUEUE);
2037 seq_reg_dump(m, dump, n_regs);
2038
2039 kfree(dump);
2040 }
2041
2042 for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) {
2043 int pipe_offset = pipe * get_queues_per_pipe(dqm);
2044
2045 for (queue = 0; queue < get_queues_per_pipe(dqm); queue++) {
2046 if (!test_bit(pipe_offset + queue,
2047 dqm->dev->shared_resources.cp_queue_bitmap))
2048 continue;
2049
2050 r = dqm->dev->kfd2kgd->hqd_dump(
2051 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
2052 if (r)
2053 break;
2054
2055 seq_printf(m, " CP Pipe %d, Queue %d\n",
2056 pipe, queue);
2057 seq_reg_dump(m, dump, n_regs);
2058
2059 kfree(dump);
2060 }
2061 }
2062
2063 for (pipe = 0; pipe < get_num_all_sdma_engines(dqm); pipe++) {
2064 for (queue = 0;
2065 queue < dqm->dev->device_info->num_sdma_queues_per_engine;
2066 queue++) {
2067 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
2068 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
2069 if (r)
2070 break;
2071
2072 seq_printf(m, " SDMA Engine %d, RLC %d\n",
2073 pipe, queue);
2074 seq_reg_dump(m, dump, n_regs);
2075
2076 kfree(dump);
2077 }
2078 }
2079
2080 return r;
2081 }
2082
2083 int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
2084 {
2085 int r = 0;
2086
2087 dqm_lock(dqm);
2088 dqm->active_runlist = true;
2089 r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
2090 dqm_unlock(dqm);
2091
2092 return r;
2093 }
2094
2095 #endif
Linux with added FPC-III support