2 * Copyright 2014 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
23 #include <linux/types.h>
24 #include <linux/kernel.h>
25 #include <linux/pci.h>
26 #include <linux/errno.h>
27 #include <linux/acpi.h>
28 #include <linux/hash.h>
29 #include <linux/cpufreq.h>
30 #include <linux/log2.h>
31 #include <linux/dmi.h>
32 #include <linux/atomic.h>
36 #include "kfd_topology.h"
37 #include "kfd_device_queue_manager.h"
38 #include "kfd_iommu.h"
39 #include "amdgpu_amdkfd.h"
40 #include "amdgpu_ras.h"
42 /* topology_device_list - Master list of all topology devices */
43 static struct list_head topology_device_list
;
44 static struct kfd_system_properties sys_props
;
46 static DECLARE_RWSEM(topology_lock
);
47 static atomic_t topology_crat_proximity_domain
;
49 struct kfd_topology_device
*kfd_topology_device_by_proximity_domain(
50 uint32_t proximity_domain
)
52 struct kfd_topology_device
*top_dev
;
53 struct kfd_topology_device
*device
= NULL
;
55 down_read(&topology_lock
);
57 list_for_each_entry(top_dev
, &topology_device_list
, list
)
58 if (top_dev
->proximity_domain
== proximity_domain
) {
63 up_read(&topology_lock
);
68 struct kfd_topology_device
*kfd_topology_device_by_id(uint32_t gpu_id
)
70 struct kfd_topology_device
*top_dev
= NULL
;
71 struct kfd_topology_device
*ret
= NULL
;
73 down_read(&topology_lock
);
75 list_for_each_entry(top_dev
, &topology_device_list
, list
)
76 if (top_dev
->gpu_id
== gpu_id
) {
81 up_read(&topology_lock
);
86 struct kfd_dev
*kfd_device_by_id(uint32_t gpu_id
)
88 struct kfd_topology_device
*top_dev
;
90 top_dev
= kfd_topology_device_by_id(gpu_id
);
97 struct kfd_dev
*kfd_device_by_pci_dev(const struct pci_dev
*pdev
)
99 struct kfd_topology_device
*top_dev
;
100 struct kfd_dev
*device
= NULL
;
102 down_read(&topology_lock
);
104 list_for_each_entry(top_dev
, &topology_device_list
, list
)
105 if (top_dev
->gpu
&& top_dev
->gpu
->pdev
== pdev
) {
106 device
= top_dev
->gpu
;
110 up_read(&topology_lock
);
115 struct kfd_dev
*kfd_device_by_kgd(const struct kgd_dev
*kgd
)
117 struct kfd_topology_device
*top_dev
;
118 struct kfd_dev
*device
= NULL
;
120 down_read(&topology_lock
);
122 list_for_each_entry(top_dev
, &topology_device_list
, list
)
123 if (top_dev
->gpu
&& top_dev
->gpu
->kgd
== kgd
) {
124 device
= top_dev
->gpu
;
128 up_read(&topology_lock
);
133 /* Called with write topology_lock acquired */
134 static void kfd_release_topology_device(struct kfd_topology_device
*dev
)
136 struct kfd_mem_properties
*mem
;
137 struct kfd_cache_properties
*cache
;
138 struct kfd_iolink_properties
*iolink
;
139 struct kfd_perf_properties
*perf
;
141 list_del(&dev
->list
);
143 while (dev
->mem_props
.next
!= &dev
->mem_props
) {
144 mem
= container_of(dev
->mem_props
.next
,
145 struct kfd_mem_properties
, list
);
146 list_del(&mem
->list
);
150 while (dev
->cache_props
.next
!= &dev
->cache_props
) {
151 cache
= container_of(dev
->cache_props
.next
,
152 struct kfd_cache_properties
, list
);
153 list_del(&cache
->list
);
157 while (dev
->io_link_props
.next
!= &dev
->io_link_props
) {
158 iolink
= container_of(dev
->io_link_props
.next
,
159 struct kfd_iolink_properties
, list
);
160 list_del(&iolink
->list
);
164 while (dev
->perf_props
.next
!= &dev
->perf_props
) {
165 perf
= container_of(dev
->perf_props
.next
,
166 struct kfd_perf_properties
, list
);
167 list_del(&perf
->list
);
174 void kfd_release_topology_device_list(struct list_head
*device_list
)
176 struct kfd_topology_device
*dev
;
178 while (!list_empty(device_list
)) {
179 dev
= list_first_entry(device_list
,
180 struct kfd_topology_device
, list
);
181 kfd_release_topology_device(dev
);
185 static void kfd_release_live_view(void)
187 kfd_release_topology_device_list(&topology_device_list
);
188 memset(&sys_props
, 0, sizeof(sys_props
));
191 struct kfd_topology_device
*kfd_create_topology_device(
192 struct list_head
*device_list
)
194 struct kfd_topology_device
*dev
;
196 dev
= kfd_alloc_struct(dev
);
198 pr_err("No memory to allocate a topology device");
202 INIT_LIST_HEAD(&dev
->mem_props
);
203 INIT_LIST_HEAD(&dev
->cache_props
);
204 INIT_LIST_HEAD(&dev
->io_link_props
);
205 INIT_LIST_HEAD(&dev
->perf_props
);
207 list_add_tail(&dev
->list
, device_list
);
213 #define sysfs_show_gen_prop(buffer, offs, fmt, ...) \
214 (offs += snprintf(buffer+offs, PAGE_SIZE-offs, \
216 #define sysfs_show_32bit_prop(buffer, offs, name, value) \
217 sysfs_show_gen_prop(buffer, offs, "%s %u\n", name, value)
218 #define sysfs_show_64bit_prop(buffer, offs, name, value) \
219 sysfs_show_gen_prop(buffer, offs, "%s %llu\n", name, value)
220 #define sysfs_show_32bit_val(buffer, offs, value) \
221 sysfs_show_gen_prop(buffer, offs, "%u\n", value)
222 #define sysfs_show_str_val(buffer, offs, value) \
223 sysfs_show_gen_prop(buffer, offs, "%s\n", value)
225 static ssize_t
sysprops_show(struct kobject
*kobj
, struct attribute
*attr
,
230 /* Making sure that the buffer is an empty string */
233 if (attr
== &sys_props
.attr_genid
) {
234 sysfs_show_32bit_val(buffer
, offs
,
235 sys_props
.generation_count
);
236 } else if (attr
== &sys_props
.attr_props
) {
237 sysfs_show_64bit_prop(buffer
, offs
, "platform_oem",
238 sys_props
.platform_oem
);
239 sysfs_show_64bit_prop(buffer
, offs
, "platform_id",
240 sys_props
.platform_id
);
241 sysfs_show_64bit_prop(buffer
, offs
, "platform_rev",
242 sys_props
.platform_rev
);
250 static void kfd_topology_kobj_release(struct kobject
*kobj
)
255 static const struct sysfs_ops sysprops_ops
= {
256 .show
= sysprops_show
,
259 static struct kobj_type sysprops_type
= {
260 .release
= kfd_topology_kobj_release
,
261 .sysfs_ops
= &sysprops_ops
,
264 static ssize_t
iolink_show(struct kobject
*kobj
, struct attribute
*attr
,
268 struct kfd_iolink_properties
*iolink
;
270 /* Making sure that the buffer is an empty string */
273 iolink
= container_of(attr
, struct kfd_iolink_properties
, attr
);
274 if (iolink
->gpu
&& kfd_devcgroup_check_permission(iolink
->gpu
))
276 sysfs_show_32bit_prop(buffer
, offs
, "type", iolink
->iolink_type
);
277 sysfs_show_32bit_prop(buffer
, offs
, "version_major", iolink
->ver_maj
);
278 sysfs_show_32bit_prop(buffer
, offs
, "version_minor", iolink
->ver_min
);
279 sysfs_show_32bit_prop(buffer
, offs
, "node_from", iolink
->node_from
);
280 sysfs_show_32bit_prop(buffer
, offs
, "node_to", iolink
->node_to
);
281 sysfs_show_32bit_prop(buffer
, offs
, "weight", iolink
->weight
);
282 sysfs_show_32bit_prop(buffer
, offs
, "min_latency", iolink
->min_latency
);
283 sysfs_show_32bit_prop(buffer
, offs
, "max_latency", iolink
->max_latency
);
284 sysfs_show_32bit_prop(buffer
, offs
, "min_bandwidth",
285 iolink
->min_bandwidth
);
286 sysfs_show_32bit_prop(buffer
, offs
, "max_bandwidth",
287 iolink
->max_bandwidth
);
288 sysfs_show_32bit_prop(buffer
, offs
, "recommended_transfer_size",
289 iolink
->rec_transfer_size
);
290 sysfs_show_32bit_prop(buffer
, offs
, "flags", iolink
->flags
);
295 static const struct sysfs_ops iolink_ops
= {
299 static struct kobj_type iolink_type
= {
300 .release
= kfd_topology_kobj_release
,
301 .sysfs_ops
= &iolink_ops
,
304 static ssize_t
mem_show(struct kobject
*kobj
, struct attribute
*attr
,
308 struct kfd_mem_properties
*mem
;
310 /* Making sure that the buffer is an empty string */
313 mem
= container_of(attr
, struct kfd_mem_properties
, attr
);
314 if (mem
->gpu
&& kfd_devcgroup_check_permission(mem
->gpu
))
316 sysfs_show_32bit_prop(buffer
, offs
, "heap_type", mem
->heap_type
);
317 sysfs_show_64bit_prop(buffer
, offs
, "size_in_bytes",
319 sysfs_show_32bit_prop(buffer
, offs
, "flags", mem
->flags
);
320 sysfs_show_32bit_prop(buffer
, offs
, "width", mem
->width
);
321 sysfs_show_32bit_prop(buffer
, offs
, "mem_clk_max",
327 static const struct sysfs_ops mem_ops
= {
331 static struct kobj_type mem_type
= {
332 .release
= kfd_topology_kobj_release
,
333 .sysfs_ops
= &mem_ops
,
336 static ssize_t
kfd_cache_show(struct kobject
*kobj
, struct attribute
*attr
,
341 struct kfd_cache_properties
*cache
;
343 /* Making sure that the buffer is an empty string */
346 cache
= container_of(attr
, struct kfd_cache_properties
, attr
);
347 if (cache
->gpu
&& kfd_devcgroup_check_permission(cache
->gpu
))
349 sysfs_show_32bit_prop(buffer
, offs
, "processor_id_low",
350 cache
->processor_id_low
);
351 sysfs_show_32bit_prop(buffer
, offs
, "level", cache
->cache_level
);
352 sysfs_show_32bit_prop(buffer
, offs
, "size", cache
->cache_size
);
353 sysfs_show_32bit_prop(buffer
, offs
, "cache_line_size",
354 cache
->cacheline_size
);
355 sysfs_show_32bit_prop(buffer
, offs
, "cache_lines_per_tag",
356 cache
->cachelines_per_tag
);
357 sysfs_show_32bit_prop(buffer
, offs
, "association", cache
->cache_assoc
);
358 sysfs_show_32bit_prop(buffer
, offs
, "latency", cache
->cache_latency
);
359 sysfs_show_32bit_prop(buffer
, offs
, "type", cache
->cache_type
);
360 offs
+= snprintf(buffer
+offs
, PAGE_SIZE
-offs
, "sibling_map ");
361 for (i
= 0; i
< CRAT_SIBLINGMAP_SIZE
; i
++)
362 for (j
= 0; j
< sizeof(cache
->sibling_map
[0])*8; j
++)
364 offs
+= snprintf(buffer
+offs
, PAGE_SIZE
-offs
, "%d,",
365 (cache
->sibling_map
[i
] >> j
) & 1);
367 /* Replace the last "," with end of line */
368 buffer
[offs
-1] = '\n';
372 static const struct sysfs_ops cache_ops
= {
373 .show
= kfd_cache_show
,
376 static struct kobj_type cache_type
= {
377 .release
= kfd_topology_kobj_release
,
378 .sysfs_ops
= &cache_ops
,
381 /****** Sysfs of Performance Counters ******/
383 struct kfd_perf_attr
{
384 struct kobj_attribute attr
;
388 static ssize_t
perf_show(struct kobject
*kobj
, struct kobj_attribute
*attrs
,
392 struct kfd_perf_attr
*attr
;
395 attr
= container_of(attrs
, struct kfd_perf_attr
, attr
);
396 if (!attr
->data
) /* invalid data for PMC */
399 return sysfs_show_32bit_val(buf
, offs
, attr
->data
);
402 #define KFD_PERF_DESC(_name, _data) \
404 .attr = __ATTR(_name, 0444, perf_show, NULL), \
408 static struct kfd_perf_attr perf_attr_iommu
[] = {
409 KFD_PERF_DESC(max_concurrent
, 0),
410 KFD_PERF_DESC(num_counters
, 0),
411 KFD_PERF_DESC(counter_ids
, 0),
413 /****************************************/
415 static ssize_t
node_show(struct kobject
*kobj
, struct attribute
*attr
,
419 struct kfd_topology_device
*dev
;
420 uint32_t log_max_watch_addr
;
422 /* Making sure that the buffer is an empty string */
425 if (strcmp(attr
->name
, "gpu_id") == 0) {
426 dev
= container_of(attr
, struct kfd_topology_device
,
428 if (dev
->gpu
&& kfd_devcgroup_check_permission(dev
->gpu
))
430 return sysfs_show_32bit_val(buffer
, offs
, dev
->gpu_id
);
433 if (strcmp(attr
->name
, "name") == 0) {
434 dev
= container_of(attr
, struct kfd_topology_device
,
437 if (dev
->gpu
&& kfd_devcgroup_check_permission(dev
->gpu
))
439 return sysfs_show_str_val(buffer
, offs
, dev
->node_props
.name
);
442 dev
= container_of(attr
, struct kfd_topology_device
,
444 if (dev
->gpu
&& kfd_devcgroup_check_permission(dev
->gpu
))
446 sysfs_show_32bit_prop(buffer
, offs
, "cpu_cores_count",
447 dev
->node_props
.cpu_cores_count
);
448 sysfs_show_32bit_prop(buffer
, offs
, "simd_count",
449 dev
->gpu
? dev
->node_props
.simd_count
: 0);
450 sysfs_show_32bit_prop(buffer
, offs
, "mem_banks_count",
451 dev
->node_props
.mem_banks_count
);
452 sysfs_show_32bit_prop(buffer
, offs
, "caches_count",
453 dev
->node_props
.caches_count
);
454 sysfs_show_32bit_prop(buffer
, offs
, "io_links_count",
455 dev
->node_props
.io_links_count
);
456 sysfs_show_32bit_prop(buffer
, offs
, "cpu_core_id_base",
457 dev
->node_props
.cpu_core_id_base
);
458 sysfs_show_32bit_prop(buffer
, offs
, "simd_id_base",
459 dev
->node_props
.simd_id_base
);
460 sysfs_show_32bit_prop(buffer
, offs
, "max_waves_per_simd",
461 dev
->node_props
.max_waves_per_simd
);
462 sysfs_show_32bit_prop(buffer
, offs
, "lds_size_in_kb",
463 dev
->node_props
.lds_size_in_kb
);
464 sysfs_show_32bit_prop(buffer
, offs
, "gds_size_in_kb",
465 dev
->node_props
.gds_size_in_kb
);
466 sysfs_show_32bit_prop(buffer
, offs
, "num_gws",
467 dev
->node_props
.num_gws
);
468 sysfs_show_32bit_prop(buffer
, offs
, "wave_front_size",
469 dev
->node_props
.wave_front_size
);
470 sysfs_show_32bit_prop(buffer
, offs
, "array_count",
471 dev
->node_props
.array_count
);
472 sysfs_show_32bit_prop(buffer
, offs
, "simd_arrays_per_engine",
473 dev
->node_props
.simd_arrays_per_engine
);
474 sysfs_show_32bit_prop(buffer
, offs
, "cu_per_simd_array",
475 dev
->node_props
.cu_per_simd_array
);
476 sysfs_show_32bit_prop(buffer
, offs
, "simd_per_cu",
477 dev
->node_props
.simd_per_cu
);
478 sysfs_show_32bit_prop(buffer
, offs
, "max_slots_scratch_cu",
479 dev
->node_props
.max_slots_scratch_cu
);
480 sysfs_show_32bit_prop(buffer
, offs
, "vendor_id",
481 dev
->node_props
.vendor_id
);
482 sysfs_show_32bit_prop(buffer
, offs
, "device_id",
483 dev
->node_props
.device_id
);
484 sysfs_show_32bit_prop(buffer
, offs
, "location_id",
485 dev
->node_props
.location_id
);
486 sysfs_show_32bit_prop(buffer
, offs
, "domain",
487 dev
->node_props
.domain
);
488 sysfs_show_32bit_prop(buffer
, offs
, "drm_render_minor",
489 dev
->node_props
.drm_render_minor
);
490 sysfs_show_64bit_prop(buffer
, offs
, "hive_id",
491 dev
->node_props
.hive_id
);
492 sysfs_show_32bit_prop(buffer
, offs
, "num_sdma_engines",
493 dev
->node_props
.num_sdma_engines
);
494 sysfs_show_32bit_prop(buffer
, offs
, "num_sdma_xgmi_engines",
495 dev
->node_props
.num_sdma_xgmi_engines
);
496 sysfs_show_32bit_prop(buffer
, offs
, "num_sdma_queues_per_engine",
497 dev
->node_props
.num_sdma_queues_per_engine
);
498 sysfs_show_32bit_prop(buffer
, offs
, "num_cp_queues",
499 dev
->node_props
.num_cp_queues
);
500 sysfs_show_64bit_prop(buffer
, offs
, "unique_id",
501 dev
->node_props
.unique_id
);
505 __ilog2_u32(dev
->gpu
->device_info
->num_of_watch_points
);
507 if (log_max_watch_addr
) {
508 dev
->node_props
.capability
|=
509 HSA_CAP_WATCH_POINTS_SUPPORTED
;
511 dev
->node_props
.capability
|=
512 ((log_max_watch_addr
<<
513 HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT
) &
514 HSA_CAP_WATCH_POINTS_TOTALBITS_MASK
);
517 if (dev
->gpu
->device_info
->asic_family
== CHIP_TONGA
)
518 dev
->node_props
.capability
|=
519 HSA_CAP_AQL_QUEUE_DOUBLE_MAP
;
521 sysfs_show_32bit_prop(buffer
, offs
, "max_engine_clk_fcompute",
522 dev
->node_props
.max_engine_clk_fcompute
);
524 sysfs_show_64bit_prop(buffer
, offs
, "local_mem_size", 0ULL);
526 sysfs_show_32bit_prop(buffer
, offs
, "fw_version",
527 dev
->gpu
->mec_fw_version
);
528 sysfs_show_32bit_prop(buffer
, offs
, "capability",
529 dev
->node_props
.capability
);
530 sysfs_show_32bit_prop(buffer
, offs
, "sdma_fw_version",
531 dev
->gpu
->sdma_fw_version
);
534 return sysfs_show_32bit_prop(buffer
, offs
, "max_engine_clk_ccompute",
535 cpufreq_quick_get_max(0)/1000);
538 static const struct sysfs_ops node_ops
= {
542 static struct kobj_type node_type
= {
543 .release
= kfd_topology_kobj_release
,
544 .sysfs_ops
= &node_ops
,
547 static void kfd_remove_sysfs_file(struct kobject
*kobj
, struct attribute
*attr
)
549 sysfs_remove_file(kobj
, attr
);
554 static void kfd_remove_sysfs_node_entry(struct kfd_topology_device
*dev
)
556 struct kfd_iolink_properties
*iolink
;
557 struct kfd_cache_properties
*cache
;
558 struct kfd_mem_properties
*mem
;
559 struct kfd_perf_properties
*perf
;
561 if (dev
->kobj_iolink
) {
562 list_for_each_entry(iolink
, &dev
->io_link_props
, list
)
564 kfd_remove_sysfs_file(iolink
->kobj
,
568 kobject_del(dev
->kobj_iolink
);
569 kobject_put(dev
->kobj_iolink
);
570 dev
->kobj_iolink
= NULL
;
573 if (dev
->kobj_cache
) {
574 list_for_each_entry(cache
, &dev
->cache_props
, list
)
576 kfd_remove_sysfs_file(cache
->kobj
,
580 kobject_del(dev
->kobj_cache
);
581 kobject_put(dev
->kobj_cache
);
582 dev
->kobj_cache
= NULL
;
586 list_for_each_entry(mem
, &dev
->mem_props
, list
)
588 kfd_remove_sysfs_file(mem
->kobj
, &mem
->attr
);
591 kobject_del(dev
->kobj_mem
);
592 kobject_put(dev
->kobj_mem
);
593 dev
->kobj_mem
= NULL
;
596 if (dev
->kobj_perf
) {
597 list_for_each_entry(perf
, &dev
->perf_props
, list
) {
598 kfree(perf
->attr_group
);
599 perf
->attr_group
= NULL
;
601 kobject_del(dev
->kobj_perf
);
602 kobject_put(dev
->kobj_perf
);
603 dev
->kobj_perf
= NULL
;
606 if (dev
->kobj_node
) {
607 sysfs_remove_file(dev
->kobj_node
, &dev
->attr_gpuid
);
608 sysfs_remove_file(dev
->kobj_node
, &dev
->attr_name
);
609 sysfs_remove_file(dev
->kobj_node
, &dev
->attr_props
);
610 kobject_del(dev
->kobj_node
);
611 kobject_put(dev
->kobj_node
);
612 dev
->kobj_node
= NULL
;
616 static int kfd_build_sysfs_node_entry(struct kfd_topology_device
*dev
,
619 struct kfd_iolink_properties
*iolink
;
620 struct kfd_cache_properties
*cache
;
621 struct kfd_mem_properties
*mem
;
622 struct kfd_perf_properties
*perf
;
624 uint32_t i
, num_attrs
;
625 struct attribute
**attrs
;
627 if (WARN_ON(dev
->kobj_node
))
631 * Creating the sysfs folders
633 dev
->kobj_node
= kfd_alloc_struct(dev
->kobj_node
);
637 ret
= kobject_init_and_add(dev
->kobj_node
, &node_type
,
638 sys_props
.kobj_nodes
, "%d", id
);
640 kobject_put(dev
->kobj_node
);
644 dev
->kobj_mem
= kobject_create_and_add("mem_banks", dev
->kobj_node
);
648 dev
->kobj_cache
= kobject_create_and_add("caches", dev
->kobj_node
);
649 if (!dev
->kobj_cache
)
652 dev
->kobj_iolink
= kobject_create_and_add("io_links", dev
->kobj_node
);
653 if (!dev
->kobj_iolink
)
656 dev
->kobj_perf
= kobject_create_and_add("perf", dev
->kobj_node
);
661 * Creating sysfs files for node properties
663 dev
->attr_gpuid
.name
= "gpu_id";
664 dev
->attr_gpuid
.mode
= KFD_SYSFS_FILE_MODE
;
665 sysfs_attr_init(&dev
->attr_gpuid
);
666 dev
->attr_name
.name
= "name";
667 dev
->attr_name
.mode
= KFD_SYSFS_FILE_MODE
;
668 sysfs_attr_init(&dev
->attr_name
);
669 dev
->attr_props
.name
= "properties";
670 dev
->attr_props
.mode
= KFD_SYSFS_FILE_MODE
;
671 sysfs_attr_init(&dev
->attr_props
);
672 ret
= sysfs_create_file(dev
->kobj_node
, &dev
->attr_gpuid
);
675 ret
= sysfs_create_file(dev
->kobj_node
, &dev
->attr_name
);
678 ret
= sysfs_create_file(dev
->kobj_node
, &dev
->attr_props
);
683 list_for_each_entry(mem
, &dev
->mem_props
, list
) {
684 mem
->kobj
= kzalloc(sizeof(struct kobject
), GFP_KERNEL
);
687 ret
= kobject_init_and_add(mem
->kobj
, &mem_type
,
688 dev
->kobj_mem
, "%d", i
);
690 kobject_put(mem
->kobj
);
694 mem
->attr
.name
= "properties";
695 mem
->attr
.mode
= KFD_SYSFS_FILE_MODE
;
696 sysfs_attr_init(&mem
->attr
);
697 ret
= sysfs_create_file(mem
->kobj
, &mem
->attr
);
704 list_for_each_entry(cache
, &dev
->cache_props
, list
) {
705 cache
->kobj
= kzalloc(sizeof(struct kobject
), GFP_KERNEL
);
708 ret
= kobject_init_and_add(cache
->kobj
, &cache_type
,
709 dev
->kobj_cache
, "%d", i
);
711 kobject_put(cache
->kobj
);
715 cache
->attr
.name
= "properties";
716 cache
->attr
.mode
= KFD_SYSFS_FILE_MODE
;
717 sysfs_attr_init(&cache
->attr
);
718 ret
= sysfs_create_file(cache
->kobj
, &cache
->attr
);
725 list_for_each_entry(iolink
, &dev
->io_link_props
, list
) {
726 iolink
->kobj
= kzalloc(sizeof(struct kobject
), GFP_KERNEL
);
729 ret
= kobject_init_and_add(iolink
->kobj
, &iolink_type
,
730 dev
->kobj_iolink
, "%d", i
);
732 kobject_put(iolink
->kobj
);
736 iolink
->attr
.name
= "properties";
737 iolink
->attr
.mode
= KFD_SYSFS_FILE_MODE
;
738 sysfs_attr_init(&iolink
->attr
);
739 ret
= sysfs_create_file(iolink
->kobj
, &iolink
->attr
);
745 /* All hardware blocks have the same number of attributes. */
746 num_attrs
= ARRAY_SIZE(perf_attr_iommu
);
747 list_for_each_entry(perf
, &dev
->perf_props
, list
) {
748 perf
->attr_group
= kzalloc(sizeof(struct kfd_perf_attr
)
749 * num_attrs
+ sizeof(struct attribute_group
),
751 if (!perf
->attr_group
)
754 attrs
= (struct attribute
**)(perf
->attr_group
+ 1);
755 if (!strcmp(perf
->block_name
, "iommu")) {
756 /* Information of IOMMU's num_counters and counter_ids is shown
757 * under /sys/bus/event_source/devices/amd_iommu. We don't
760 perf_attr_iommu
[0].data
= perf
->max_concurrent
;
761 for (i
= 0; i
< num_attrs
; i
++)
762 attrs
[i
] = &perf_attr_iommu
[i
].attr
.attr
;
764 perf
->attr_group
->name
= perf
->block_name
;
765 perf
->attr_group
->attrs
= attrs
;
766 ret
= sysfs_create_group(dev
->kobj_perf
, perf
->attr_group
);
774 /* Called with write topology lock acquired */
775 static int kfd_build_sysfs_node_tree(void)
777 struct kfd_topology_device
*dev
;
781 list_for_each_entry(dev
, &topology_device_list
, list
) {
782 ret
= kfd_build_sysfs_node_entry(dev
, i
);
791 /* Called with write topology lock acquired */
792 static void kfd_remove_sysfs_node_tree(void)
794 struct kfd_topology_device
*dev
;
796 list_for_each_entry(dev
, &topology_device_list
, list
)
797 kfd_remove_sysfs_node_entry(dev
);
800 static int kfd_topology_update_sysfs(void)
804 if (!sys_props
.kobj_topology
) {
805 sys_props
.kobj_topology
=
806 kfd_alloc_struct(sys_props
.kobj_topology
);
807 if (!sys_props
.kobj_topology
)
810 ret
= kobject_init_and_add(sys_props
.kobj_topology
,
811 &sysprops_type
, &kfd_device
->kobj
,
814 kobject_put(sys_props
.kobj_topology
);
818 sys_props
.kobj_nodes
= kobject_create_and_add("nodes",
819 sys_props
.kobj_topology
);
820 if (!sys_props
.kobj_nodes
)
823 sys_props
.attr_genid
.name
= "generation_id";
824 sys_props
.attr_genid
.mode
= KFD_SYSFS_FILE_MODE
;
825 sysfs_attr_init(&sys_props
.attr_genid
);
826 ret
= sysfs_create_file(sys_props
.kobj_topology
,
827 &sys_props
.attr_genid
);
831 sys_props
.attr_props
.name
= "system_properties";
832 sys_props
.attr_props
.mode
= KFD_SYSFS_FILE_MODE
;
833 sysfs_attr_init(&sys_props
.attr_props
);
834 ret
= sysfs_create_file(sys_props
.kobj_topology
,
835 &sys_props
.attr_props
);
840 kfd_remove_sysfs_node_tree();
842 return kfd_build_sysfs_node_tree();
845 static void kfd_topology_release_sysfs(void)
847 kfd_remove_sysfs_node_tree();
848 if (sys_props
.kobj_topology
) {
849 sysfs_remove_file(sys_props
.kobj_topology
,
850 &sys_props
.attr_genid
);
851 sysfs_remove_file(sys_props
.kobj_topology
,
852 &sys_props
.attr_props
);
853 if (sys_props
.kobj_nodes
) {
854 kobject_del(sys_props
.kobj_nodes
);
855 kobject_put(sys_props
.kobj_nodes
);
856 sys_props
.kobj_nodes
= NULL
;
858 kobject_del(sys_props
.kobj_topology
);
859 kobject_put(sys_props
.kobj_topology
);
860 sys_props
.kobj_topology
= NULL
;
864 /* Called with write topology_lock acquired */
865 static void kfd_topology_update_device_list(struct list_head
*temp_list
,
866 struct list_head
*master_list
)
868 while (!list_empty(temp_list
)) {
869 list_move_tail(temp_list
->next
, master_list
);
870 sys_props
.num_devices
++;
874 static void kfd_debug_print_topology(void)
876 struct kfd_topology_device
*dev
;
878 down_read(&topology_lock
);
880 dev
= list_last_entry(&topology_device_list
,
881 struct kfd_topology_device
, list
);
883 if (dev
->node_props
.cpu_cores_count
&&
884 dev
->node_props
.simd_count
) {
885 pr_info("Topology: Add APU node [0x%0x:0x%0x]\n",
886 dev
->node_props
.device_id
,
887 dev
->node_props
.vendor_id
);
888 } else if (dev
->node_props
.cpu_cores_count
)
889 pr_info("Topology: Add CPU node\n");
890 else if (dev
->node_props
.simd_count
)
891 pr_info("Topology: Add dGPU node [0x%0x:0x%0x]\n",
892 dev
->node_props
.device_id
,
893 dev
->node_props
.vendor_id
);
895 up_read(&topology_lock
);
898 /* Helper function for intializing platform_xx members of
899 * kfd_system_properties. Uses OEM info from the last CPU/APU node.
901 static void kfd_update_system_properties(void)
903 struct kfd_topology_device
*dev
;
905 down_read(&topology_lock
);
906 dev
= list_last_entry(&topology_device_list
,
907 struct kfd_topology_device
, list
);
909 sys_props
.platform_id
=
910 (*((uint64_t *)dev
->oem_id
)) & CRAT_OEMID_64BIT_MASK
;
911 sys_props
.platform_oem
= *((uint64_t *)dev
->oem_table_id
);
912 sys_props
.platform_rev
= dev
->oem_revision
;
914 up_read(&topology_lock
);
917 static void find_system_memory(const struct dmi_header
*dm
,
920 struct kfd_mem_properties
*mem
;
921 u16 mem_width
, mem_clock
;
922 struct kfd_topology_device
*kdev
=
923 (struct kfd_topology_device
*)private;
924 const u8
*dmi_data
= (const u8
*)(dm
+ 1);
926 if (dm
->type
== DMI_ENTRY_MEM_DEVICE
&& dm
->length
>= 0x15) {
927 mem_width
= (u16
)(*(const u16
*)(dmi_data
+ 0x6));
928 mem_clock
= (u16
)(*(const u16
*)(dmi_data
+ 0x11));
929 list_for_each_entry(mem
, &kdev
->mem_props
, list
) {
930 if (mem_width
!= 0xFFFF && mem_width
!= 0)
931 mem
->width
= mem_width
;
933 mem
->mem_clk_max
= mem_clock
;
939 * Performance counters information is not part of CRAT but we would like to
940 * put them in the sysfs under topology directory for Thunk to get the data.
941 * This function is called before updating the sysfs.
943 static int kfd_add_perf_to_topology(struct kfd_topology_device
*kdev
)
945 /* These are the only counters supported so far */
946 return kfd_iommu_add_perf_counters(kdev
);
949 /* kfd_add_non_crat_information - Add information that is not currently
950 * defined in CRAT but is necessary for KFD topology
951 * @dev - topology device to which addition info is added
953 static void kfd_add_non_crat_information(struct kfd_topology_device
*kdev
)
955 /* Check if CPU only node. */
957 /* Add system memory information */
958 dmi_walk(find_system_memory
, kdev
);
960 /* TODO: For GPU node, rearrange code from kfd_topology_add_device */
963 /* kfd_is_acpi_crat_invalid - CRAT from ACPI is valid only for AMD APU devices.
964 * Ignore CRAT for all other devices. AMD APU is identified if both CPU
965 * and GPU cores are present.
966 * @device_list - topology device list created by parsing ACPI CRAT table.
967 * @return - TRUE if invalid, FALSE is valid.
969 static bool kfd_is_acpi_crat_invalid(struct list_head
*device_list
)
971 struct kfd_topology_device
*dev
;
973 list_for_each_entry(dev
, device_list
, list
) {
974 if (dev
->node_props
.cpu_cores_count
&&
975 dev
->node_props
.simd_count
)
978 pr_info("Ignoring ACPI CRAT on non-APU system\n");
982 int kfd_topology_init(void)
984 void *crat_image
= NULL
;
985 size_t image_size
= 0;
987 struct list_head temp_topology_device_list
;
988 int cpu_only_node
= 0;
989 struct kfd_topology_device
*kdev
;
990 int proximity_domain
;
992 /* topology_device_list - Master list of all topology devices
993 * temp_topology_device_list - temporary list created while parsing CRAT
994 * or VCRAT. Once parsing is complete the contents of list is moved to
995 * topology_device_list
998 /* Initialize the head for the both the lists */
999 INIT_LIST_HEAD(&topology_device_list
);
1000 INIT_LIST_HEAD(&temp_topology_device_list
);
1001 init_rwsem(&topology_lock
);
1003 memset(&sys_props
, 0, sizeof(sys_props
));
1005 /* Proximity domains in ACPI CRAT tables start counting at
1006 * 0. The same should be true for virtual CRAT tables created
1007 * at this stage. GPUs added later in kfd_topology_add_device
1010 proximity_domain
= 0;
1013 * Get the CRAT image from the ACPI. If ACPI doesn't have one
1014 * or if ACPI CRAT is invalid create a virtual CRAT.
1015 * NOTE: The current implementation expects all AMD APUs to have
1016 * CRAT. If no CRAT is available, it is assumed to be a CPU
1018 ret
= kfd_create_crat_image_acpi(&crat_image
, &image_size
);
1020 ret
= kfd_parse_crat_table(crat_image
,
1021 &temp_topology_device_list
,
1024 kfd_is_acpi_crat_invalid(&temp_topology_device_list
)) {
1025 kfd_release_topology_device_list(
1026 &temp_topology_device_list
);
1027 kfd_destroy_crat_image(crat_image
);
1033 ret
= kfd_create_crat_image_virtual(&crat_image
, &image_size
,
1034 COMPUTE_UNIT_CPU
, NULL
,
1038 pr_err("Error creating VCRAT table for CPU\n");
1042 ret
= kfd_parse_crat_table(crat_image
,
1043 &temp_topology_device_list
,
1046 pr_err("Error parsing VCRAT table for CPU\n");
1051 kdev
= list_first_entry(&temp_topology_device_list
,
1052 struct kfd_topology_device
, list
);
1053 kfd_add_perf_to_topology(kdev
);
1055 down_write(&topology_lock
);
1056 kfd_topology_update_device_list(&temp_topology_device_list
,
1057 &topology_device_list
);
1058 atomic_set(&topology_crat_proximity_domain
, sys_props
.num_devices
-1);
1059 ret
= kfd_topology_update_sysfs();
1060 up_write(&topology_lock
);
1063 sys_props
.generation_count
++;
1064 kfd_update_system_properties();
1065 kfd_debug_print_topology();
1067 pr_err("Failed to update topology in sysfs ret=%d\n", ret
);
1069 /* For nodes with GPU, this information gets added
1070 * when GPU is detected (kfd_topology_add_device).
1072 if (cpu_only_node
) {
1073 /* Add additional information to CPU only node created above */
1074 down_write(&topology_lock
);
1075 kdev
= list_first_entry(&topology_device_list
,
1076 struct kfd_topology_device
, list
);
1077 up_write(&topology_lock
);
1078 kfd_add_non_crat_information(kdev
);
1082 kfd_destroy_crat_image(crat_image
);
1086 void kfd_topology_shutdown(void)
1088 down_write(&topology_lock
);
1089 kfd_topology_release_sysfs();
1090 kfd_release_live_view();
1091 up_write(&topology_lock
);
1094 static uint32_t kfd_generate_gpu_id(struct kfd_dev
*gpu
)
1098 uint64_t local_mem_size
;
1100 struct kfd_local_mem_info local_mem_info
;
1105 amdgpu_amdkfd_get_local_mem_info(gpu
->kgd
, &local_mem_info
);
1107 local_mem_size
= local_mem_info
.local_mem_size_private
+
1108 local_mem_info
.local_mem_size_public
;
1110 buf
[0] = gpu
->pdev
->devfn
;
1111 buf
[1] = gpu
->pdev
->subsystem_vendor
|
1112 (gpu
->pdev
->subsystem_device
<< 16);
1113 buf
[2] = pci_domain_nr(gpu
->pdev
->bus
);
1114 buf
[3] = gpu
->pdev
->device
;
1115 buf
[4] = gpu
->pdev
->bus
->number
;
1116 buf
[5] = lower_32_bits(local_mem_size
);
1117 buf
[6] = upper_32_bits(local_mem_size
);
1119 for (i
= 0, hashout
= 0; i
< 7; i
++)
1120 hashout
^= hash_32(buf
[i
], KFD_GPU_ID_HASH_WIDTH
);
1124 /* kfd_assign_gpu - Attach @gpu to the correct kfd topology device. If
1125 * the GPU device is not already present in the topology device
1126 * list then return NULL. This means a new topology device has to
1127 * be created for this GPU.
1129 static struct kfd_topology_device
*kfd_assign_gpu(struct kfd_dev
*gpu
)
1131 struct kfd_topology_device
*dev
;
1132 struct kfd_topology_device
*out_dev
= NULL
;
1133 struct kfd_mem_properties
*mem
;
1134 struct kfd_cache_properties
*cache
;
1135 struct kfd_iolink_properties
*iolink
;
1137 down_write(&topology_lock
);
1138 list_for_each_entry(dev
, &topology_device_list
, list
) {
1139 /* Discrete GPUs need their own topology device list
1140 * entries. Don't assign them to CPU/APU nodes.
1142 if (!gpu
->use_iommu_v2
&&
1143 dev
->node_props
.cpu_cores_count
)
1146 if (!dev
->gpu
&& (dev
->node_props
.simd_count
> 0)) {
1150 list_for_each_entry(mem
, &dev
->mem_props
, list
)
1151 mem
->gpu
= dev
->gpu
;
1152 list_for_each_entry(cache
, &dev
->cache_props
, list
)
1153 cache
->gpu
= dev
->gpu
;
1154 list_for_each_entry(iolink
, &dev
->io_link_props
, list
)
1155 iolink
->gpu
= dev
->gpu
;
1159 up_write(&topology_lock
);
1163 static void kfd_notify_gpu_change(uint32_t gpu_id
, int arrival
)
1166 * TODO: Generate an event for thunk about the arrival/removal
1171 /* kfd_fill_mem_clk_max_info - Since CRAT doesn't have memory clock info,
1172 * patch this after CRAT parsing.
1174 static void kfd_fill_mem_clk_max_info(struct kfd_topology_device
*dev
)
1176 struct kfd_mem_properties
*mem
;
1177 struct kfd_local_mem_info local_mem_info
;
1182 /* Currently, amdgpu driver (amdgpu_mc) deals only with GPUs with
1183 * single bank of VRAM local memory.
1184 * for dGPUs - VCRAT reports only one bank of Local Memory
1185 * for APUs - If CRAT from ACPI reports more than one bank, then
1186 * all the banks will report the same mem_clk_max information
1188 amdgpu_amdkfd_get_local_mem_info(dev
->gpu
->kgd
, &local_mem_info
);
1190 list_for_each_entry(mem
, &dev
->mem_props
, list
)
1191 mem
->mem_clk_max
= local_mem_info
.mem_clk_max
;
1194 static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device
*dev
)
1196 struct kfd_iolink_properties
*link
, *cpu_link
;
1197 struct kfd_topology_device
*cpu_dev
;
1199 uint32_t cpu_flag
= CRAT_IOLINK_FLAGS_ENABLED
;
1200 uint32_t flag
= CRAT_IOLINK_FLAGS_ENABLED
;
1202 if (!dev
|| !dev
->gpu
)
1205 pcie_capability_read_dword(dev
->gpu
->pdev
,
1206 PCI_EXP_DEVCAP2
, &cap
);
1208 if (!(cap
& (PCI_EXP_DEVCAP2_ATOMIC_COMP32
|
1209 PCI_EXP_DEVCAP2_ATOMIC_COMP64
)))
1210 cpu_flag
|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT
|
1211 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT
;
1213 if (!dev
->gpu
->pci_atomic_requested
||
1214 dev
->gpu
->device_info
->asic_family
== CHIP_HAWAII
)
1215 flag
|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT
|
1216 CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT
;
1218 /* GPU only creates direct links so apply flags setting to all */
1219 list_for_each_entry(link
, &dev
->io_link_props
, list
) {
1221 cpu_dev
= kfd_topology_device_by_proximity_domain(
1224 list_for_each_entry(cpu_link
,
1225 &cpu_dev
->io_link_props
, list
)
1226 if (cpu_link
->node_to
== link
->node_from
)
1227 cpu_link
->flags
= cpu_flag
;
1232 int kfd_topology_add_device(struct kfd_dev
*gpu
)
1235 struct kfd_topology_device
*dev
;
1236 struct kfd_cu_info cu_info
;
1238 struct list_head temp_topology_device_list
;
1239 void *crat_image
= NULL
;
1240 size_t image_size
= 0;
1241 int proximity_domain
;
1242 struct amdgpu_device
*adev
;
1244 INIT_LIST_HEAD(&temp_topology_device_list
);
1246 gpu_id
= kfd_generate_gpu_id(gpu
);
1248 pr_debug("Adding new GPU (ID: 0x%x) to topology\n", gpu_id
);
1250 proximity_domain
= atomic_inc_return(&topology_crat_proximity_domain
);
1252 /* Check to see if this gpu device exists in the topology_device_list.
1253 * If so, assign the gpu to that device,
1254 * else create a Virtual CRAT for this gpu device and then parse that
1255 * CRAT to create a new topology device. Once created assign the gpu to
1256 * that topology device
1258 dev
= kfd_assign_gpu(gpu
);
1260 res
= kfd_create_crat_image_virtual(&crat_image
, &image_size
,
1261 COMPUTE_UNIT_GPU
, gpu
,
1264 pr_err("Error creating VCRAT for GPU (ID: 0x%x)\n",
1268 res
= kfd_parse_crat_table(crat_image
,
1269 &temp_topology_device_list
,
1272 pr_err("Error parsing VCRAT for GPU (ID: 0x%x)\n",
1277 down_write(&topology_lock
);
1278 kfd_topology_update_device_list(&temp_topology_device_list
,
1279 &topology_device_list
);
1281 /* Update the SYSFS tree, since we added another topology
1284 res
= kfd_topology_update_sysfs();
1285 up_write(&topology_lock
);
1288 sys_props
.generation_count
++;
1290 pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n",
1292 dev
= kfd_assign_gpu(gpu
);
1293 if (WARN_ON(!dev
)) {
1299 dev
->gpu_id
= gpu_id
;
1302 /* TODO: Move the following lines to function
1303 * kfd_add_non_crat_information
1306 /* Fill-in additional information that is not available in CRAT but
1307 * needed for the topology
1310 amdgpu_amdkfd_get_cu_info(dev
->gpu
->kgd
, &cu_info
);
1312 strncpy(dev
->node_props
.name
, gpu
->device_info
->asic_name
,
1313 KFD_TOPOLOGY_PUBLIC_NAME_SIZE
);
1315 dev
->node_props
.simd_arrays_per_engine
=
1316 cu_info
.num_shader_arrays_per_engine
;
1318 dev
->node_props
.vendor_id
= gpu
->pdev
->vendor
;
1319 dev
->node_props
.device_id
= gpu
->pdev
->device
;
1320 dev
->node_props
.capability
|=
1321 ((amdgpu_amdkfd_get_asic_rev_id(dev
->gpu
->kgd
) <<
1322 HSA_CAP_ASIC_REVISION_SHIFT
) &
1323 HSA_CAP_ASIC_REVISION_MASK
);
1324 dev
->node_props
.location_id
= pci_dev_id(gpu
->pdev
);
1325 dev
->node_props
.domain
= pci_domain_nr(gpu
->pdev
->bus
);
1326 dev
->node_props
.max_engine_clk_fcompute
=
1327 amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev
->gpu
->kgd
);
1328 dev
->node_props
.max_engine_clk_ccompute
=
1329 cpufreq_quick_get_max(0) / 1000;
1330 dev
->node_props
.drm_render_minor
=
1331 gpu
->shared_resources
.drm_render_minor
;
1333 dev
->node_props
.hive_id
= gpu
->hive_id
;
1334 dev
->node_props
.num_sdma_engines
= gpu
->device_info
->num_sdma_engines
;
1335 dev
->node_props
.num_sdma_xgmi_engines
=
1336 gpu
->device_info
->num_xgmi_sdma_engines
;
1337 dev
->node_props
.num_sdma_queues_per_engine
=
1338 gpu
->device_info
->num_sdma_queues_per_engine
;
1339 dev
->node_props
.num_gws
= (dev
->gpu
->gws
&&
1340 dev
->gpu
->dqm
->sched_policy
!= KFD_SCHED_POLICY_NO_HWS
) ?
1341 amdgpu_amdkfd_get_num_gws(dev
->gpu
->kgd
) : 0;
1342 dev
->node_props
.num_cp_queues
= get_cp_queues_num(dev
->gpu
->dqm
);
1343 dev
->node_props
.unique_id
= amdgpu_amdkfd_get_unique_id(dev
->gpu
->kgd
);
1345 kfd_fill_mem_clk_max_info(dev
);
1346 kfd_fill_iolink_non_crat_info(dev
);
1348 switch (dev
->gpu
->device_info
->asic_family
) {
1352 dev
->node_props
.capability
|= ((HSA_CAP_DOORBELL_TYPE_PRE_1_0
<<
1353 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT
) &
1354 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK
);
1358 case CHIP_POLARIS10
:
1359 case CHIP_POLARIS11
:
1360 case CHIP_POLARIS12
:
1362 pr_debug("Adding doorbell packet type capability\n");
1363 dev
->node_props
.capability
|= ((HSA_CAP_DOORBELL_TYPE_1_0
<<
1364 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT
) &
1365 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK
);
1376 case CHIP_SIENNA_CICHLID
:
1377 case CHIP_NAVY_FLOUNDER
:
1379 case CHIP_DIMGREY_CAVEFISH
:
1380 dev
->node_props
.capability
|= ((HSA_CAP_DOORBELL_TYPE_2_0
<<
1381 HSA_CAP_DOORBELL_TYPE_TOTALBITS_SHIFT
) &
1382 HSA_CAP_DOORBELL_TYPE_TOTALBITS_MASK
);
1385 WARN(1, "Unexpected ASIC family %u",
1386 dev
->gpu
->device_info
->asic_family
);
1390 * Overwrite ATS capability according to needs_iommu_device to fix
1391 * potential missing corresponding bit in CRAT of BIOS.
1393 if (dev
->gpu
->use_iommu_v2
)
1394 dev
->node_props
.capability
|= HSA_CAP_ATS_PRESENT
;
1396 dev
->node_props
.capability
&= ~HSA_CAP_ATS_PRESENT
;
1398 /* Fix errors in CZ CRAT.
1399 * simd_count: Carrizo CRAT reports wrong simd_count, probably
1400 * because it doesn't consider masked out CUs
1401 * max_waves_per_simd: Carrizo reports wrong max_waves_per_simd
1403 if (dev
->gpu
->device_info
->asic_family
== CHIP_CARRIZO
) {
1404 dev
->node_props
.simd_count
=
1405 cu_info
.simd_per_cu
* cu_info
.cu_active_number
;
1406 dev
->node_props
.max_waves_per_simd
= 10;
1409 adev
= (struct amdgpu_device
*)(dev
->gpu
->kgd
);
1410 /* kfd only concerns sram ecc on GFX and HBM ecc on UMC */
1411 dev
->node_props
.capability
|=
1412 ((adev
->ras_features
& BIT(AMDGPU_RAS_BLOCK__GFX
)) != 0) ?
1413 HSA_CAP_SRAM_EDCSUPPORTED
: 0;
1414 dev
->node_props
.capability
|= ((adev
->ras_features
& BIT(AMDGPU_RAS_BLOCK__UMC
)) != 0) ?
1415 HSA_CAP_MEM_EDCSUPPORTED
: 0;
1417 if (adev
->asic_type
!= CHIP_VEGA10
)
1418 dev
->node_props
.capability
|= (adev
->ras_features
!= 0) ?
1419 HSA_CAP_RASEVENTNOTIFY
: 0;
1421 kfd_debug_print_topology();
1424 kfd_notify_gpu_change(gpu_id
, 1);
1426 kfd_destroy_crat_image(crat_image
);
1430 int kfd_topology_remove_device(struct kfd_dev
*gpu
)
1432 struct kfd_topology_device
*dev
, *tmp
;
1436 down_write(&topology_lock
);
1438 list_for_each_entry_safe(dev
, tmp
, &topology_device_list
, list
)
1439 if (dev
->gpu
== gpu
) {
1440 gpu_id
= dev
->gpu_id
;
1441 kfd_remove_sysfs_node_entry(dev
);
1442 kfd_release_topology_device(dev
);
1443 sys_props
.num_devices
--;
1445 if (kfd_topology_update_sysfs() < 0)
1446 kfd_topology_release_sysfs();
1450 up_write(&topology_lock
);
1453 kfd_notify_gpu_change(gpu_id
, 0);
1458 /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD
1459 * topology. If GPU device is found @idx, then valid kfd_dev pointer is
1460 * returned through @kdev
1461 * Return - 0: On success (@kdev will be NULL for non GPU nodes)
1462 * -1: If end of list
1464 int kfd_topology_enum_kfd_devices(uint8_t idx
, struct kfd_dev
**kdev
)
1467 struct kfd_topology_device
*top_dev
;
1468 uint8_t device_idx
= 0;
1471 down_read(&topology_lock
);
1473 list_for_each_entry(top_dev
, &topology_device_list
, list
) {
1474 if (device_idx
== idx
) {
1475 *kdev
= top_dev
->gpu
;
1476 up_read(&topology_lock
);
1483 up_read(&topology_lock
);
1489 static int kfd_cpumask_to_apic_id(const struct cpumask
*cpumask
)
1491 int first_cpu_of_numa_node
;
1493 if (!cpumask
|| cpumask
== cpu_none_mask
)
1495 first_cpu_of_numa_node
= cpumask_first(cpumask
);
1496 if (first_cpu_of_numa_node
>= nr_cpu_ids
)
1498 #ifdef CONFIG_X86_64
1499 return cpu_data(first_cpu_of_numa_node
).apicid
;
1501 return first_cpu_of_numa_node
;
1505 /* kfd_numa_node_to_apic_id - Returns the APIC ID of the first logical processor
1506 * of the given NUMA node (numa_node_id)
1507 * Return -1 on failure
1509 int kfd_numa_node_to_apic_id(int numa_node_id
)
1511 if (numa_node_id
== -1) {
1512 pr_warn("Invalid NUMA Node. Use online CPU mask\n");
1513 return kfd_cpumask_to_apic_id(cpu_online_mask
);
1515 return kfd_cpumask_to_apic_id(cpumask_of_node(numa_node_id
));
1518 void kfd_double_confirm_iommu_support(struct kfd_dev
*gpu
)
1520 struct kfd_topology_device
*dev
;
1522 gpu
->use_iommu_v2
= false;
1524 if (!gpu
->device_info
->needs_iommu_device
)
1527 down_read(&topology_lock
);
1529 /* Only use IOMMUv2 if there is an APU topology node with no GPU
1530 * assigned yet. This GPU will be assigned to it.
1532 list_for_each_entry(dev
, &topology_device_list
, list
)
1533 if (dev
->node_props
.cpu_cores_count
&&
1534 dev
->node_props
.simd_count
&&
1536 gpu
->use_iommu_v2
= true;
1538 up_read(&topology_lock
);
1541 #if defined(CONFIG_DEBUG_FS)
1543 int kfd_debugfs_hqds_by_device(struct seq_file
*m
, void *data
)
1545 struct kfd_topology_device
*dev
;
1549 down_read(&topology_lock
);
1551 list_for_each_entry(dev
, &topology_device_list
, list
) {
1557 seq_printf(m
, "Node %u, gpu_id %x:\n", i
++, dev
->gpu
->id
);
1558 r
= dqm_debugfs_hqds(m
, dev
->gpu
->dqm
);
1563 up_read(&topology_lock
);
1568 int kfd_debugfs_rls_by_device(struct seq_file
*m
, void *data
)
1570 struct kfd_topology_device
*dev
;
1574 down_read(&topology_lock
);
1576 list_for_each_entry(dev
, &topology_device_list
, list
) {
1582 seq_printf(m
, "Node %u, gpu_id %x:\n", i
++, dev
->gpu
->id
);
1583 r
= pm_debugfs_runlist(m
, &dev
->gpu
->dqm
->packets
);
1588 up_read(&topology_lock
);