drivers/gpu/drm/amd/amdkfd/kfd_device.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/bsearch.h>
  24 #include <linux/pci.h>
  25 #include <linux/slab.h>
  26 #include "kfd_priv.h"
  27 #include "kfd_device_queue_manager.h"
  28 #include "kfd_pm4_headers_vi.h"
  29 #include "cwsr_trap_handler.h"
  30 #include "kfd_iommu.h"
  31 #include "amdgpu_amdkfd.h"
  32 #include "kfd_smi_events.h"
  33
  34 #define MQD_SIZE_ALIGNED 768
  35
  36 /*
  37  * kfd_locked is used to lock the kfd driver during suspend or reset
  38  * once locked, kfd driver will stop any further GPU execution.
  39  * create process (open) will return -EAGAIN.
  40  */
  41 static atomic_t kfd_locked = ATOMIC_INIT(0);
  42
  43 #ifdef CONFIG_DRM_AMDGPU_CIK
  44 extern const struct kfd2kgd_calls gfx_v7_kfd2kgd;
  45 #endif
  46 extern const struct kfd2kgd_calls gfx_v8_kfd2kgd;
  47 extern const struct kfd2kgd_calls gfx_v9_kfd2kgd;
  48 extern const struct kfd2kgd_calls arcturus_kfd2kgd;
  49 extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;
  50 extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;
  51
  52 static const struct kfd2kgd_calls *kfd2kgd_funcs[] = {
  53 #ifdef KFD_SUPPORT_IOMMU_V2
  54 #ifdef CONFIG_DRM_AMDGPU_CIK
  55         [CHIP_KAVERI] = &gfx_v7_kfd2kgd,
  56 #endif
  57         [CHIP_CARRIZO] = &gfx_v8_kfd2kgd,
  58         [CHIP_RAVEN] = &gfx_v9_kfd2kgd,
  59 #endif
  60 #ifdef CONFIG_DRM_AMDGPU_CIK
  61         [CHIP_HAWAII] = &gfx_v7_kfd2kgd,
  62 #endif
  63         [CHIP_TONGA] = &gfx_v8_kfd2kgd,
  64         [CHIP_FIJI] = &gfx_v8_kfd2kgd,
  65         [CHIP_POLARIS10] = &gfx_v8_kfd2kgd,
  66         [CHIP_POLARIS11] = &gfx_v8_kfd2kgd,
  67         [CHIP_POLARIS12] = &gfx_v8_kfd2kgd,
  68         [CHIP_VEGAM] = &gfx_v8_kfd2kgd,
  69         [CHIP_VEGA10] = &gfx_v9_kfd2kgd,
  70         [CHIP_VEGA12] = &gfx_v9_kfd2kgd,
  71         [CHIP_VEGA20] = &gfx_v9_kfd2kgd,
  72         [CHIP_RENOIR] = &gfx_v9_kfd2kgd,
  73         [CHIP_ARCTURUS] = &arcturus_kfd2kgd,
  74         [CHIP_NAVI10] = &gfx_v10_kfd2kgd,
  75         [CHIP_NAVI12] = &gfx_v10_kfd2kgd,
  76         [CHIP_NAVI14] = &gfx_v10_kfd2kgd,
  77         [CHIP_SIENNA_CICHLID] = &gfx_v10_3_kfd2kgd,
  78         [CHIP_NAVY_FLOUNDER] = &gfx_v10_3_kfd2kgd,
  79         [CHIP_VANGOGH] = &gfx_v10_3_kfd2kgd,
  80         [CHIP_DIMGREY_CAVEFISH] = &gfx_v10_3_kfd2kgd,
  81 };
  82
  83 #ifdef KFD_SUPPORT_IOMMU_V2
  84 static const struct kfd_device_info kaveri_device_info = {
  85         .asic_family = CHIP_KAVERI,
  86         .asic_name = "kaveri",
  87         .max_pasid_bits = 16,
  88         /* max num of queues for KV.TODO should be a dynamic value */
  89         .max_no_of_hqd  = 24,
  90         .doorbell_size  = 4,
  91         .ih_ring_entry_size = 4 * sizeof(uint32_t),
  92         .event_interrupt_class = &event_interrupt_class_cik,
  93         .num_of_watch_points = 4,
  94         .mqd_size_aligned = MQD_SIZE_ALIGNED,
  95         .supports_cwsr = false,
  96         .needs_iommu_device = true,
  97         .needs_pci_atomics = false,
  98         .num_sdma_engines = 2,
  99         .num_xgmi_sdma_engines = 0,
 100         .num_sdma_queues_per_engine = 2,
 101 };
 102
 103 static const struct kfd_device_info carrizo_device_info = {
 104         .asic_family = CHIP_CARRIZO,
 105         .asic_name = "carrizo",
 106         .max_pasid_bits = 16,
 107         /* max num of queues for CZ.TODO should be a dynamic value */
 108         .max_no_of_hqd  = 24,
 109         .doorbell_size  = 4,
 110         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 111         .event_interrupt_class = &event_interrupt_class_cik,
 112         .num_of_watch_points = 4,
 113         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 114         .supports_cwsr = true,
 115         .needs_iommu_device = true,
 116         .needs_pci_atomics = false,
 117         .num_sdma_engines = 2,
 118         .num_xgmi_sdma_engines = 0,
 119         .num_sdma_queues_per_engine = 2,
 120 };
 121 #endif
 122
 123 static const struct kfd_device_info raven_device_info = {
 124         .asic_family = CHIP_RAVEN,
 125         .asic_name = "raven",
 126         .max_pasid_bits = 16,
 127         .max_no_of_hqd  = 24,
 128         .doorbell_size  = 8,
 129         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 130         .event_interrupt_class = &event_interrupt_class_v9,
 131         .num_of_watch_points = 4,
 132         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 133         .supports_cwsr = true,
 134         .needs_iommu_device = true,
 135         .needs_pci_atomics = true,
 136         .num_sdma_engines = 1,
 137         .num_xgmi_sdma_engines = 0,
 138         .num_sdma_queues_per_engine = 2,
 139 };
 140
 141 static const struct kfd_device_info hawaii_device_info = {
 142         .asic_family = CHIP_HAWAII,
 143         .asic_name = "hawaii",
 144         .max_pasid_bits = 16,
 145         /* max num of queues for KV.TODO should be a dynamic value */
 146         .max_no_of_hqd  = 24,
 147         .doorbell_size  = 4,
 148         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 149         .event_interrupt_class = &event_interrupt_class_cik,
 150         .num_of_watch_points = 4,
 151         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 152         .supports_cwsr = false,
 153         .needs_iommu_device = false,
 154         .needs_pci_atomics = false,
 155         .num_sdma_engines = 2,
 156         .num_xgmi_sdma_engines = 0,
 157         .num_sdma_queues_per_engine = 2,
 158 };
 159
 160 static const struct kfd_device_info tonga_device_info = {
 161         .asic_family = CHIP_TONGA,
 162         .asic_name = "tonga",
 163         .max_pasid_bits = 16,
 164         .max_no_of_hqd  = 24,
 165         .doorbell_size  = 4,
 166         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 167         .event_interrupt_class = &event_interrupt_class_cik,
 168         .num_of_watch_points = 4,
 169         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 170         .supports_cwsr = false,
 171         .needs_iommu_device = false,
 172         .needs_pci_atomics = true,
 173         .num_sdma_engines = 2,
 174         .num_xgmi_sdma_engines = 0,
 175         .num_sdma_queues_per_engine = 2,
 176 };
 177
 178 static const struct kfd_device_info fiji_device_info = {
 179         .asic_family = CHIP_FIJI,
 180         .asic_name = "fiji",
 181         .max_pasid_bits = 16,
 182         .max_no_of_hqd  = 24,
 183         .doorbell_size  = 4,
 184         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 185         .event_interrupt_class = &event_interrupt_class_cik,
 186         .num_of_watch_points = 4,
 187         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 188         .supports_cwsr = true,
 189         .needs_iommu_device = false,
 190         .needs_pci_atomics = true,
 191         .num_sdma_engines = 2,
 192         .num_xgmi_sdma_engines = 0,
 193         .num_sdma_queues_per_engine = 2,
 194 };
 195
 196 static const struct kfd_device_info fiji_vf_device_info = {
 197         .asic_family = CHIP_FIJI,
 198         .asic_name = "fiji",
 199         .max_pasid_bits = 16,
 200         .max_no_of_hqd  = 24,
 201         .doorbell_size  = 4,
 202         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 203         .event_interrupt_class = &event_interrupt_class_cik,
 204         .num_of_watch_points = 4,
 205         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 206         .supports_cwsr = true,
 207         .needs_iommu_device = false,
 208         .needs_pci_atomics = false,
 209         .num_sdma_engines = 2,
 210         .num_xgmi_sdma_engines = 0,
 211         .num_sdma_queues_per_engine = 2,
 212 };
 213
 214
 215 static const struct kfd_device_info polaris10_device_info = {
 216         .asic_family = CHIP_POLARIS10,
 217         .asic_name = "polaris10",
 218         .max_pasid_bits = 16,
 219         .max_no_of_hqd  = 24,
 220         .doorbell_size  = 4,
 221         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 222         .event_interrupt_class = &event_interrupt_class_cik,
 223         .num_of_watch_points = 4,
 224         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 225         .supports_cwsr = true,
 226         .needs_iommu_device = false,
 227         .needs_pci_atomics = true,
 228         .num_sdma_engines = 2,
 229         .num_xgmi_sdma_engines = 0,
 230         .num_sdma_queues_per_engine = 2,
 231 };
 232
 233 static const struct kfd_device_info polaris10_vf_device_info = {
 234         .asic_family = CHIP_POLARIS10,
 235         .asic_name = "polaris10",
 236         .max_pasid_bits = 16,
 237         .max_no_of_hqd  = 24,
 238         .doorbell_size  = 4,
 239         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 240         .event_interrupt_class = &event_interrupt_class_cik,
 241         .num_of_watch_points = 4,
 242         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 243         .supports_cwsr = true,
 244         .needs_iommu_device = false,
 245         .needs_pci_atomics = false,
 246         .num_sdma_engines = 2,
 247         .num_xgmi_sdma_engines = 0,
 248         .num_sdma_queues_per_engine = 2,
 249 };
 250
 251 static const struct kfd_device_info polaris11_device_info = {
 252         .asic_family = CHIP_POLARIS11,
 253         .asic_name = "polaris11",
 254         .max_pasid_bits = 16,
 255         .max_no_of_hqd  = 24,
 256         .doorbell_size  = 4,
 257         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 258         .event_interrupt_class = &event_interrupt_class_cik,
 259         .num_of_watch_points = 4,
 260         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 261         .supports_cwsr = true,
 262         .needs_iommu_device = false,
 263         .needs_pci_atomics = true,
 264         .num_sdma_engines = 2,
 265         .num_xgmi_sdma_engines = 0,
 266         .num_sdma_queues_per_engine = 2,
 267 };
 268
 269 static const struct kfd_device_info polaris12_device_info = {
 270         .asic_family = CHIP_POLARIS12,
 271         .asic_name = "polaris12",
 272         .max_pasid_bits = 16,
 273         .max_no_of_hqd  = 24,
 274         .doorbell_size  = 4,
 275         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 276         .event_interrupt_class = &event_interrupt_class_cik,
 277         .num_of_watch_points = 4,
 278         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 279         .supports_cwsr = true,
 280         .needs_iommu_device = false,
 281         .needs_pci_atomics = true,
 282         .num_sdma_engines = 2,
 283         .num_xgmi_sdma_engines = 0,
 284         .num_sdma_queues_per_engine = 2,
 285 };
 286
 287 static const struct kfd_device_info vegam_device_info = {
 288         .asic_family = CHIP_VEGAM,
 289         .asic_name = "vegam",
 290         .max_pasid_bits = 16,
 291         .max_no_of_hqd  = 24,
 292         .doorbell_size  = 4,
 293         .ih_ring_entry_size = 4 * sizeof(uint32_t),
 294         .event_interrupt_class = &event_interrupt_class_cik,
 295         .num_of_watch_points = 4,
 296         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 297         .supports_cwsr = true,
 298         .needs_iommu_device = false,
 299         .needs_pci_atomics = true,
 300         .num_sdma_engines = 2,
 301         .num_xgmi_sdma_engines = 0,
 302         .num_sdma_queues_per_engine = 2,
 303 };
 304
 305 static const struct kfd_device_info vega10_device_info = {
 306         .asic_family = CHIP_VEGA10,
 307         .asic_name = "vega10",
 308         .max_pasid_bits = 16,
 309         .max_no_of_hqd  = 24,
 310         .doorbell_size  = 8,
 311         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 312         .event_interrupt_class = &event_interrupt_class_v9,
 313         .num_of_watch_points = 4,
 314         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 315         .supports_cwsr = true,
 316         .needs_iommu_device = false,
 317         .needs_pci_atomics = false,
 318         .num_sdma_engines = 2,
 319         .num_xgmi_sdma_engines = 0,
 320         .num_sdma_queues_per_engine = 2,
 321 };
 322
 323 static const struct kfd_device_info vega10_vf_device_info = {
 324         .asic_family = CHIP_VEGA10,
 325         .asic_name = "vega10",
 326         .max_pasid_bits = 16,
 327         .max_no_of_hqd  = 24,
 328         .doorbell_size  = 8,
 329         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 330         .event_interrupt_class = &event_interrupt_class_v9,
 331         .num_of_watch_points = 4,
 332         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 333         .supports_cwsr = true,
 334         .needs_iommu_device = false,
 335         .needs_pci_atomics = false,
 336         .num_sdma_engines = 2,
 337         .num_xgmi_sdma_engines = 0,
 338         .num_sdma_queues_per_engine = 2,
 339 };
 340
 341 static const struct kfd_device_info vega12_device_info = {
 342         .asic_family = CHIP_VEGA12,
 343         .asic_name = "vega12",
 344         .max_pasid_bits = 16,
 345         .max_no_of_hqd  = 24,
 346         .doorbell_size  = 8,
 347         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 348         .event_interrupt_class = &event_interrupt_class_v9,
 349         .num_of_watch_points = 4,
 350         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 351         .supports_cwsr = true,
 352         .needs_iommu_device = false,
 353         .needs_pci_atomics = false,
 354         .num_sdma_engines = 2,
 355         .num_xgmi_sdma_engines = 0,
 356         .num_sdma_queues_per_engine = 2,
 357 };
 358
 359 static const struct kfd_device_info vega20_device_info = {
 360         .asic_family = CHIP_VEGA20,
 361         .asic_name = "vega20",
 362         .max_pasid_bits = 16,
 363         .max_no_of_hqd  = 24,
 364         .doorbell_size  = 8,
 365         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 366         .event_interrupt_class = &event_interrupt_class_v9,
 367         .num_of_watch_points = 4,
 368         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 369         .supports_cwsr = true,
 370         .needs_iommu_device = false,
 371         .needs_pci_atomics = false,
 372         .num_sdma_engines = 2,
 373         .num_xgmi_sdma_engines = 0,
 374         .num_sdma_queues_per_engine = 8,
 375 };
 376
 377 static const struct kfd_device_info arcturus_device_info = {
 378         .asic_family = CHIP_ARCTURUS,
 379         .asic_name = "arcturus",
 380         .max_pasid_bits = 16,
 381         .max_no_of_hqd  = 24,
 382         .doorbell_size  = 8,
 383         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 384         .event_interrupt_class = &event_interrupt_class_v9,
 385         .num_of_watch_points = 4,
 386         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 387         .supports_cwsr = true,
 388         .needs_iommu_device = false,
 389         .needs_pci_atomics = false,
 390         .num_sdma_engines = 2,
 391         .num_xgmi_sdma_engines = 6,
 392         .num_sdma_queues_per_engine = 8,
 393 };
 394
 395 static const struct kfd_device_info renoir_device_info = {
 396         .asic_family = CHIP_RENOIR,
 397         .asic_name = "renoir",
 398         .max_pasid_bits = 16,
 399         .max_no_of_hqd  = 24,
 400         .doorbell_size  = 8,
 401         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 402         .event_interrupt_class = &event_interrupt_class_v9,
 403         .num_of_watch_points = 4,
 404         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 405         .supports_cwsr = true,
 406         .needs_iommu_device = false,
 407         .needs_pci_atomics = false,
 408         .num_sdma_engines = 1,
 409         .num_xgmi_sdma_engines = 0,
 410         .num_sdma_queues_per_engine = 2,
 411 };
 412
 413 static const struct kfd_device_info navi10_device_info = {
 414         .asic_family = CHIP_NAVI10,
 415         .asic_name = "navi10",
 416         .max_pasid_bits = 16,
 417         .max_no_of_hqd  = 24,
 418         .doorbell_size  = 8,
 419         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 420         .event_interrupt_class = &event_interrupt_class_v9,
 421         .num_of_watch_points = 4,
 422         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 423         .needs_iommu_device = false,
 424         .supports_cwsr = true,
 425         .needs_pci_atomics = true,
 426         .num_sdma_engines = 2,
 427         .num_xgmi_sdma_engines = 0,
 428         .num_sdma_queues_per_engine = 8,
 429 };
 430
 431 static const struct kfd_device_info navi12_device_info = {
 432         .asic_family = CHIP_NAVI12,
 433         .asic_name = "navi12",
 434         .max_pasid_bits = 16,
 435         .max_no_of_hqd  = 24,
 436         .doorbell_size  = 8,
 437         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 438         .event_interrupt_class = &event_interrupt_class_v9,
 439         .num_of_watch_points = 4,
 440         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 441         .needs_iommu_device = false,
 442         .supports_cwsr = true,
 443         .needs_pci_atomics = true,
 444         .num_sdma_engines = 2,
 445         .num_xgmi_sdma_engines = 0,
 446         .num_sdma_queues_per_engine = 8,
 447 };
 448
 449 static const struct kfd_device_info navi14_device_info = {
 450         .asic_family = CHIP_NAVI14,
 451         .asic_name = "navi14",
 452         .max_pasid_bits = 16,
 453         .max_no_of_hqd  = 24,
 454         .doorbell_size  = 8,
 455         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 456         .event_interrupt_class = &event_interrupt_class_v9,
 457         .num_of_watch_points = 4,
 458         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 459         .needs_iommu_device = false,
 460         .supports_cwsr = true,
 461         .needs_pci_atomics = true,
 462         .num_sdma_engines = 2,
 463         .num_xgmi_sdma_engines = 0,
 464         .num_sdma_queues_per_engine = 8,
 465 };
 466
 467 static const struct kfd_device_info sienna_cichlid_device_info = {
 468         .asic_family = CHIP_SIENNA_CICHLID,
 469         .asic_name = "sienna_cichlid",
 470         .max_pasid_bits = 16,
 471         .max_no_of_hqd  = 24,
 472         .doorbell_size  = 8,
 473         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 474         .event_interrupt_class = &event_interrupt_class_v9,
 475         .num_of_watch_points = 4,
 476         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 477         .needs_iommu_device = false,
 478         .supports_cwsr = true,
 479         .needs_pci_atomics = true,
 480         .num_sdma_engines = 4,
 481         .num_xgmi_sdma_engines = 0,
 482         .num_sdma_queues_per_engine = 8,
 483 };
 484
 485 static const struct kfd_device_info navy_flounder_device_info = {
 486         .asic_family = CHIP_NAVY_FLOUNDER,
 487         .asic_name = "navy_flounder",
 488         .max_pasid_bits = 16,
 489         .max_no_of_hqd  = 24,
 490         .doorbell_size  = 8,
 491         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 492         .event_interrupt_class = &event_interrupt_class_v9,
 493         .num_of_watch_points = 4,
 494         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 495         .needs_iommu_device = false,
 496         .supports_cwsr = true,
 497         .needs_pci_atomics = true,
 498         .num_sdma_engines = 2,
 499         .num_xgmi_sdma_engines = 0,
 500         .num_sdma_queues_per_engine = 8,
 501 };
 502
 503 static const struct kfd_device_info vangogh_device_info = {
 504         .asic_family = CHIP_VANGOGH,
 505         .asic_name = "vangogh",
 506         .max_pasid_bits = 16,
 507         .max_no_of_hqd  = 24,
 508         .doorbell_size  = 8,
 509         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 510         .event_interrupt_class = &event_interrupt_class_v9,
 511         .num_of_watch_points = 4,
 512         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 513         .needs_iommu_device = false,
 514         .supports_cwsr = true,
 515         .needs_pci_atomics = false,
 516         .num_sdma_engines = 1,
 517         .num_xgmi_sdma_engines = 0,
 518         .num_sdma_queues_per_engine = 2,
 519 };
 520
 521 static const struct kfd_device_info dimgrey_cavefish_device_info = {
 522         .asic_family = CHIP_DIMGREY_CAVEFISH,
 523         .asic_name = "dimgrey_cavefish",
 524         .max_pasid_bits = 16,
 525         .max_no_of_hqd  = 24,
 526         .doorbell_size  = 8,
 527         .ih_ring_entry_size = 8 * sizeof(uint32_t),
 528         .event_interrupt_class = &event_interrupt_class_v9,
 529         .num_of_watch_points = 4,
 530         .mqd_size_aligned = MQD_SIZE_ALIGNED,
 531         .needs_iommu_device = false,
 532         .supports_cwsr = true,
 533         .needs_pci_atomics = true,
 534         .num_sdma_engines = 2,
 535         .num_xgmi_sdma_engines = 0,
 536         .num_sdma_queues_per_engine = 8,
 537 };
 538
 539
 540 /* For each entry, [0] is regular and [1] is virtualisation device. */
 541 static const struct kfd_device_info *kfd_supported_devices[][2] = {
 542 #ifdef KFD_SUPPORT_IOMMU_V2
 543         [CHIP_KAVERI] = {&kaveri_device_info, NULL},
 544         [CHIP_CARRIZO] = {&carrizo_device_info, NULL},
 545 #endif
 546         [CHIP_RAVEN] = {&raven_device_info, NULL},
 547         [CHIP_HAWAII] = {&hawaii_device_info, NULL},
 548         [CHIP_TONGA] = {&tonga_device_info, NULL},
 549         [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info},
 550         [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info},
 551         [CHIP_POLARIS11] = {&polaris11_device_info, NULL},
 552         [CHIP_POLARIS12] = {&polaris12_device_info, NULL},
 553         [CHIP_VEGAM] = {&vegam_device_info, NULL},
 554         [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info},
 555         [CHIP_VEGA12] = {&vega12_device_info, NULL},
 556         [CHIP_VEGA20] = {&vega20_device_info, NULL},
 557         [CHIP_RENOIR] = {&renoir_device_info, NULL},
 558         [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info},
 559         [CHIP_NAVI10] = {&navi10_device_info, NULL},
 560         [CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info},
 561         [CHIP_NAVI14] = {&navi14_device_info, NULL},
 562         [CHIP_SIENNA_CICHLID] = {&sienna_cichlid_device_info, &sienna_cichlid_device_info},
 563         [CHIP_NAVY_FLOUNDER] = {&navy_flounder_device_info, &navy_flounder_device_info},
 564         [CHIP_VANGOGH] = {&vangogh_device_info, NULL},
 565         [CHIP_DIMGREY_CAVEFISH] = {&dimgrey_cavefish_device_info, &dimgrey_cavefish_device_info},
 566 };
 567
 568 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
 569                                 unsigned int chunk_size);
 570 static void kfd_gtt_sa_fini(struct kfd_dev *kfd);
 571
 572 static int kfd_resume(struct kfd_dev *kfd);
 573
 574 struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd,
 575         struct pci_dev *pdev, unsigned int asic_type, bool vf)
 576 {
 577         struct kfd_dev *kfd;
 578         const struct kfd_device_info *device_info;
 579         const struct kfd2kgd_calls *f2g;
 580
 581         if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2)
 582                 || asic_type >= sizeof(kfd2kgd_funcs) / sizeof(void *)) {
 583                 dev_err(kfd_device, "asic_type %d out of range\n", asic_type);
 584                 return NULL; /* asic_type out of range */
 585         }
 586
 587         device_info = kfd_supported_devices[asic_type][vf];
 588         f2g = kfd2kgd_funcs[asic_type];
 589
 590         if (!device_info || !f2g) {
 591                 dev_err(kfd_device, "%s %s not supported in kfd\n",
 592                         amdgpu_asic_name[asic_type], vf ? "VF" : "");
 593                 return NULL;
 594         }
 595
 596         kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);
 597         if (!kfd)
 598                 return NULL;
 599
 600         /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
 601          * 32 and 64-bit requests are possible and must be
 602          * supported.
 603          */
 604         kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd);
 605         if (device_info->needs_pci_atomics &&
 606             !kfd->pci_atomic_requested) {
 607                 dev_info(kfd_device,
 608                          "skipped device %x:%x, PCI rejects atomics\n",
 609                          pdev->vendor, pdev->device);
 610                 kfree(kfd);
 611                 return NULL;
 612         }
 613
 614         kfd->kgd = kgd;
 615         kfd->device_info = device_info;
 616         kfd->pdev = pdev;
 617         kfd->init_complete = false;
 618         kfd->kfd2kgd = f2g;
 619         atomic_set(&kfd->compute_profile, 0);
 620
 621         mutex_init(&kfd->doorbell_mutex);
 622         memset(&kfd->doorbell_available_index, 0,
 623                 sizeof(kfd->doorbell_available_index));
 624
 625         atomic_set(&kfd->sram_ecc_flag, 0);
 626
 627         ida_init(&kfd->doorbell_ida);
 628
 629         return kfd;
 630 }
 631
 632 static void kfd_cwsr_init(struct kfd_dev *kfd)
 633 {
 634         if (cwsr_enable && kfd->device_info->supports_cwsr) {
 635                 if (kfd->device_info->asic_family < CHIP_VEGA10) {
 636                         BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
 637                         kfd->cwsr_isa = cwsr_trap_gfx8_hex;
 638                         kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
 639                 } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) {
 640                         BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE);
 641                         kfd->cwsr_isa = cwsr_trap_arcturus_hex;
 642                         kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);
 643                 } else if (kfd->device_info->asic_family < CHIP_NAVI10) {
 644                         BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
 645                         kfd->cwsr_isa = cwsr_trap_gfx9_hex;
 646                         kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
 647                 } else if (kfd->device_info->asic_family < CHIP_SIENNA_CICHLID) {
 648                         BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > PAGE_SIZE);
 649                         kfd->cwsr_isa = cwsr_trap_nv1x_hex;
 650                         kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex);
 651                 } else {
 652                         BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE);
 653                         kfd->cwsr_isa = cwsr_trap_gfx10_hex;
 654                         kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);
 655                 }
 656
 657                 kfd->cwsr_enabled = true;
 658         }
 659 }
 660
 661 static int kfd_gws_init(struct kfd_dev *kfd)
 662 {
 663         int ret = 0;
 664
 665         if (kfd->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
 666                 return 0;
 667
 668         if (hws_gws_support
 669                 || (kfd->device_info->asic_family == CHIP_VEGA10
 670                         && kfd->mec2_fw_version >= 0x81b3)
 671                 || (kfd->device_info->asic_family >= CHIP_VEGA12
 672                         && kfd->device_info->asic_family <= CHIP_RAVEN
 673                         && kfd->mec2_fw_version >= 0x1b3)
 674                 || (kfd->device_info->asic_family == CHIP_ARCTURUS
 675                         && kfd->mec2_fw_version >= 0x30))
 676                 ret = amdgpu_amdkfd_alloc_gws(kfd->kgd,
 677                                 amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws);
 678
 679         return ret;
 680 }
 681
 682 static void kfd_smi_init(struct kfd_dev *dev) {
 683         INIT_LIST_HEAD(&dev->smi_clients);
 684         spin_lock_init(&dev->smi_lock);
 685 }
 686
 687 bool kgd2kfd_device_init(struct kfd_dev *kfd,
 688                          struct drm_device *ddev,
 689                          const struct kgd2kfd_shared_resources *gpu_resources)
 690 {
 691         unsigned int size;
 692
 693         kfd->ddev = ddev;
 694         kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
 695                         KGD_ENGINE_MEC1);
 696         kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
 697                         KGD_ENGINE_MEC2);
 698         kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
 699                         KGD_ENGINE_SDMA1);
 700         kfd->shared_resources = *gpu_resources;
 701
 702         kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
 703         kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;
 704         kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd
 705                         - kfd->vm_info.first_vmid_kfd + 1;
 706
 707         /* Verify module parameters regarding mapped process number*/
 708         if ((hws_max_conc_proc < 0)
 709                         || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) {
 710                 dev_err(kfd_device,
 711                         "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n",
 712                         hws_max_conc_proc, kfd->vm_info.vmid_num_kfd,
 713                         kfd->vm_info.vmid_num_kfd);
 714                 kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd;
 715         } else
 716                 kfd->max_proc_per_quantum = hws_max_conc_proc;
 717
 718         /* calculate max size of mqds needed for queues */
 719         size = max_num_of_queues_per_device *
 720                         kfd->device_info->mqd_size_aligned;
 721
 722         /*
 723          * calculate max size of runlist packet.
 724          * There can be only 2 packets at once
 725          */
 726         size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
 727                 max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
 728                 + sizeof(struct pm4_mes_runlist)) * 2;
 729
 730         /* Add size of HIQ & DIQ */
 731         size += KFD_KERNEL_QUEUE_SIZE * 2;
 732
 733         /* add another 512KB for all other allocations on gart (HPD, fences) */
 734         size += 512 * 1024;
 735
 736         if (amdgpu_amdkfd_alloc_gtt_mem(
 737                         kfd->kgd, size, &kfd->gtt_mem,
 738                         &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr,
 739                         false)) {
 740                 dev_err(kfd_device, "Could not allocate %d bytes\n", size);
 741                 goto alloc_gtt_mem_failure;
 742         }
 743
 744         dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
 745
 746         /* Initialize GTT sa with 512 byte chunk size */
 747         if (kfd_gtt_sa_init(kfd, size, 512) != 0) {
 748                 dev_err(kfd_device, "Error initializing gtt sub-allocator\n");
 749                 goto kfd_gtt_sa_init_error;
 750         }
 751
 752         if (kfd_doorbell_init(kfd)) {
 753                 dev_err(kfd_device,
 754                         "Error initializing doorbell aperture\n");
 755                 goto kfd_doorbell_error;
 756         }
 757
 758         kfd->hive_id = amdgpu_amdkfd_get_hive_id(kfd->kgd);
 759
 760         kfd->noretry = amdgpu_amdkfd_get_noretry(kfd->kgd);
 761
 762         if (kfd_interrupt_init(kfd)) {
 763                 dev_err(kfd_device, "Error initializing interrupts\n");
 764                 goto kfd_interrupt_error;
 765         }
 766
 767         kfd->dqm = device_queue_manager_init(kfd);
 768         if (!kfd->dqm) {
 769                 dev_err(kfd_device, "Error initializing queue manager\n");
 770                 goto device_queue_manager_error;
 771         }
 772
 773         /* If supported on this device, allocate global GWS that is shared
 774          * by all KFD processes
 775          */
 776         if (kfd_gws_init(kfd)) {
 777                 dev_err(kfd_device, "Could not allocate %d gws\n",
 778                         amdgpu_amdkfd_get_num_gws(kfd->kgd));
 779                 goto gws_error;
 780         }
 781
 782         /* If CRAT is broken, won't set iommu enabled */
 783         kfd_double_confirm_iommu_support(kfd);
 784
 785         if (kfd_iommu_device_init(kfd)) {
 786                 dev_err(kfd_device, "Error initializing iommuv2\n");
 787                 goto device_iommu_error;
 788         }
 789
 790         kfd_cwsr_init(kfd);
 791
 792         if (kfd_resume(kfd))
 793                 goto kfd_resume_error;
 794
 795         kfd->dbgmgr = NULL;
 796
 797         if (kfd_topology_add_device(kfd)) {
 798                 dev_err(kfd_device, "Error adding device to topology\n");
 799                 goto kfd_topology_add_device_error;
 800         }
 801
 802         kfd_smi_init(kfd);
 803
 804         kfd->init_complete = true;
 805         dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
 806                  kfd->pdev->device);
 807
 808         pr_debug("Starting kfd with the following scheduling policy %d\n",
 809                 kfd->dqm->sched_policy);
 810
 811         goto out;
 812
 813 kfd_topology_add_device_error:
 814 kfd_resume_error:
 815 device_iommu_error:
 816 gws_error:
 817         device_queue_manager_uninit(kfd->dqm);
 818 device_queue_manager_error:
 819         kfd_interrupt_exit(kfd);
 820 kfd_interrupt_error:
 821         kfd_doorbell_fini(kfd);
 822 kfd_doorbell_error:
 823         kfd_gtt_sa_fini(kfd);
 824 kfd_gtt_sa_init_error:
 825         amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
 826 alloc_gtt_mem_failure:
 827         if (kfd->gws)
 828                 amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
 829         dev_err(kfd_device,
 830                 "device %x:%x NOT added due to errors\n",
 831                 kfd->pdev->vendor, kfd->pdev->device);
 832 out:
 833         return kfd->init_complete;
 834 }
 835
 836 void kgd2kfd_device_exit(struct kfd_dev *kfd)
 837 {
 838         if (kfd->init_complete) {
 839                 kgd2kfd_suspend(kfd, false);
 840                 device_queue_manager_uninit(kfd->dqm);
 841                 kfd_interrupt_exit(kfd);
 842                 kfd_topology_remove_device(kfd);
 843                 kfd_doorbell_fini(kfd);
 844                 ida_destroy(&kfd->doorbell_ida);
 845                 kfd_gtt_sa_fini(kfd);
 846                 amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
 847                 if (kfd->gws)
 848                         amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
 849         }
 850
 851         kfree(kfd);
 852 }
 853
 854 int kgd2kfd_pre_reset(struct kfd_dev *kfd)
 855 {
 856         if (!kfd->init_complete)
 857                 return 0;
 858
 859         kfd_smi_event_update_gpu_reset(kfd, false);
 860
 861         kfd->dqm->ops.pre_reset(kfd->dqm);
 862
 863         kgd2kfd_suspend(kfd, false);
 864
 865         kfd_signal_reset_event(kfd);
 866         return 0;
 867 }
 868
 869 /*
 870  * Fix me. KFD won't be able to resume existing process for now.
 871  * We will keep all existing process in a evicted state and
 872  * wait the process to be terminated.
 873  */
 874
 875 int kgd2kfd_post_reset(struct kfd_dev *kfd)
 876 {
 877         int ret;
 878
 879         if (!kfd->init_complete)
 880                 return 0;
 881
 882         ret = kfd_resume(kfd);
 883         if (ret)
 884                 return ret;
 885         atomic_dec(&kfd_locked);
 886
 887         atomic_set(&kfd->sram_ecc_flag, 0);
 888
 889         kfd_smi_event_update_gpu_reset(kfd, true);
 890
 891         return 0;
 892 }
 893
 894 bool kfd_is_locked(void)
 895 {
 896         return  (atomic_read(&kfd_locked) > 0);
 897 }
 898
 899 void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
 900 {
 901         if (!kfd->init_complete)
 902                 return;
 903
 904         /* for runtime suspend, skip locking kfd */
 905         if (!run_pm) {
 906                 /* For first KFD device suspend all the KFD processes */
 907                 if (atomic_inc_return(&kfd_locked) == 1)
 908                         kfd_suspend_all_processes();
 909         }
 910
 911         kfd->dqm->ops.stop(kfd->dqm);
 912         kfd_iommu_suspend(kfd);
 913 }
 914
 915 int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
 916 {
 917         int ret, count;
 918
 919         if (!kfd->init_complete)
 920                 return 0;
 921
 922         ret = kfd_resume(kfd);
 923         if (ret)
 924                 return ret;
 925
 926         /* for runtime resume, skip unlocking kfd */
 927         if (!run_pm) {
 928                 count = atomic_dec_return(&kfd_locked);
 929                 WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
 930                 if (count == 0)
 931                         ret = kfd_resume_all_processes();
 932         }
 933
 934         return ret;
 935 }
 936
 937 static int kfd_resume(struct kfd_dev *kfd)
 938 {
 939         int err = 0;
 940
 941         err = kfd_iommu_resume(kfd);
 942         if (err) {
 943                 dev_err(kfd_device,
 944                         "Failed to resume IOMMU for device %x:%x\n",
 945                         kfd->pdev->vendor, kfd->pdev->device);
 946                 return err;
 947         }
 948
 949         err = kfd->dqm->ops.start(kfd->dqm);
 950         if (err) {
 951                 dev_err(kfd_device,
 952                         "Error starting queue manager for device %x:%x\n",
 953                         kfd->pdev->vendor, kfd->pdev->device);
 954                 goto dqm_start_error;
 955         }
 956
 957         return err;
 958
 959 dqm_start_error:
 960         kfd_iommu_suspend(kfd);
 961         return err;
 962 }
 963
 964 static inline void kfd_queue_work(struct workqueue_struct *wq,
 965                                   struct work_struct *work)
 966 {
 967         int cpu, new_cpu;
 968
 969         cpu = new_cpu = smp_processor_id();
 970         do {
 971                 new_cpu = cpumask_next(new_cpu, cpu_online_mask) % nr_cpu_ids;
 972                 if (cpu_to_node(new_cpu) == numa_node_id())
 973                         break;
 974         } while (cpu != new_cpu);
 975
 976         queue_work_on(new_cpu, wq, work);
 977 }
 978
 979 /* This is called directly from KGD at ISR. */
 980 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 981 {
 982         uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
 983         bool is_patched = false;
 984         unsigned long flags;
 985
 986         if (!kfd->init_complete)
 987                 return;
 988
 989         if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) {
 990                 dev_err_once(kfd_device, "Ring entry too small\n");
 991                 return;
 992         }
 993
 994         spin_lock_irqsave(&kfd->interrupt_lock, flags);
 995
 996         if (kfd->interrupts_active
 997             && interrupt_is_wanted(kfd, ih_ring_entry,
 998                                    patched_ihre, &is_patched)
 999             && enqueue_ih_ring_entry(kfd,
1000                                      is_patched ? patched_ihre : ih_ring_entry))
1001                 kfd_queue_work(kfd->ih_wq, &kfd->interrupt_work);
1002
1003         spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
1004 }
1005
1006 int kgd2kfd_quiesce_mm(struct mm_struct *mm)
1007 {
1008         struct kfd_process *p;
1009         int r;
1010
1011         /* Because we are called from arbitrary context (workqueue) as opposed
1012          * to process context, kfd_process could attempt to exit while we are
1013          * running so the lookup function increments the process ref count.
1014          */
1015         p = kfd_lookup_process_by_mm(mm);
1016         if (!p)
1017                 return -ESRCH;
1018
1019         WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
1020         r = kfd_process_evict_queues(p);
1021
1022         kfd_unref_process(p);
1023         return r;
1024 }
1025
1026 int kgd2kfd_resume_mm(struct mm_struct *mm)
1027 {
1028         struct kfd_process *p;
1029         int r;
1030
1031         /* Because we are called from arbitrary context (workqueue) as opposed
1032          * to process context, kfd_process could attempt to exit while we are
1033          * running so the lookup function increments the process ref count.
1034          */
1035         p = kfd_lookup_process_by_mm(mm);
1036         if (!p)
1037                 return -ESRCH;
1038
1039         r = kfd_process_restore_queues(p);
1040
1041         kfd_unref_process(p);
1042         return r;
1043 }
1044
1045 /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
1046  *   prepare for safe eviction of KFD BOs that belong to the specified
1047  *   process.
1048  *
1049  * @mm: mm_struct that identifies the specified KFD process
1050  * @fence: eviction fence attached to KFD process BOs
1051  *
1052  */
1053 int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
1054                                                struct dma_fence *fence)
1055 {
1056         struct kfd_process *p;
1057         unsigned long active_time;
1058         unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);
1059
1060         if (!fence)
1061                 return -EINVAL;
1062
1063         if (dma_fence_is_signaled(fence))
1064                 return 0;
1065
1066         p = kfd_lookup_process_by_mm(mm);
1067         if (!p)
1068                 return -ENODEV;
1069
1070         if (fence->seqno == p->last_eviction_seqno)
1071                 goto out;
1072
1073         p->last_eviction_seqno = fence->seqno;
1074
1075         /* Avoid KFD process starvation. Wait for at least
1076          * PROCESS_ACTIVE_TIME_MS before evicting the process again
1077          */
1078         active_time = get_jiffies_64() - p->last_restore_timestamp;
1079         if (delay_jiffies > active_time)
1080                 delay_jiffies -= active_time;
1081         else
1082                 delay_jiffies = 0;
1083
1084         /* During process initialization eviction_work.dwork is initialized
1085          * to kfd_evict_bo_worker
1086          */
1087         WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies",
1088              p->lead_thread->pid, delay_jiffies);
1089         schedule_delayed_work(&p->eviction_work, delay_jiffies);
1090 out:
1091         kfd_unref_process(p);
1092         return 0;
1093 }
1094
1095 static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
1096                                 unsigned int chunk_size)
1097 {
1098         unsigned int num_of_longs;
1099
1100         if (WARN_ON(buf_size < chunk_size))
1101                 return -EINVAL;
1102         if (WARN_ON(buf_size == 0))
1103                 return -EINVAL;
1104         if (WARN_ON(chunk_size == 0))
1105                 return -EINVAL;
1106
1107         kfd->gtt_sa_chunk_size = chunk_size;
1108         kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;
1109
1110         num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) /
1111                 BITS_PER_LONG;
1112
1113         kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL);
1114
1115         if (!kfd->gtt_sa_bitmap)
1116                 return -ENOMEM;
1117
1118         pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",
1119                         kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);
1120
1121         mutex_init(&kfd->gtt_sa_lock);
1122
1123         return 0;
1124
1125 }
1126
1127 static void kfd_gtt_sa_fini(struct kfd_dev *kfd)
1128 {
1129         mutex_destroy(&kfd->gtt_sa_lock);
1130         kfree(kfd->gtt_sa_bitmap);
1131 }
1132
1133 static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr,
1134                                                 unsigned int bit_num,
1135                                                 unsigned int chunk_size)
1136 {
1137         return start_addr + bit_num * chunk_size;
1138 }
1139
1140 static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr,
1141                                                 unsigned int bit_num,
1142                                                 unsigned int chunk_size)
1143 {
1144         return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size);
1145 }
1146
1147 int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
1148                         struct kfd_mem_obj **mem_obj)
1149 {
1150         unsigned int found, start_search, cur_size;
1151
1152         if (size == 0)
1153                 return -EINVAL;
1154
1155         if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
1156                 return -ENOMEM;
1157
1158         *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
1159         if (!(*mem_obj))
1160                 return -ENOMEM;
1161
1162         pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
1163
1164         start_search = 0;
1165
1166         mutex_lock(&kfd->gtt_sa_lock);
1167
1168 kfd_gtt_restart_search:
1169         /* Find the first chunk that is free */
1170         found = find_next_zero_bit(kfd->gtt_sa_bitmap,
1171                                         kfd->gtt_sa_num_of_chunks,
1172                                         start_search);
1173
1174         pr_debug("Found = %d\n", found);
1175
1176         /* If there wasn't any free chunk, bail out */
1177         if (found == kfd->gtt_sa_num_of_chunks)
1178                 goto kfd_gtt_no_free_chunk;
1179
1180         /* Update fields of mem_obj */
1181         (*mem_obj)->range_start = found;
1182         (*mem_obj)->range_end = found;
1183         (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr(
1184                                         kfd->gtt_start_gpu_addr,
1185                                         found,
1186                                         kfd->gtt_sa_chunk_size);
1187         (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr(
1188                                         kfd->gtt_start_cpu_ptr,
1189                                         found,
1190                                         kfd->gtt_sa_chunk_size);
1191
1192         pr_debug("gpu_addr = %p, cpu_addr = %p\n",
1193                         (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr);
1194
1195         /* If we need only one chunk, mark it as allocated and get out */
1196         if (size <= kfd->gtt_sa_chunk_size) {
1197                 pr_debug("Single bit\n");
1198                 set_bit(found, kfd->gtt_sa_bitmap);
1199                 goto kfd_gtt_out;
1200         }
1201
1202         /* Otherwise, try to see if we have enough contiguous chunks */
1203         cur_size = size - kfd->gtt_sa_chunk_size;
1204         do {
1205                 (*mem_obj)->range_end =
1206                         find_next_zero_bit(kfd->gtt_sa_bitmap,
1207                                         kfd->gtt_sa_num_of_chunks, ++found);
1208                 /*
1209                  * If next free chunk is not contiguous than we need to
1210                  * restart our search from the last free chunk we found (which
1211                  * wasn't contiguous to the previous ones
1212                  */
1213                 if ((*mem_obj)->range_end != found) {
1214                         start_search = found;
1215                         goto kfd_gtt_restart_search;
1216                 }
1217
1218                 /*
1219                  * If we reached end of buffer, bail out with error
1220                  */
1221                 if (found == kfd->gtt_sa_num_of_chunks)
1222                         goto kfd_gtt_no_free_chunk;
1223
1224                 /* Check if we don't need another chunk */
1225                 if (cur_size <= kfd->gtt_sa_chunk_size)
1226                         cur_size = 0;
1227                 else
1228                         cur_size -= kfd->gtt_sa_chunk_size;
1229
1230         } while (cur_size > 0);
1231
1232         pr_debug("range_start = %d, range_end = %d\n",
1233                 (*mem_obj)->range_start, (*mem_obj)->range_end);
1234
1235         /* Mark the chunks as allocated */
1236         for (found = (*mem_obj)->range_start;
1237                 found <= (*mem_obj)->range_end;
1238                 found++)
1239                 set_bit(found, kfd->gtt_sa_bitmap);
1240
1241 kfd_gtt_out:
1242         mutex_unlock(&kfd->gtt_sa_lock);
1243         return 0;
1244
1245 kfd_gtt_no_free_chunk:
1246         pr_debug("Allocation failed with mem_obj = %p\n", *mem_obj);
1247         mutex_unlock(&kfd->gtt_sa_lock);
1248         kfree(*mem_obj);
1249         return -ENOMEM;
1250 }
1251
1252 int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj)
1253 {
1254         unsigned int bit;
1255
1256         /* Act like kfree when trying to free a NULL object */
1257         if (!mem_obj)
1258                 return 0;
1259
1260         pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n",
1261                         mem_obj, mem_obj->range_start, mem_obj->range_end);
1262
1263         mutex_lock(&kfd->gtt_sa_lock);
1264
1265         /* Mark the chunks as free */
1266         for (bit = mem_obj->range_start;
1267                 bit <= mem_obj->range_end;
1268                 bit++)
1269                 clear_bit(bit, kfd->gtt_sa_bitmap);
1270
1271         mutex_unlock(&kfd->gtt_sa_lock);
1272
1273         kfree(mem_obj);
1274         return 0;
1275 }
1276
1277 void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
1278 {
1279         if (kfd)
1280                 atomic_inc(&kfd->sram_ecc_flag);
1281 }
1282
1283 void kfd_inc_compute_active(struct kfd_dev *kfd)
1284 {
1285         if (atomic_inc_return(&kfd->compute_profile) == 1)
1286                 amdgpu_amdkfd_set_compute_idle(kfd->kgd, false);
1287 }
1288
1289 void kfd_dec_compute_active(struct kfd_dev *kfd)
1290 {
1291         int count = atomic_dec_return(&kfd->compute_profile);
1292
1293         if (count == 0)
1294                 amdgpu_amdkfd_set_compute_idle(kfd->kgd, true);
1295         WARN_ONCE(count < 0, "Compute profile ref. count error");
1296 }
1297
1298 void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask)
1299 {
1300         if (kfd)
1301                 kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask);
1302 }
1303
1304 #if defined(CONFIG_DEBUG_FS)
1305
1306 /* This function will send a package to HIQ to hang the HWS
1307  * which will trigger a GPU reset and bring the HWS back to normal state
1308  */
1309 int kfd_debugfs_hang_hws(struct kfd_dev *dev)
1310 {
1311         int r = 0;
1312
1313         if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
1314                 pr_err("HWS is not enabled");
1315                 return -EINVAL;
1316         }
1317
1318         r = pm_debugfs_hang_hws(&dev->dqm->packets);
1319         if (!r)
1320                 r = dqm_debugfs_execute_queues(dev->dqm);
1321
1322         return r;
1323 }
1324
1325 #endif