drivers/gpu/drm/amd/amdkfd/kfd_iommu.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/printk.h>
  24 #include <linux/device.h>
  25 #include <linux/slab.h>
  26 #include <linux/pci.h>
  27 #include <linux/amd-iommu.h>
  28 #include "kfd_priv.h"
  29 #include "kfd_dbgmgr.h"
  30 #include "kfd_topology.h"
  31 #include "kfd_iommu.h"
  32
  33 static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
  34                                         AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
  35                                         AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
  36
  37 /** kfd_iommu_check_device - Check whether IOMMU is available for device
  38  */
  39 int kfd_iommu_check_device(struct kfd_dev *kfd)
  40 {
  41         struct amd_iommu_device_info iommu_info;
  42         int err;
  43
  44         if (!kfd->device_info->needs_iommu_device)
  45                 return -ENODEV;
  46
  47         iommu_info.flags = 0;
  48         err = amd_iommu_device_info(kfd->pdev, &iommu_info);
  49         if (err)
  50                 return err;
  51
  52         if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
  53                 return -ENODEV;
  54
  55         return 0;
  56 }
  57
  58 /** kfd_iommu_device_init - Initialize IOMMU for device
  59  */
  60 int kfd_iommu_device_init(struct kfd_dev *kfd)
  61 {
  62         struct amd_iommu_device_info iommu_info;
  63         unsigned int pasid_limit;
  64         int err;
  65         struct kfd_topology_device *top_dev;
  66
  67         top_dev = kfd_topology_device_by_id(kfd->id);
  68
  69         /*
  70          * Overwrite ATS capability according to needs_iommu_device to fix
  71          * potential missing corresponding bit in CRAT of BIOS.
  72          */
  73         if (!kfd->device_info->needs_iommu_device) {
  74                 top_dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
  75                 return 0;
  76         }
  77
  78         top_dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
  79
  80         iommu_info.flags = 0;
  81         err = amd_iommu_device_info(kfd->pdev, &iommu_info);
  82         if (err < 0) {
  83                 dev_err(kfd_device,
  84                         "error getting iommu info. is the iommu enabled?\n");
  85                 return -ENODEV;
  86         }
  87
  88         if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
  89                 dev_err(kfd_device,
  90                         "error required iommu flags ats %i, pri %i, pasid %i\n",
  91                        (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
  92                        (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
  93                        (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
  94                                                                         != 0);
  95                 return -ENODEV;
  96         }
  97
  98         pasid_limit = min_t(unsigned int,
  99                         (unsigned int)(1 << kfd->device_info->max_pasid_bits),
 100                         iommu_info.max_pasids);
 101
 102         if (!kfd_set_pasid_limit(pasid_limit)) {
 103                 dev_err(kfd_device, "error setting pasid limit\n");
 104                 return -EBUSY;
 105         }
 106
 107         return 0;
 108 }
 109
 110 /** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
 111  *
 112  * Binds the given process to the given device using its PASID. This
 113  * enables IOMMUv2 address translation for the process on the device.
 114  *
 115  * This function assumes that the process mutex is held.
 116  */
 117 int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
 118 {
 119         struct kfd_dev *dev = pdd->dev;
 120         struct kfd_process *p = pdd->process;
 121         int err;
 122
 123         if (!dev->device_info->needs_iommu_device || pdd->bound == PDD_BOUND)
 124                 return 0;
 125
 126         if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
 127                 pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
 128                 return -EINVAL;
 129         }
 130
 131         err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
 132         if (!err)
 133                 pdd->bound = PDD_BOUND;
 134
 135         return err;
 136 }
 137
 138 /** kfd_iommu_unbind_process - Unbind process from all devices
 139  *
 140  * This removes all IOMMU device bindings of the process. To be used
 141  * before process termination.
 142  */
 143 void kfd_iommu_unbind_process(struct kfd_process *p)
 144 {
 145         struct kfd_process_device *pdd;
 146
 147         list_for_each_entry(pdd, &p->per_device_data, per_device_list)
 148                 if (pdd->bound == PDD_BOUND)
 149                         amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
 150 }
 151
 152 /* Callback for process shutdown invoked by the IOMMU driver */
 153 static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
 154 {
 155         struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
 156         struct kfd_process *p;
 157         struct kfd_process_device *pdd;
 158
 159         if (!dev)
 160                 return;
 161
 162         /*
 163          * Look for the process that matches the pasid. If there is no such
 164          * process, we either released it in amdkfd's own notifier, or there
 165          * is a bug. Unfortunately, there is no way to tell...
 166          */
 167         p = kfd_lookup_process_by_pasid(pasid);
 168         if (!p)
 169                 return;
 170
 171         pr_debug("Unbinding process %d from IOMMU\n", pasid);
 172
 173         mutex_lock(kfd_get_dbgmgr_mutex());
 174
 175         if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
 176                 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
 177                         kfd_dbgmgr_destroy(dev->dbgmgr);
 178                         dev->dbgmgr = NULL;
 179                 }
 180         }
 181
 182         mutex_unlock(kfd_get_dbgmgr_mutex());
 183
 184         mutex_lock(&p->mutex);
 185
 186         pdd = kfd_get_process_device_data(dev, p);
 187         if (pdd)
 188                 /* For GPU relying on IOMMU, we need to dequeue here
 189                  * when PASID is still bound.
 190                  */
 191                 kfd_process_dequeue_from_device(pdd);
 192
 193         mutex_unlock(&p->mutex);
 194
 195         kfd_unref_process(p);
 196 }
 197
 198 /* This function called by IOMMU driver on PPR failure */
 199 static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
 200                 unsigned long address, u16 flags)
 201 {
 202         struct kfd_dev *dev;
 203
 204         dev_warn_ratelimited(kfd_device,
 205                         "Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",
 206                         PCI_BUS_NUM(pdev->devfn),
 207                         PCI_SLOT(pdev->devfn),
 208                         PCI_FUNC(pdev->devfn),
 209                         pasid,
 210                         address,
 211                         flags);
 212
 213         dev = kfd_device_by_pci_dev(pdev);
 214         if (!WARN_ON(!dev))
 215                 kfd_signal_iommu_event(dev, pasid, address,
 216                         flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
 217
 218         return AMD_IOMMU_INV_PRI_RSP_INVALID;
 219 }
 220
 221 /*
 222  * Bind processes do the device that have been temporarily unbound
 223  * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
 224  */
 225 static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
 226 {
 227         struct kfd_process_device *pdd;
 228         struct kfd_process *p;
 229         unsigned int temp;
 230         int err = 0;
 231
 232         int idx = srcu_read_lock(&kfd_processes_srcu);
 233
 234         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 235                 mutex_lock(&p->mutex);
 236                 pdd = kfd_get_process_device_data(kfd, p);
 237
 238                 if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
 239                         mutex_unlock(&p->mutex);
 240                         continue;
 241                 }
 242
 243                 err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
 244                                 p->lead_thread);
 245                 if (err < 0) {
 246                         pr_err("Unexpected pasid %d binding failure\n",
 247                                         p->pasid);
 248                         mutex_unlock(&p->mutex);
 249                         break;
 250                 }
 251
 252                 pdd->bound = PDD_BOUND;
 253                 mutex_unlock(&p->mutex);
 254         }
 255
 256         srcu_read_unlock(&kfd_processes_srcu, idx);
 257
 258         return err;
 259 }
 260
 261 /*
 262  * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
 263  * processes will be restored to PDD_BOUND state in
 264  * kfd_bind_processes_to_device.
 265  */
 266 static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
 267 {
 268         struct kfd_process_device *pdd;
 269         struct kfd_process *p;
 270         unsigned int temp;
 271
 272         int idx = srcu_read_lock(&kfd_processes_srcu);
 273
 274         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
 275                 mutex_lock(&p->mutex);
 276                 pdd = kfd_get_process_device_data(kfd, p);
 277
 278                 if (WARN_ON(!pdd)) {
 279                         mutex_unlock(&p->mutex);
 280                         continue;
 281                 }
 282
 283                 if (pdd->bound == PDD_BOUND)
 284                         pdd->bound = PDD_BOUND_SUSPENDED;
 285                 mutex_unlock(&p->mutex);
 286         }
 287
 288         srcu_read_unlock(&kfd_processes_srcu, idx);
 289 }
 290
 291 /** kfd_iommu_suspend - Prepare IOMMU for suspend
 292  *
 293  * This unbinds processes from the device and disables the IOMMU for
 294  * the device.
 295  */
 296 void kfd_iommu_suspend(struct kfd_dev *kfd)
 297 {
 298         if (!kfd->device_info->needs_iommu_device)
 299                 return;
 300
 301         kfd_unbind_processes_from_device(kfd);
 302
 303         amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
 304         amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
 305         amd_iommu_free_device(kfd->pdev);
 306 }
 307
 308 /** kfd_iommu_resume - Restore IOMMU after resume
 309  *
 310  * This reinitializes the IOMMU for the device and re-binds previously
 311  * suspended processes to the device.
 312  */
 313 int kfd_iommu_resume(struct kfd_dev *kfd)
 314 {
 315         unsigned int pasid_limit;
 316         int err;
 317
 318         if (!kfd->device_info->needs_iommu_device)
 319                 return 0;
 320
 321         pasid_limit = kfd_get_pasid_limit();
 322
 323         err = amd_iommu_init_device(kfd->pdev, pasid_limit);
 324         if (err)
 325                 return -ENXIO;
 326
 327         amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
 328                                         iommu_pasid_shutdown_callback);
 329         amd_iommu_set_invalid_ppr_cb(kfd->pdev,
 330                                      iommu_invalid_ppr_cb);
 331
 332         err = kfd_bind_processes_to_device(kfd);
 333         if (err) {
 334                 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
 335                 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
 336                 amd_iommu_free_device(kfd->pdev);
 337                 return err;
 338         }
 339
 340         return 0;
 341 }
 342
 343 extern bool amd_iommu_pc_supported(void);
 344 extern u8 amd_iommu_pc_get_max_banks(u16 devid);
 345 extern u8 amd_iommu_pc_get_max_counters(u16 devid);
 346
 347 /** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
 348  */
 349 int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
 350 {
 351         struct kfd_perf_properties *props;
 352
 353         if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
 354                 return 0;
 355
 356         if (!amd_iommu_pc_supported())
 357                 return 0;
 358
 359         props = kfd_alloc_struct(props);
 360         if (!props)
 361                 return -ENOMEM;
 362         strcpy(props->block_name, "iommu");
 363         props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
 364                 amd_iommu_pc_get_max_counters(0); /* assume one iommu */
 365         list_add_tail(&props->list, &kdev->perf_props);
 366
 367         return 0;
 368 }