drivers/iommu/intel/svm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright © 2015 Intel Corporation.
   4  *
   5  * Authors: David Woodhouse <dwmw2@infradead.org>
   6  */
   7
   8 #include <linux/mmu_notifier.h>
   9 #include <linux/sched.h>
  10 #include <linux/sched/mm.h>
  11 #include <linux/slab.h>
  12 #include <linux/rculist.h>
  13 #include <linux/pci.h>
  14 #include <linux/pci-ats.h>
  15 #include <linux/dmar.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/mm_types.h>
  18 #include <linux/xarray.h>
  19 #include <asm/page.h>
  20 #include <asm/fpu/api.h>
  21
  22 #include "iommu.h"
  23 #include "pasid.h"
  24 #include "perf.h"
  25 #include "../iommu-pages.h"
  26 #include "trace.h"
  27
  28 void intel_svm_check(struct intel_iommu *iommu)
  29 {
  30         if (!pasid_supported(iommu))
  31                 return;
  32
  33         if (cpu_feature_enabled(X86_FEATURE_GBPAGES) &&
  34             !cap_fl1gp_support(iommu->cap)) {
  35                 pr_err("%s SVM disabled, incompatible 1GB page capability\n",
  36                        iommu->name);
  37                 return;
  38         }
  39
  40         if (cpu_feature_enabled(X86_FEATURE_LA57) &&
  41             !cap_fl5lp_support(iommu->cap)) {
  42                 pr_err("%s SVM disabled, incompatible paging mode\n",
  43                        iommu->name);
  44                 return;
  45         }
  46
  47         iommu->flags |= VTD_FLAG_SVM_CAPABLE;
  48 }
  49
  50 /* Pages have been freed at this point */
  51 static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
  52                                         struct mm_struct *mm,
  53                                         unsigned long start, unsigned long end)
  54 {
  55         struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier);
  56
  57         if (start == 0 && end == ULONG_MAX) {
  58                 cache_tag_flush_all(domain);
  59                 return;
  60         }
  61
  62         /*
  63          * The mm_types defines vm_end as the first byte after the end address,
  64          * different from IOMMU subsystem using the last address of an address
  65          * range.
  66          */
  67         cache_tag_flush_range(domain, start, end - 1, 0);
  68 }
  69
  70 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
  71 {
  72         struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier);
  73         struct dev_pasid_info *dev_pasid;
  74         struct device_domain_info *info;
  75         unsigned long flags;
  76
  77         /* This might end up being called from exit_mmap(), *before* the page
  78          * tables are cleared. And __mmu_notifier_release() will delete us from
  79          * the list of notifiers so that our invalidate_range() callback doesn't
  80          * get called when the page tables are cleared. So we need to protect
  81          * against hardware accessing those page tables.
  82          *
  83          * We do it by clearing the entry in the PASID table and then flushing
  84          * the IOTLB and the PASID table caches. This might upset hardware;
  85          * perhaps we'll want to point the PASID to a dummy PGD (like the zero
  86          * page) so that we end up taking a fault that the hardware really
  87          * *has* to handle gracefully without affecting other processes.
  88          */
  89         spin_lock_irqsave(&domain->lock, flags);
  90         list_for_each_entry(dev_pasid, &domain->dev_pasids, link_domain) {
  91                 info = dev_iommu_priv_get(dev_pasid->dev);
  92                 intel_pasid_tear_down_entry(info->iommu, dev_pasid->dev,
  93                                             dev_pasid->pasid, true);
  94         }
  95         spin_unlock_irqrestore(&domain->lock, flags);
  96
  97 }
  98
  99 static void intel_mm_free_notifier(struct mmu_notifier *mn)
 100 {
 101         struct dmar_domain *domain = container_of(mn, struct dmar_domain, notifier);
 102
 103         kfree(domain->qi_batch);
 104         kfree(domain);
 105 }
 106
 107 static const struct mmu_notifier_ops intel_mmuops = {
 108         .release = intel_mm_release,
 109         .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs,
 110         .free_notifier = intel_mm_free_notifier,
 111 };
 112
 113 static int intel_svm_set_dev_pasid(struct iommu_domain *domain,
 114                                    struct device *dev, ioasid_t pasid,
 115                                    struct iommu_domain *old)
 116 {
 117         struct device_domain_info *info = dev_iommu_priv_get(dev);
 118         struct intel_iommu *iommu = info->iommu;
 119         struct mm_struct *mm = domain->mm;
 120         struct dev_pasid_info *dev_pasid;
 121         unsigned long sflags;
 122         int ret = 0;
 123
 124         dev_pasid = domain_add_dev_pasid(domain, dev, pasid);
 125         if (IS_ERR(dev_pasid))
 126                 return PTR_ERR(dev_pasid);
 127
 128         /* Setup the pasid table: */
 129         sflags = cpu_feature_enabled(X86_FEATURE_LA57) ? PASID_FLAG_FL5LP : 0;
 130         ret = __domain_setup_first_level(iommu, dev, pasid,
 131                                          FLPT_DEFAULT_DID, mm->pgd,
 132                                          sflags, old);
 133         if (ret)
 134                 goto out_remove_dev_pasid;
 135
 136         domain_remove_dev_pasid(old, dev, pasid);
 137
 138         return 0;
 139
 140 out_remove_dev_pasid:
 141         domain_remove_dev_pasid(domain, dev, pasid);
 142         return ret;
 143 }
 144
 145 static void intel_svm_domain_free(struct iommu_domain *domain)
 146 {
 147         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
 148
 149         /* dmar_domain free is deferred to the mmu free_notifier callback. */
 150         mmu_notifier_put(&dmar_domain->notifier);
 151 }
 152
 153 static const struct iommu_domain_ops intel_svm_domain_ops = {
 154         .set_dev_pasid          = intel_svm_set_dev_pasid,
 155         .free                   = intel_svm_domain_free
 156 };
 157
 158 struct iommu_domain *intel_svm_domain_alloc(struct device *dev,
 159                                             struct mm_struct *mm)
 160 {
 161         struct dmar_domain *domain;
 162         int ret;
 163
 164         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
 165         if (!domain)
 166                 return ERR_PTR(-ENOMEM);
 167
 168         domain->domain.ops = &intel_svm_domain_ops;
 169         domain->use_first_level = true;
 170         INIT_LIST_HEAD(&domain->dev_pasids);
 171         INIT_LIST_HEAD(&domain->cache_tags);
 172         spin_lock_init(&domain->cache_lock);
 173         spin_lock_init(&domain->lock);
 174
 175         domain->notifier.ops = &intel_mmuops;
 176         ret = mmu_notifier_register(&domain->notifier, mm);
 177         if (ret) {
 178                 kfree(domain);
 179                 return ERR_PTR(ret);
 180         }
 181
 182         return &domain->domain;
 183 }