drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Implementation of the IOMMU SVA API for the ARM SMMUv3
   4  */
   5
   6 #include <linux/mm.h>
   7 #include <linux/mmu_context.h>
   8 #include <linux/mmu_notifier.h>
   9 #include <linux/sched/mm.h>
  10 #include <linux/slab.h>
  11 #include <kunit/visibility.h>
  12
  13 #include "arm-smmu-v3.h"
  14 #include "../../io-pgtable-arm.h"
  15
  16 static DEFINE_MUTEX(sva_lock);
  17
  18 static void __maybe_unused
  19 arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain)
  20 {
  21         struct arm_smmu_master_domain *master_domain;
  22         struct arm_smmu_cd target_cd;
  23         unsigned long flags;
  24
  25         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
  26         list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) {
  27                 struct arm_smmu_master *master = master_domain->master;
  28                 struct arm_smmu_cd *cdptr;
  29
  30                 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid);
  31                 if (WARN_ON(!cdptr))
  32                         continue;
  33
  34                 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
  35                 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
  36                                         &target_cd);
  37         }
  38         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
  39 }
  40
  41 static u64 page_size_to_cd(void)
  42 {
  43         static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K ||
  44                       PAGE_SIZE == SZ_64K);
  45         if (PAGE_SIZE == SZ_64K)
  46                 return ARM_LPAE_TCR_TG0_64K;
  47         if (PAGE_SIZE == SZ_16K)
  48                 return ARM_LPAE_TCR_TG0_16K;
  49         return ARM_LPAE_TCR_TG0_4K;
  50 }
  51
  52 VISIBLE_IF_KUNIT
  53 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
  54                           struct arm_smmu_master *master, struct mm_struct *mm,
  55                           u16 asid)
  56 {
  57         u64 par;
  58
  59         memset(target, 0, sizeof(*target));
  60
  61         par = cpuid_feature_extract_unsigned_field(
  62                 read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1),
  63                 ID_AA64MMFR0_EL1_PARANGE_SHIFT);
  64
  65         target->data[0] = cpu_to_le64(
  66                 CTXDESC_CD_0_TCR_EPD1 |
  67 #ifdef __BIG_ENDIAN
  68                 CTXDESC_CD_0_ENDI |
  69 #endif
  70                 CTXDESC_CD_0_V |
  71                 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) |
  72                 CTXDESC_CD_0_AA64 |
  73                 (master->stall_enabled ? CTXDESC_CD_0_S : 0) |
  74                 CTXDESC_CD_0_R |
  75                 CTXDESC_CD_0_A |
  76                 CTXDESC_CD_0_ASET |
  77                 FIELD_PREP(CTXDESC_CD_0_ASID, asid));
  78
  79         /*
  80          * If no MM is passed then this creates a SVA entry that faults
  81          * everything. arm_smmu_write_cd_entry() can hitlessly go between these
  82          * two entries types since TTB0 is ignored by HW when EPD0 is set.
  83          */
  84         if (mm) {
  85                 target->data[0] |= cpu_to_le64(
  86                         FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ,
  87                                    64ULL - vabits_actual) |
  88                         FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) |
  89                         FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0,
  90                                    ARM_LPAE_TCR_RGN_WBWA) |
  91                         FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0,
  92                                    ARM_LPAE_TCR_RGN_WBWA) |
  93                         FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS));
  94
  95                 target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) &
  96                                               CTXDESC_CD_1_TTB0_MASK);
  97         } else {
  98                 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0);
  99
 100                 /*
 101                  * Disable stall and immediately generate an abort if stall
 102                  * disable is permitted. This speeds up cleanup for an unclean
 103                  * exit if the device is still doing a lot of DMA.
 104                  */
 105                 if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
 106                         target->data[0] &=
 107                                 cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R));
 108         }
 109
 110         /*
 111          * MAIR value is pretty much constant and global, so we can just get it
 112          * from the current CPU register
 113          */
 114         target->data[3] = cpu_to_le64(read_sysreg(mair_el1));
 115 }
 116 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd);
 117
 118 /*
 119  * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this
 120  * is used as a threshold to replace per-page TLBI commands to issue in the
 121  * command queue with an address-space TLBI command, when SMMU w/o a range
 122  * invalidation feature handles too many per-page TLBI commands, which will
 123  * otherwise result in a soft lockup.
 124  */
 125 #define CMDQ_MAX_TLBI_OPS               (1 << (PAGE_SHIFT - 3))
 126
 127 static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
 128                                                 struct mm_struct *mm,
 129                                                 unsigned long start,
 130                                                 unsigned long end)
 131 {
 132         struct arm_smmu_domain *smmu_domain =
 133                 container_of(mn, struct arm_smmu_domain, mmu_notifier);
 134         size_t size;
 135
 136         /*
 137          * The mm_types defines vm_end as the first byte after the end address,
 138          * different from IOMMU subsystem using the last address of an address
 139          * range. So do a simple translation here by calculating size correctly.
 140          */
 141         size = end - start;
 142         if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) {
 143                 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE)
 144                         size = 0;
 145         } else {
 146                 if (size == ULONG_MAX)
 147                         size = 0;
 148         }
 149
 150         if (!size)
 151                 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
 152         else
 153                 arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid,
 154                                             PAGE_SIZE, false, smmu_domain);
 155
 156         arm_smmu_atc_inv_domain(smmu_domain, start, size);
 157 }
 158
 159 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
 160 {
 161         struct arm_smmu_domain *smmu_domain =
 162                 container_of(mn, struct arm_smmu_domain, mmu_notifier);
 163         struct arm_smmu_master_domain *master_domain;
 164         unsigned long flags;
 165
 166         /*
 167          * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
 168          * but disable translation.
 169          */
 170         spin_lock_irqsave(&smmu_domain->devices_lock, flags);
 171         list_for_each_entry(master_domain, &smmu_domain->devices,
 172                             devices_elm) {
 173                 struct arm_smmu_master *master = master_domain->master;
 174                 struct arm_smmu_cd target;
 175                 struct arm_smmu_cd *cdptr;
 176
 177                 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid);
 178                 if (WARN_ON(!cdptr))
 179                         continue;
 180                 arm_smmu_make_sva_cd(&target, master, NULL,
 181                                      smmu_domain->cd.asid);
 182                 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
 183                                         &target);
 184         }
 185         spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
 186
 187         arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
 188         arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
 189 }
 190
 191 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
 192 {
 193         kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier));
 194 }
 195
 196 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
 197         .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs,
 198         .release                        = arm_smmu_mm_release,
 199         .free_notifier                  = arm_smmu_mmu_notifier_free,
 200 };
 201
 202 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
 203 {
 204         unsigned long reg, fld;
 205         unsigned long oas;
 206         unsigned long asid_bits;
 207         u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
 208
 209         if (vabits_actual == 52)
 210                 feat_mask |= ARM_SMMU_FEAT_VAX;
 211
 212         if ((smmu->features & feat_mask) != feat_mask)
 213                 return false;
 214
 215         if (!(smmu->pgsize_bitmap & PAGE_SIZE))
 216                 return false;
 217
 218         /*
 219          * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're
 220          * not even pretending to support AArch32 here. Abort if the MMU outputs
 221          * addresses larger than what we support.
 222          */
 223         reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 224         fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 225         oas = id_aa64mmfr0_parange_to_phys_shift(fld);
 226         if (smmu->oas < oas)
 227                 return false;
 228
 229         /* We can support bigger ASIDs than the CPU, but not smaller */
 230         fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
 231         asid_bits = fld ? 16 : 8;
 232         if (smmu->asid_bits < asid_bits)
 233                 return false;
 234
 235         /*
 236          * See max_pinned_asids in arch/arm64/mm/context.c. The following is
 237          * generally the maximum number of bindable processes.
 238          */
 239         if (arm64_kernel_unmapped_at_el0())
 240                 asid_bits--;
 241         dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) -
 242                 num_possible_cpus() - 2);
 243
 244         return true;
 245 }
 246
 247 bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
 248 {
 249         /* We're not keeping track of SIDs in fault events */
 250         if (master->num_streams != 1)
 251                 return false;
 252
 253         return master->stall_enabled;
 254 }
 255
 256 bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
 257 {
 258         if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
 259                 return false;
 260
 261         /* SSID support is mandatory for the moment */
 262         return master->ssid_bits;
 263 }
 264
 265 bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
 266 {
 267         bool enabled;
 268
 269         mutex_lock(&sva_lock);
 270         enabled = master->sva_enabled;
 271         mutex_unlock(&sva_lock);
 272         return enabled;
 273 }
 274
 275 static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master)
 276 {
 277         struct device *dev = master->dev;
 278
 279         /*
 280          * Drivers for devices supporting PRI or stall should enable IOPF first.
 281          * Others have device-specific fault handlers and don't need IOPF.
 282          */
 283         if (!arm_smmu_master_iopf_supported(master))
 284                 return 0;
 285
 286         if (!master->iopf_enabled)
 287                 return -EINVAL;
 288
 289         return iopf_queue_add_device(master->smmu->evtq.iopf, dev);
 290 }
 291
 292 static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master)
 293 {
 294         struct device *dev = master->dev;
 295
 296         if (!master->iopf_enabled)
 297                 return;
 298
 299         iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
 300 }
 301
 302 int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
 303 {
 304         int ret;
 305
 306         mutex_lock(&sva_lock);
 307         ret = arm_smmu_master_sva_enable_iopf(master);
 308         if (!ret)
 309                 master->sva_enabled = true;
 310         mutex_unlock(&sva_lock);
 311
 312         return ret;
 313 }
 314
 315 int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
 316 {
 317         mutex_lock(&sva_lock);
 318         arm_smmu_master_sva_disable_iopf(master);
 319         master->sva_enabled = false;
 320         mutex_unlock(&sva_lock);
 321
 322         return 0;
 323 }
 324
 325 void arm_smmu_sva_notifier_synchronize(void)
 326 {
 327         /*
 328          * Some MMU notifiers may still be waiting to be freed, using
 329          * arm_smmu_mmu_notifier_free(). Wait for them.
 330          */
 331         mmu_notifier_synchronize();
 332 }
 333
 334 static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
 335                                       struct device *dev, ioasid_t id,
 336                                       struct iommu_domain *old)
 337 {
 338         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 339         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 340         struct arm_smmu_cd target;
 341         int ret;
 342
 343         /* Prevent arm_smmu_mm_release from being called while we are attaching */
 344         if (!mmget_not_zero(domain->mm))
 345                 return -EINVAL;
 346
 347         /*
 348          * This does not need the arm_smmu_asid_lock because SVA domains never
 349          * get reassigned
 350          */
 351         arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid);
 352         ret = arm_smmu_set_pasid(master, smmu_domain, id, &target, old);
 353
 354         mmput(domain->mm);
 355         return ret;
 356 }
 357
 358 static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
 359 {
 360         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 361
 362         /*
 363          * Ensure the ASID is empty in the iommu cache before allowing reuse.
 364          */
 365         arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
 366
 367         /*
 368          * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can
 369          * still be called/running at this point. We allow the ASID to be
 370          * reused, and if there is a race then it just suffers harmless
 371          * unnecessary invalidation.
 372          */
 373         xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
 374
 375         /*
 376          * Actual free is defered to the SRCU callback
 377          * arm_smmu_mmu_notifier_free()
 378          */
 379         mmu_notifier_put(&smmu_domain->mmu_notifier);
 380 }
 381
 382 static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
 383         .set_dev_pasid          = arm_smmu_sva_set_dev_pasid,
 384         .free                   = arm_smmu_sva_domain_free
 385 };
 386
 387 struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
 388                                                struct mm_struct *mm)
 389 {
 390         struct arm_smmu_master *master = dev_iommu_priv_get(dev);
 391         struct arm_smmu_device *smmu = master->smmu;
 392         struct arm_smmu_domain *smmu_domain;
 393         u32 asid;
 394         int ret;
 395
 396         smmu_domain = arm_smmu_domain_alloc();
 397         if (IS_ERR(smmu_domain))
 398                 return ERR_CAST(smmu_domain);
 399         smmu_domain->domain.type = IOMMU_DOMAIN_SVA;
 400         smmu_domain->domain.ops = &arm_smmu_sva_domain_ops;
 401         smmu_domain->smmu = smmu;
 402
 403         ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
 404                        XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
 405         if (ret)
 406                 goto err_free;
 407
 408         smmu_domain->cd.asid = asid;
 409         smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops;
 410         ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm);
 411         if (ret)
 412                 goto err_asid;
 413
 414         return &smmu_domain->domain;
 415
 416 err_asid:
 417         xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
 418 err_free:
 419         kfree(smmu_domain);
 420         return ERR_PTR(ret);
 421 }