drivers/iommu/fsl_pamu_domain.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *
   4  * Copyright (C) 2013 Freescale Semiconductor, Inc.
   5  * Author: Varun Sethi <varun.sethi@freescale.com>
   6  */
   7
   8 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
   9
  10 #include "fsl_pamu_domain.h"
  11
  12 #include <linux/platform_device.h>
  13 #include <sysdev/fsl_pci.h>
  14
  15 /*
  16  * Global spinlock that needs to be held while
  17  * configuring PAMU.
  18  */
  19 static DEFINE_SPINLOCK(iommu_lock);
  20
  21 static struct kmem_cache *fsl_pamu_domain_cache;
  22 static struct kmem_cache *iommu_devinfo_cache;
  23 static DEFINE_SPINLOCK(device_domain_lock);
  24
  25 struct iommu_device pamu_iommu; /* IOMMU core code handle */
  26
  27 static struct fsl_dma_domain *to_fsl_dma_domain(struct iommu_domain *dom)
  28 {
  29         return container_of(dom, struct fsl_dma_domain, iommu_domain);
  30 }
  31
  32 static int __init iommu_init_mempool(void)
  33 {
  34         fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  35                                                   sizeof(struct fsl_dma_domain),
  36                                                   0,
  37                                                   SLAB_HWCACHE_ALIGN,
  38                                                   NULL);
  39         if (!fsl_pamu_domain_cache) {
  40                 pr_debug("Couldn't create fsl iommu_domain cache\n");
  41                 return -ENOMEM;
  42         }
  43
  44         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  45                                                 sizeof(struct device_domain_info),
  46                                                 0,
  47                                                 SLAB_HWCACHE_ALIGN,
  48                                                 NULL);
  49         if (!iommu_devinfo_cache) {
  50                 pr_debug("Couldn't create devinfo cache\n");
  51                 kmem_cache_destroy(fsl_pamu_domain_cache);
  52                 return -ENOMEM;
  53         }
  54
  55         return 0;
  56 }
  57
  58 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
  59                               u32 val)
  60 {
  61         int ret = 0;
  62         unsigned long flags;
  63
  64         spin_lock_irqsave(&iommu_lock, flags);
  65         ret = pamu_update_paace_stash(liodn, val);
  66         if (ret) {
  67                 pr_debug("Failed to update SPAACE for liodn %d\n ", liodn);
  68                 spin_unlock_irqrestore(&iommu_lock, flags);
  69                 return ret;
  70         }
  71
  72         spin_unlock_irqrestore(&iommu_lock, flags);
  73
  74         return ret;
  75 }
  76
  77 /* Set the geometry parameters for a LIODN */
  78 static int pamu_set_liodn(struct fsl_dma_domain *dma_domain, struct device *dev,
  79                           int liodn)
  80 {
  81         u32 omi_index = ~(u32)0;
  82         unsigned long flags;
  83         int ret;
  84
  85         /*
  86          * Configure the omi_index at the geometry setup time.
  87          * This is a static value which depends on the type of
  88          * device and would not change thereafter.
  89          */
  90         get_ome_index(&omi_index, dev);
  91
  92         spin_lock_irqsave(&iommu_lock, flags);
  93         ret = pamu_disable_liodn(liodn);
  94         if (ret)
  95                 goto out_unlock;
  96         ret = pamu_config_ppaace(liodn, omi_index, dma_domain->stash_id, 0);
  97         if (ret)
  98                 goto out_unlock;
  99         ret = pamu_config_ppaace(liodn, ~(u32)0, dma_domain->stash_id,
 100                                  PAACE_AP_PERMS_QUERY | PAACE_AP_PERMS_UPDATE);
 101 out_unlock:
 102         spin_unlock_irqrestore(&iommu_lock, flags);
 103         if (ret) {
 104                 pr_debug("PAACE configuration failed for liodn %d\n",
 105                          liodn);
 106         }
 107         return ret;
 108 }
 109
 110 static void remove_device_ref(struct device_domain_info *info)
 111 {
 112         unsigned long flags;
 113
 114         list_del(&info->link);
 115         spin_lock_irqsave(&iommu_lock, flags);
 116         pamu_disable_liodn(info->liodn);
 117         spin_unlock_irqrestore(&iommu_lock, flags);
 118         spin_lock_irqsave(&device_domain_lock, flags);
 119         dev_iommu_priv_set(info->dev, NULL);
 120         kmem_cache_free(iommu_devinfo_cache, info);
 121         spin_unlock_irqrestore(&device_domain_lock, flags);
 122 }
 123
 124 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
 125 {
 126         struct device_domain_info *info, *tmp;
 127         unsigned long flags;
 128
 129         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 130         /* Remove the device from the domain device list */
 131         list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
 132                 if (!dev || (info->dev == dev))
 133                         remove_device_ref(info);
 134         }
 135         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 136 }
 137
 138 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
 139 {
 140         struct device_domain_info *info, *old_domain_info;
 141         unsigned long flags;
 142
 143         spin_lock_irqsave(&device_domain_lock, flags);
 144         /*
 145          * Check here if the device is already attached to domain or not.
 146          * If the device is already attached to a domain detach it.
 147          */
 148         old_domain_info = dev_iommu_priv_get(dev);
 149         if (old_domain_info && old_domain_info->domain != dma_domain) {
 150                 spin_unlock_irqrestore(&device_domain_lock, flags);
 151                 detach_device(dev, old_domain_info->domain);
 152                 spin_lock_irqsave(&device_domain_lock, flags);
 153         }
 154
 155         info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
 156
 157         info->dev = dev;
 158         info->liodn = liodn;
 159         info->domain = dma_domain;
 160
 161         list_add(&info->link, &dma_domain->devices);
 162         /*
 163          * In case of devices with multiple LIODNs just store
 164          * the info for the first LIODN as all
 165          * LIODNs share the same domain
 166          */
 167         if (!dev_iommu_priv_get(dev))
 168                 dev_iommu_priv_set(dev, info);
 169         spin_unlock_irqrestore(&device_domain_lock, flags);
 170 }
 171
 172 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
 173                                          dma_addr_t iova)
 174 {
 175         if (iova < domain->geometry.aperture_start ||
 176             iova > domain->geometry.aperture_end)
 177                 return 0;
 178         return iova;
 179 }
 180
 181 static bool fsl_pamu_capable(struct device *dev, enum iommu_cap cap)
 182 {
 183         return cap == IOMMU_CAP_CACHE_COHERENCY;
 184 }
 185
 186 static void fsl_pamu_domain_free(struct iommu_domain *domain)
 187 {
 188         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 189
 190         /* remove all the devices from the device list */
 191         detach_device(NULL, dma_domain);
 192         kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 193 }
 194
 195 static struct iommu_domain *fsl_pamu_domain_alloc(unsigned type)
 196 {
 197         struct fsl_dma_domain *dma_domain;
 198
 199         /*
 200          * FIXME: This isn't creating an unmanaged domain since the
 201          * default_domain_ops do not have any map/unmap function it doesn't meet
 202          * the requirements for __IOMMU_DOMAIN_PAGING. The only purpose seems to
 203          * allow drivers/soc/fsl/qbman/qman_portal.c to do
 204          * fsl_pamu_configure_l1_stash()
 205          */
 206         if (type != IOMMU_DOMAIN_UNMANAGED)
 207                 return NULL;
 208
 209         dma_domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
 210         if (!dma_domain)
 211                 return NULL;
 212
 213         dma_domain->stash_id = ~(u32)0;
 214         INIT_LIST_HEAD(&dma_domain->devices);
 215         spin_lock_init(&dma_domain->domain_lock);
 216
 217         /* default geometry 64 GB i.e. maximum system address */
 218         dma_domain->iommu_domain. geometry.aperture_start = 0;
 219         dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
 220         dma_domain->iommu_domain.geometry.force_aperture = true;
 221
 222         return &dma_domain->iommu_domain;
 223 }
 224
 225 /* Update stash destination for all LIODNs associated with the domain */
 226 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
 227 {
 228         struct device_domain_info *info;
 229         int ret = 0;
 230
 231         list_for_each_entry(info, &dma_domain->devices, link) {
 232                 ret = update_liodn_stash(info->liodn, dma_domain, val);
 233                 if (ret)
 234                         break;
 235         }
 236
 237         return ret;
 238 }
 239
 240 static int fsl_pamu_attach_device(struct iommu_domain *domain,
 241                                   struct device *dev)
 242 {
 243         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 244         unsigned long flags;
 245         int len, ret = 0, i;
 246         const u32 *liodn;
 247         struct pci_dev *pdev = NULL;
 248         struct pci_controller *pci_ctl;
 249
 250         /*
 251          * Use LIODN of the PCI controller while attaching a
 252          * PCI device.
 253          */
 254         if (dev_is_pci(dev)) {
 255                 pdev = to_pci_dev(dev);
 256                 pci_ctl = pci_bus_to_host(pdev->bus);
 257                 /*
 258                  * make dev point to pci controller device
 259                  * so we can get the LIODN programmed by
 260                  * u-boot.
 261                  */
 262                 dev = pci_ctl->parent;
 263         }
 264
 265         liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
 266         if (!liodn) {
 267                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 268                 return -ENODEV;
 269         }
 270
 271         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 272         for (i = 0; i < len / sizeof(u32); i++) {
 273                 /* Ensure that LIODN value is valid */
 274                 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
 275                         pr_debug("Invalid liodn %d, attach device failed for %pOF\n",
 276                                  liodn[i], dev->of_node);
 277                         ret = -ENODEV;
 278                         break;
 279                 }
 280
 281                 attach_device(dma_domain, liodn[i], dev);
 282                 ret = pamu_set_liodn(dma_domain, dev, liodn[i]);
 283                 if (ret)
 284                         break;
 285                 ret = pamu_enable_liodn(liodn[i]);
 286                 if (ret)
 287                         break;
 288         }
 289         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 290         return ret;
 291 }
 292
 293 /*
 294  * FIXME: fsl/pamu is completely broken in terms of how it works with the iommu
 295  * API. Immediately after probe the HW is left in an IDENTITY translation and
 296  * the driver provides a non-working UNMANAGED domain that it can switch over
 297  * to. However it cannot switch back to an IDENTITY translation, instead it
 298  * switches to what looks like BLOCKING.
 299  */
 300 static int fsl_pamu_platform_attach(struct iommu_domain *platform_domain,
 301                                     struct device *dev)
 302 {
 303         struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
 304         struct fsl_dma_domain *dma_domain;
 305         const u32 *prop;
 306         int len;
 307         struct pci_dev *pdev = NULL;
 308         struct pci_controller *pci_ctl;
 309
 310         /*
 311          * Hack to keep things working as they always have, only leaving an
 312          * UNMANAGED domain makes it BLOCKING.
 313          */
 314         if (domain == platform_domain || !domain ||
 315             domain->type != IOMMU_DOMAIN_UNMANAGED)
 316                 return 0;
 317
 318         dma_domain = to_fsl_dma_domain(domain);
 319
 320         /*
 321          * Use LIODN of the PCI controller while detaching a
 322          * PCI device.
 323          */
 324         if (dev_is_pci(dev)) {
 325                 pdev = to_pci_dev(dev);
 326                 pci_ctl = pci_bus_to_host(pdev->bus);
 327                 /*
 328                  * make dev point to pci controller device
 329                  * so we can get the LIODN programmed by
 330                  * u-boot.
 331                  */
 332                 dev = pci_ctl->parent;
 333         }
 334
 335         prop = of_get_property(dev->of_node, "fsl,liodn", &len);
 336         if (prop)
 337                 detach_device(dev, dma_domain);
 338         else
 339                 pr_debug("missing fsl,liodn property at %pOF\n", dev->of_node);
 340         return 0;
 341 }
 342
 343 static struct iommu_domain_ops fsl_pamu_platform_ops = {
 344         .attach_dev = fsl_pamu_platform_attach,
 345 };
 346
 347 static struct iommu_domain fsl_pamu_platform_domain = {
 348         .type = IOMMU_DOMAIN_PLATFORM,
 349         .ops = &fsl_pamu_platform_ops,
 350 };
 351
 352 /* Set the domain stash attribute */
 353 int fsl_pamu_configure_l1_stash(struct iommu_domain *domain, u32 cpu)
 354 {
 355         struct fsl_dma_domain *dma_domain = to_fsl_dma_domain(domain);
 356         unsigned long flags;
 357         int ret;
 358
 359         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 360         dma_domain->stash_id = get_stash_id(PAMU_ATTR_CACHE_L1, cpu);
 361         if (dma_domain->stash_id == ~(u32)0) {
 362                 pr_debug("Invalid stash attributes\n");
 363                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 364                 return -EINVAL;
 365         }
 366         ret = update_domain_stash(dma_domain, dma_domain->stash_id);
 367         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 368
 369         return ret;
 370 }
 371
 372 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
 373 {
 374         u32 version;
 375
 376         /* Check the PCI controller version number by readding BRR1 register */
 377         version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
 378         version &= PCI_FSL_BRR1_VER;
 379         /* If PCI controller version is >= 0x204 we can partition endpoints */
 380         return version >= 0x204;
 381 }
 382
 383 static struct iommu_group *fsl_pamu_device_group(struct device *dev)
 384 {
 385         struct iommu_group *group;
 386         struct pci_dev *pdev;
 387
 388         /*
 389          * For platform devices we allocate a separate group for each of the
 390          * devices.
 391          */
 392         if (!dev_is_pci(dev))
 393                 return generic_device_group(dev);
 394
 395         /*
 396          * We can partition PCIe devices so assign device group to the device
 397          */
 398         pdev = to_pci_dev(dev);
 399         if (check_pci_ctl_endpt_part(pci_bus_to_host(pdev->bus)))
 400                 return pci_device_group(&pdev->dev);
 401
 402         /*
 403          * All devices connected to the controller will share the same device
 404          * group.
 405          *
 406          * Due to ordering between fsl_pamu_init() and fsl_pci_init() it is
 407          * guaranteed that the pci_ctl->parent platform_device will have the
 408          * iommu driver bound and will already have a group set. So we just
 409          * re-use this group as the group for every device in the hose.
 410          */
 411         group = iommu_group_get(pci_bus_to_host(pdev->bus)->parent);
 412         if (WARN_ON(!group))
 413                 return ERR_PTR(-EINVAL);
 414         return group;
 415 }
 416
 417 static struct iommu_device *fsl_pamu_probe_device(struct device *dev)
 418 {
 419         /*
 420          * uboot must fill the fsl,liodn for platform devices to be supported by
 421          * the iommu.
 422          */
 423         if (!dev_is_pci(dev) &&
 424             !of_property_present(dev->of_node, "fsl,liodn"))
 425                 return ERR_PTR(-ENODEV);
 426
 427         return &pamu_iommu;
 428 }
 429
 430 static const struct iommu_ops fsl_pamu_ops = {
 431         .default_domain = &fsl_pamu_platform_domain,
 432         .capable        = fsl_pamu_capable,
 433         .domain_alloc   = fsl_pamu_domain_alloc,
 434         .probe_device   = fsl_pamu_probe_device,
 435         .device_group   = fsl_pamu_device_group,
 436         .default_domain_ops = &(const struct iommu_domain_ops) {
 437                 .attach_dev     = fsl_pamu_attach_device,
 438                 .iova_to_phys   = fsl_pamu_iova_to_phys,
 439                 .free           = fsl_pamu_domain_free,
 440         }
 441 };
 442
 443 int __init pamu_domain_init(void)
 444 {
 445         int ret = 0;
 446
 447         ret = iommu_init_mempool();
 448         if (ret)
 449                 return ret;
 450
 451         ret = iommu_device_sysfs_add(&pamu_iommu, NULL, NULL, "iommu0");
 452         if (ret)
 453                 return ret;
 454
 455         ret = iommu_device_register(&pamu_iommu, &fsl_pamu_ops, NULL);
 456         if (ret) {
 457                 iommu_device_sysfs_remove(&pamu_iommu);
 458                 pr_err("Can't register iommu device\n");
 459         }
 460
 461         return ret;
 462 }