drivers/iommu/tegra-gart.c

   1 /*
   2  * IOMMU API for Graphics Address Relocation Table on Tegra20
   3  *
   4  * Copyright (c) 2010-2012, NVIDIA CORPORATION.  All rights reserved.
   5  *
   6  * Author: Hiroshi DOYU <hdoyu@nvidia.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms and conditions of the GNU General Public License,
  10  * version 2, as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope it will be useful, but WITHOUT
  13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15  * more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along with
  18  * this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  20  */
  21
  22 #define dev_fmt(fmt)    "gart: " fmt
  23
  24 #include <linux/io.h>
  25 #include <linux/iommu.h>
  26 #include <linux/moduleparam.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/slab.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/vmalloc.h>
  31
  32 #include <soc/tegra/mc.h>
  33
  34 #define GART_REG_BASE           0x24
  35 #define GART_CONFIG             (0x24 - GART_REG_BASE)
  36 #define GART_ENTRY_ADDR         (0x28 - GART_REG_BASE)
  37 #define GART_ENTRY_DATA         (0x2c - GART_REG_BASE)
  38
  39 #define GART_ENTRY_PHYS_ADDR_VALID      BIT(31)
  40
  41 #define GART_PAGE_SHIFT         12
  42 #define GART_PAGE_SIZE          (1 << GART_PAGE_SHIFT)
  43 #define GART_PAGE_MASK          GENMASK(30, GART_PAGE_SHIFT)
  44
  45 /* bitmap of the page sizes currently supported */
  46 #define GART_IOMMU_PGSIZES      (GART_PAGE_SIZE)
  47
  48 struct gart_device {
  49         void __iomem            *regs;
  50         u32                     *savedata;
  51         unsigned long           iovmm_base;     /* offset to vmm_area start */
  52         unsigned long           iovmm_end;      /* offset to vmm_area end */
  53         spinlock_t              pte_lock;       /* for pagetable */
  54         spinlock_t              dom_lock;       /* for active domain */
  55         unsigned int            active_devices; /* number of active devices */
  56         struct iommu_domain     *active_domain; /* current active domain */
  57         struct iommu_device     iommu;          /* IOMMU Core handle */
  58         struct device           *dev;
  59 };
  60
  61 static struct gart_device *gart_handle; /* unique for a system */
  62
  63 static bool gart_debug;
  64
  65 /*
  66  * Any interaction between any block on PPSB and a block on APB or AHB
  67  * must have these read-back to ensure the APB/AHB bus transaction is
  68  * complete before initiating activity on the PPSB block.
  69  */
  70 #define FLUSH_GART_REGS(gart)   readl_relaxed((gart)->regs + GART_CONFIG)
  71
  72 #define for_each_gart_pte(gart, iova)                                   \
  73         for (iova = gart->iovmm_base;                                   \
  74              iova < gart->iovmm_end;                                    \
  75              iova += GART_PAGE_SIZE)
  76
  77 static inline void gart_set_pte(struct gart_device *gart,
  78                                 unsigned long iova, unsigned long pte)
  79 {
  80         writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
  81         writel_relaxed(pte, gart->regs + GART_ENTRY_DATA);
  82 }
  83
  84 static inline unsigned long gart_read_pte(struct gart_device *gart,
  85                                           unsigned long iova)
  86 {
  87         unsigned long pte;
  88
  89         writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
  90         pte = readl_relaxed(gart->regs + GART_ENTRY_DATA);
  91
  92         return pte;
  93 }
  94
  95 static void do_gart_setup(struct gart_device *gart, const u32 *data)
  96 {
  97         unsigned long iova;
  98
  99         for_each_gart_pte(gart, iova)
 100                 gart_set_pte(gart, iova, data ? *(data++) : 0);
 101
 102         writel_relaxed(1, gart->regs + GART_CONFIG);
 103         FLUSH_GART_REGS(gart);
 104 }
 105
 106 static inline bool gart_iova_range_invalid(struct gart_device *gart,
 107                                            unsigned long iova, size_t bytes)
 108 {
 109         return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE ||
 110                         iova + bytes > gart->iovmm_end);
 111 }
 112
 113 static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova)
 114 {
 115         return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID);
 116 }
 117
 118 static int gart_iommu_attach_dev(struct iommu_domain *domain,
 119                                  struct device *dev)
 120 {
 121         struct gart_device *gart = gart_handle;
 122         int ret = 0;
 123
 124         spin_lock(&gart->dom_lock);
 125
 126         if (gart->active_domain && gart->active_domain != domain) {
 127                 ret = -EBUSY;
 128         } else if (dev->archdata.iommu != domain) {
 129                 dev->archdata.iommu = domain;
 130                 gart->active_domain = domain;
 131                 gart->active_devices++;
 132         }
 133
 134         spin_unlock(&gart->dom_lock);
 135
 136         return ret;
 137 }
 138
 139 static void gart_iommu_detach_dev(struct iommu_domain *domain,
 140                                   struct device *dev)
 141 {
 142         struct gart_device *gart = gart_handle;
 143
 144         spin_lock(&gart->dom_lock);
 145
 146         if (dev->archdata.iommu == domain) {
 147                 dev->archdata.iommu = NULL;
 148
 149                 if (--gart->active_devices == 0)
 150                         gart->active_domain = NULL;
 151         }
 152
 153         spin_unlock(&gart->dom_lock);
 154 }
 155
 156 static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
 157 {
 158         struct iommu_domain *domain;
 159
 160         if (type != IOMMU_DOMAIN_UNMANAGED)
 161                 return NULL;
 162
 163         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
 164         if (domain) {
 165                 domain->geometry.aperture_start = gart_handle->iovmm_base;
 166                 domain->geometry.aperture_end = gart_handle->iovmm_end - 1;
 167                 domain->geometry.force_aperture = true;
 168         }
 169
 170         return domain;
 171 }
 172
 173 static void gart_iommu_domain_free(struct iommu_domain *domain)
 174 {
 175         WARN_ON(gart_handle->active_domain == domain);
 176         kfree(domain);
 177 }
 178
 179 static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova,
 180                                    unsigned long pa)
 181 {
 182         if (unlikely(gart_debug && gart_pte_valid(gart, iova))) {
 183                 dev_err(gart->dev, "Page entry is in-use\n");
 184                 return -EINVAL;
 185         }
 186
 187         gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa);
 188
 189         return 0;
 190 }
 191
 192 static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
 193                           phys_addr_t pa, size_t bytes, int prot)
 194 {
 195         struct gart_device *gart = gart_handle;
 196         int ret;
 197
 198         if (gart_iova_range_invalid(gart, iova, bytes))
 199                 return -EINVAL;
 200
 201         spin_lock(&gart->pte_lock);
 202         ret = __gart_iommu_map(gart, iova, (unsigned long)pa);
 203         spin_unlock(&gart->pte_lock);
 204
 205         return ret;
 206 }
 207
 208 static inline int __gart_iommu_unmap(struct gart_device *gart,
 209                                      unsigned long iova)
 210 {
 211         if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) {
 212                 dev_err(gart->dev, "Page entry is invalid\n");
 213                 return -EINVAL;
 214         }
 215
 216         gart_set_pte(gart, iova, 0);
 217
 218         return 0;
 219 }
 220
 221 static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
 222                                size_t bytes)
 223 {
 224         struct gart_device *gart = gart_handle;
 225         int err;
 226
 227         if (gart_iova_range_invalid(gart, iova, bytes))
 228                 return 0;
 229
 230         spin_lock(&gart->pte_lock);
 231         err = __gart_iommu_unmap(gart, iova);
 232         spin_unlock(&gart->pte_lock);
 233
 234         return err ? 0 : bytes;
 235 }
 236
 237 static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
 238                                            dma_addr_t iova)
 239 {
 240         struct gart_device *gart = gart_handle;
 241         unsigned long pte;
 242
 243         if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE))
 244                 return -EINVAL;
 245
 246         spin_lock(&gart->pte_lock);
 247         pte = gart_read_pte(gart, iova);
 248         spin_unlock(&gart->pte_lock);
 249
 250         return pte & GART_PAGE_MASK;
 251 }
 252
 253 static bool gart_iommu_capable(enum iommu_cap cap)
 254 {
 255         return false;
 256 }
 257
 258 static int gart_iommu_add_device(struct device *dev)
 259 {
 260         struct iommu_group *group;
 261
 262         if (!dev->iommu_fwspec)
 263                 return -ENODEV;
 264
 265         group = iommu_group_get_for_dev(dev);
 266         if (IS_ERR(group))
 267                 return PTR_ERR(group);
 268
 269         iommu_group_put(group);
 270
 271         iommu_device_link(&gart_handle->iommu, dev);
 272
 273         return 0;
 274 }
 275
 276 static void gart_iommu_remove_device(struct device *dev)
 277 {
 278         iommu_group_remove_device(dev);
 279         iommu_device_unlink(&gart_handle->iommu, dev);
 280 }
 281
 282 static int gart_iommu_of_xlate(struct device *dev,
 283                                struct of_phandle_args *args)
 284 {
 285         return 0;
 286 }
 287
 288 static void gart_iommu_sync(struct iommu_domain *domain)
 289 {
 290         FLUSH_GART_REGS(gart_handle);
 291 }
 292
 293 static const struct iommu_ops gart_iommu_ops = {
 294         .capable        = gart_iommu_capable,
 295         .domain_alloc   = gart_iommu_domain_alloc,
 296         .domain_free    = gart_iommu_domain_free,
 297         .attach_dev     = gart_iommu_attach_dev,
 298         .detach_dev     = gart_iommu_detach_dev,
 299         .add_device     = gart_iommu_add_device,
 300         .remove_device  = gart_iommu_remove_device,
 301         .device_group   = generic_device_group,
 302         .map            = gart_iommu_map,
 303         .unmap          = gart_iommu_unmap,
 304         .iova_to_phys   = gart_iommu_iova_to_phys,
 305         .pgsize_bitmap  = GART_IOMMU_PGSIZES,
 306         .of_xlate       = gart_iommu_of_xlate,
 307         .iotlb_sync_map = gart_iommu_sync,
 308         .iotlb_sync     = gart_iommu_sync,
 309 };
 310
 311 int tegra_gart_suspend(struct gart_device *gart)
 312 {
 313         u32 *data = gart->savedata;
 314         unsigned long iova;
 315
 316         /*
 317          * All GART users shall be suspended at this point. Disable
 318          * address translation to trap all GART accesses as invalid
 319          * memory accesses.
 320          */
 321         writel_relaxed(0, gart->regs + GART_CONFIG);
 322         FLUSH_GART_REGS(gart);
 323
 324         for_each_gart_pte(gart, iova)
 325                 *(data++) = gart_read_pte(gart, iova);
 326
 327         return 0;
 328 }
 329
 330 int tegra_gart_resume(struct gart_device *gart)
 331 {
 332         do_gart_setup(gart, gart->savedata);
 333
 334         return 0;
 335 }
 336
 337 struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc)
 338 {
 339         struct gart_device *gart;
 340         struct resource *res;
 341         int err;
 342
 343         BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
 344
 345         /* the GART memory aperture is required */
 346         res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1);
 347         if (!res) {
 348                 dev_err(dev, "Memory aperture resource unavailable\n");
 349                 return ERR_PTR(-ENXIO);
 350         }
 351
 352         gart = kzalloc(sizeof(*gart), GFP_KERNEL);
 353         if (!gart)
 354                 return ERR_PTR(-ENOMEM);
 355
 356         gart_handle = gart;
 357
 358         gart->dev = dev;
 359         gart->regs = mc->regs + GART_REG_BASE;
 360         gart->iovmm_base = res->start;
 361         gart->iovmm_end = res->end + 1;
 362         spin_lock_init(&gart->pte_lock);
 363         spin_lock_init(&gart->dom_lock);
 364
 365         do_gart_setup(gart, NULL);
 366
 367         err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart");
 368         if (err)
 369                 goto free_gart;
 370
 371         iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
 372         iommu_device_set_fwnode(&gart->iommu, dev->fwnode);
 373
 374         err = iommu_device_register(&gart->iommu);
 375         if (err)
 376                 goto remove_sysfs;
 377
 378         gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE *
 379                                  sizeof(u32));
 380         if (!gart->savedata) {
 381                 err = -ENOMEM;
 382                 goto unregister_iommu;
 383         }
 384
 385         return gart;
 386
 387 unregister_iommu:
 388         iommu_device_unregister(&gart->iommu);
 389 remove_sysfs:
 390         iommu_device_sysfs_remove(&gart->iommu);
 391 free_gart:
 392         kfree(gart);
 393
 394         return ERR_PTR(err);
 395 }
 396
 397 module_param(gart_debug, bool, 0644);
 398 MODULE_PARM_DESC(gart_debug, "Enable GART debugging");