drivers/gpu/drm/etnaviv/etnaviv_mmu.c

   1 /*
   2  * Copyright (C) 2015 Etnaviv Project
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 as published by
   6  * the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16
  17 #include "common.xml.h"
  18 #include "etnaviv_drv.h"
  19 #include "etnaviv_gem.h"
  20 #include "etnaviv_gpu.h"
  21 #include "etnaviv_iommu.h"
  22 #include "etnaviv_mmu.h"
  23
  24 static int etnaviv_fault_handler(struct iommu_domain *iommu, struct device *dev,
  25                 unsigned long iova, int flags, void *arg)
  26 {
  27         DBG("*** fault: iova=%08lx, flags=%d", iova, flags);
  28         return 0;
  29 }
  30
  31 int etnaviv_iommu_map(struct etnaviv_iommu *iommu, u32 iova,
  32                 struct sg_table *sgt, unsigned len, int prot)
  33 {
  34         struct iommu_domain *domain = iommu->domain;
  35         struct scatterlist *sg;
  36         unsigned int da = iova;
  37         unsigned int i, j;
  38         int ret;
  39
  40         if (!domain || !sgt)
  41                 return -EINVAL;
  42
  43         for_each_sg(sgt->sgl, sg, sgt->nents, i) {
  44                 u32 pa = sg_dma_address(sg) - sg->offset;
  45                 size_t bytes = sg_dma_len(sg) + sg->offset;
  46
  47                 VERB("map[%d]: %08x %08x(%zx)", i, iova, pa, bytes);
  48
  49                 ret = iommu_map(domain, da, pa, bytes, prot);
  50                 if (ret)
  51                         goto fail;
  52
  53                 da += bytes;
  54         }
  55
  56         return 0;
  57
  58 fail:
  59         da = iova;
  60
  61         for_each_sg(sgt->sgl, sg, i, j) {
  62                 size_t bytes = sg_dma_len(sg) + sg->offset;
  63
  64                 iommu_unmap(domain, da, bytes);
  65                 da += bytes;
  66         }
  67         return ret;
  68 }
  69
  70 int etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova,
  71                 struct sg_table *sgt, unsigned len)
  72 {
  73         struct iommu_domain *domain = iommu->domain;
  74         struct scatterlist *sg;
  75         unsigned int da = iova;
  76         int i;
  77
  78         for_each_sg(sgt->sgl, sg, sgt->nents, i) {
  79                 size_t bytes = sg_dma_len(sg) + sg->offset;
  80                 size_t unmapped;
  81
  82                 unmapped = iommu_unmap(domain, da, bytes);
  83                 if (unmapped < bytes)
  84                         return unmapped;
  85
  86                 VERB("unmap[%d]: %08x(%zx)", i, iova, bytes);
  87
  88                 BUG_ON(!PAGE_ALIGNED(bytes));
  89
  90                 da += bytes;
  91         }
  92
  93         return 0;
  94 }
  95
  96 static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu *mmu,
  97         struct etnaviv_vram_mapping *mapping)
  98 {
  99         struct etnaviv_gem_object *etnaviv_obj = mapping->object;
 100
 101         etnaviv_iommu_unmap(mmu, mapping->vram_node.start,
 102                             etnaviv_obj->sgt, etnaviv_obj->base.size);
 103         drm_mm_remove_node(&mapping->vram_node);
 104 }
 105
 106 static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu,
 107                                    struct drm_mm_node *node, size_t size)
 108 {
 109         struct etnaviv_vram_mapping *free = NULL;
 110         int ret;
 111
 112         lockdep_assert_held(&mmu->lock);
 113
 114         while (1) {
 115                 struct etnaviv_vram_mapping *m, *n;
 116                 struct drm_mm_scan scan;
 117                 struct list_head list;
 118                 bool found;
 119
 120                 /*
 121                  * XXX: The DRM_MM_SEARCH_BELOW is really a hack to trick
 122                  * drm_mm into giving out a low IOVA after address space
 123                  * rollover. This needs a proper fix.
 124                  */
 125                 ret = drm_mm_insert_node_in_range(&mmu->mm, node,
 126                         size, 0, mmu->last_iova, ~0UL,
 127                         mmu->last_iova ? DRM_MM_SEARCH_DEFAULT : DRM_MM_SEARCH_BELOW);
 128
 129                 if (ret != -ENOSPC)
 130                         break;
 131
 132                 /*
 133                  * If we did not search from the start of the MMU region,
 134                  * try again in case there are free slots.
 135                  */
 136                 if (mmu->last_iova) {
 137                         mmu->last_iova = 0;
 138                         mmu->need_flush = true;
 139                         continue;
 140                 }
 141
 142                 /* Try to retire some entries */
 143                 drm_mm_scan_init(&scan, &mmu->mm, size, 0, 0, 0);
 144
 145                 found = 0;
 146                 INIT_LIST_HEAD(&list);
 147                 list_for_each_entry(free, &mmu->mappings, mmu_node) {
 148                         /* If this vram node has not been used, skip this. */
 149                         if (!free->vram_node.mm)
 150                                 continue;
 151
 152                         /*
 153                          * If the iova is pinned, then it's in-use,
 154                          * so we must keep its mapping.
 155                          */
 156                         if (free->use)
 157                                 continue;
 158
 159                         list_add(&free->scan_node, &list);
 160                         if (drm_mm_scan_add_block(&scan, &free->vram_node)) {
 161                                 found = true;
 162                                 break;
 163                         }
 164                 }
 165
 166                 if (!found) {
 167                         /* Nothing found, clean up and fail */
 168                         list_for_each_entry_safe(m, n, &list, scan_node)
 169                                 BUG_ON(drm_mm_scan_remove_block(&scan, &m->vram_node));
 170                         break;
 171                 }
 172
 173                 /*
 174                  * drm_mm does not allow any other operations while
 175                  * scanning, so we have to remove all blocks first.
 176                  * If drm_mm_scan_remove_block() returns false, we
 177                  * can leave the block pinned.
 178                  */
 179                 list_for_each_entry_safe(m, n, &list, scan_node)
 180                         if (!drm_mm_scan_remove_block(&scan, &m->vram_node))
 181                                 list_del_init(&m->scan_node);
 182
 183                 /*
 184                  * Unmap the blocks which need to be reaped from the MMU.
 185                  * Clear the mmu pointer to prevent the mapping_get finding
 186                  * this mapping.
 187                  */
 188                 list_for_each_entry_safe(m, n, &list, scan_node) {
 189                         etnaviv_iommu_remove_mapping(mmu, m);
 190                         m->mmu = NULL;
 191                         list_del_init(&m->mmu_node);
 192                         list_del_init(&m->scan_node);
 193                 }
 194
 195                 /*
 196                  * We removed enough mappings so that the new allocation will
 197                  * succeed.  Ensure that the MMU will be flushed before the
 198                  * associated commit requesting this mapping, and retry the
 199                  * allocation one more time.
 200                  */
 201                 mmu->need_flush = true;
 202         }
 203
 204         return ret;
 205 }
 206
 207 int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu,
 208         struct etnaviv_gem_object *etnaviv_obj, u32 memory_base,
 209         struct etnaviv_vram_mapping *mapping)
 210 {
 211         struct sg_table *sgt = etnaviv_obj->sgt;
 212         struct drm_mm_node *node;
 213         int ret;
 214
 215         lockdep_assert_held(&etnaviv_obj->lock);
 216
 217         mutex_lock(&mmu->lock);
 218
 219         /* v1 MMU can optimize single entry (contiguous) scatterlists */
 220         if (mmu->version == ETNAVIV_IOMMU_V1 &&
 221             sgt->nents == 1 && !(etnaviv_obj->flags & ETNA_BO_FORCE_MMU)) {
 222                 u32 iova;
 223
 224                 iova = sg_dma_address(sgt->sgl) - memory_base;
 225                 if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) {
 226                         mapping->iova = iova;
 227                         list_add_tail(&mapping->mmu_node, &mmu->mappings);
 228                         mutex_unlock(&mmu->lock);
 229                         return 0;
 230                 }
 231         }
 232
 233         node = &mapping->vram_node;
 234
 235         ret = etnaviv_iommu_find_iova(mmu, node, etnaviv_obj->base.size);
 236         if (ret < 0) {
 237                 mutex_unlock(&mmu->lock);
 238                 return ret;
 239         }
 240
 241         mmu->last_iova = node->start + etnaviv_obj->base.size;
 242         mapping->iova = node->start;
 243         ret = etnaviv_iommu_map(mmu, node->start, sgt, etnaviv_obj->base.size,
 244                                 IOMMU_READ | IOMMU_WRITE);
 245
 246         if (ret < 0) {
 247                 drm_mm_remove_node(node);
 248                 mutex_unlock(&mmu->lock);
 249                 return ret;
 250         }
 251
 252         list_add_tail(&mapping->mmu_node, &mmu->mappings);
 253         mutex_unlock(&mmu->lock);
 254
 255         return ret;
 256 }
 257
 258 void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu,
 259         struct etnaviv_vram_mapping *mapping)
 260 {
 261         WARN_ON(mapping->use);
 262
 263         mutex_lock(&mmu->lock);
 264
 265         /* If the vram node is on the mm, unmap and remove the node */
 266         if (mapping->vram_node.mm == &mmu->mm)
 267                 etnaviv_iommu_remove_mapping(mmu, mapping);
 268
 269         list_del(&mapping->mmu_node);
 270         mutex_unlock(&mmu->lock);
 271 }
 272
 273 void etnaviv_iommu_destroy(struct etnaviv_iommu *mmu)
 274 {
 275         drm_mm_takedown(&mmu->mm);
 276         iommu_domain_free(mmu->domain);
 277         kfree(mmu);
 278 }
 279
 280 struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu)
 281 {
 282         enum etnaviv_iommu_version version;
 283         struct etnaviv_iommu *mmu;
 284
 285         mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
 286         if (!mmu)
 287                 return ERR_PTR(-ENOMEM);
 288
 289         if (!(gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION)) {
 290                 mmu->domain = etnaviv_iommuv1_domain_alloc(gpu);
 291                 version = ETNAVIV_IOMMU_V1;
 292         } else {
 293                 mmu->domain = etnaviv_iommuv2_domain_alloc(gpu);
 294                 version = ETNAVIV_IOMMU_V2;
 295         }
 296
 297         if (!mmu->domain) {
 298                 dev_err(gpu->dev, "Failed to allocate GPU IOMMU domain\n");
 299                 kfree(mmu);
 300                 return ERR_PTR(-ENOMEM);
 301         }
 302
 303         mmu->gpu = gpu;
 304         mmu->version = version;
 305         mutex_init(&mmu->lock);
 306         INIT_LIST_HEAD(&mmu->mappings);
 307
 308         drm_mm_init(&mmu->mm, mmu->domain->geometry.aperture_start,
 309                     mmu->domain->geometry.aperture_end -
 310                     mmu->domain->geometry.aperture_start + 1);
 311
 312         iommu_set_fault_handler(mmu->domain, etnaviv_fault_handler, gpu->dev);
 313
 314         return mmu;
 315 }
 316
 317 void etnaviv_iommu_restore(struct etnaviv_gpu *gpu)
 318 {
 319         if (gpu->mmu->version == ETNAVIV_IOMMU_V1)
 320                 etnaviv_iommuv1_restore(gpu);
 321         else
 322                 etnaviv_iommuv2_restore(gpu);
 323 }
 324
 325 u32 etnaviv_iommu_get_cmdbuf_va(struct etnaviv_gpu *gpu,
 326                                 struct etnaviv_cmdbuf *buf)
 327 {
 328         struct etnaviv_iommu *mmu = gpu->mmu;
 329
 330         if (mmu->version == ETNAVIV_IOMMU_V1) {
 331                 return buf->paddr - gpu->memory_base;
 332         } else {
 333                 int ret;
 334
 335                 if (buf->vram_node.allocated)
 336                         return (u32)buf->vram_node.start;
 337
 338                 mutex_lock(&mmu->lock);
 339                 ret = etnaviv_iommu_find_iova(mmu, &buf->vram_node,
 340                                               buf->size + SZ_64K);
 341                 if (ret < 0) {
 342                         mutex_unlock(&mmu->lock);
 343                         return 0;
 344                 }
 345                 ret = iommu_map(mmu->domain, buf->vram_node.start, buf->paddr,
 346                                 buf->size, IOMMU_READ);
 347                 if (ret < 0) {
 348                         drm_mm_remove_node(&buf->vram_node);
 349                         mutex_unlock(&mmu->lock);
 350                         return 0;
 351                 }
 352                 /*
 353                  * At least on GC3000 the FE MMU doesn't properly flush old TLB
 354                  * entries. Make sure to space the command buffers out in a way
 355                  * that the FE MMU prefetch won't load invalid entries.
 356                  */
 357                 mmu->last_iova = buf->vram_node.start + buf->size + SZ_64K;
 358                 gpu->mmu->need_flush = true;
 359                 mutex_unlock(&mmu->lock);
 360
 361                 return (u32)buf->vram_node.start;
 362         }
 363 }
 364
 365 void etnaviv_iommu_put_cmdbuf_va(struct etnaviv_gpu *gpu,
 366                                  struct etnaviv_cmdbuf *buf)
 367 {
 368         struct etnaviv_iommu *mmu = gpu->mmu;
 369
 370         if (mmu->version == ETNAVIV_IOMMU_V2 && buf->vram_node.allocated) {
 371                 mutex_lock(&mmu->lock);
 372                 iommu_unmap(mmu->domain, buf->vram_node.start, buf->size);
 373                 drm_mm_remove_node(&buf->vram_node);
 374                 mutex_unlock(&mmu->lock);
 375         }
 376 }
 377 size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu)
 378 {
 379         struct etnaviv_iommu_ops *ops;
 380
 381         ops = container_of(iommu->domain->ops, struct etnaviv_iommu_ops, ops);
 382
 383         return ops->dump_size(iommu->domain);
 384 }
 385
 386 void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf)
 387 {
 388         struct etnaviv_iommu_ops *ops;
 389
 390         ops = container_of(iommu->domain->ops, struct etnaviv_iommu_ops, ops);
 391
 392         ops->dump(iommu->domain, buf);
 393 }