drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c

   1 /*
   2  * Copyright (C) 2016 Etnaviv Project
   3   *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 as published by
   6  * the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program.  If not, see <http://www.gnu.org/licenses/>.
  15  */
  16
  17 #include <linux/iommu.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/sizes.h>
  20 #include <linux/slab.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/bitops.h>
  23
  24 #include "etnaviv_gpu.h"
  25 #include "etnaviv_mmu.h"
  26 #include "etnaviv_iommu.h"
  27 #include "state.xml.h"
  28 #include "state_hi.xml.h"
  29
  30 #define MMUv2_PTE_PRESENT               BIT(0)
  31 #define MMUv2_PTE_EXCEPTION             BIT(1)
  32 #define MMUv2_PTE_WRITEABLE             BIT(2)
  33
  34 #define MMUv2_MTLB_MASK                 0xffc00000
  35 #define MMUv2_MTLB_SHIFT                22
  36 #define MMUv2_STLB_MASK                 0x003ff000
  37 #define MMUv2_STLB_SHIFT                12
  38
  39 #define MMUv2_MAX_STLB_ENTRIES          1024
  40
  41 struct etnaviv_iommuv2_domain {
  42         struct iommu_domain domain;
  43         struct device *dev;
  44         void *bad_page_cpu;
  45         dma_addr_t bad_page_dma;
  46         /* M(aster) TLB aka first level pagetable */
  47         u32 *mtlb_cpu;
  48         dma_addr_t mtlb_dma;
  49         /* S(lave) TLB aka second level pagetable */
  50         u32 *stlb_cpu[1024];
  51         dma_addr_t stlb_dma[1024];
  52 };
  53
  54 static struct etnaviv_iommuv2_domain *to_etnaviv_domain(struct iommu_domain *domain)
  55 {
  56         return container_of(domain, struct etnaviv_iommuv2_domain, domain);
  57 }
  58
  59 static int etnaviv_iommuv2_map(struct iommu_domain *domain, unsigned long iova,
  60            phys_addr_t paddr, size_t size, int prot)
  61 {
  62         struct etnaviv_iommuv2_domain *etnaviv_domain =
  63                         to_etnaviv_domain(domain);
  64         int mtlb_entry, stlb_entry;
  65         u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
  66
  67         if (size != SZ_4K)
  68                 return -EINVAL;
  69
  70         if (prot & IOMMU_WRITE)
  71                 entry |= MMUv2_PTE_WRITEABLE;
  72
  73         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
  74         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
  75
  76         etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
  77
  78         return 0;
  79 }
  80
  81 static size_t etnaviv_iommuv2_unmap(struct iommu_domain *domain,
  82         unsigned long iova, size_t size)
  83 {
  84         struct etnaviv_iommuv2_domain *etnaviv_domain =
  85                         to_etnaviv_domain(domain);
  86         int mtlb_entry, stlb_entry;
  87
  88         if (size != SZ_4K)
  89                 return -EINVAL;
  90
  91         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
  92         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
  93
  94         etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
  95
  96         return SZ_4K;
  97 }
  98
  99 static phys_addr_t etnaviv_iommuv2_iova_to_phys(struct iommu_domain *domain,
 100         dma_addr_t iova)
 101 {
 102         struct etnaviv_iommuv2_domain *etnaviv_domain =
 103                         to_etnaviv_domain(domain);
 104         int mtlb_entry, stlb_entry;
 105
 106         mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
 107         stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
 108
 109         return etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] & ~(SZ_4K - 1);
 110 }
 111
 112 static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
 113 {
 114         u32 *p;
 115         int ret, i, j;
 116
 117         /* allocate scratch page */
 118         etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev,
 119                                                   SZ_4K,
 120                                                   &etnaviv_domain->bad_page_dma,
 121                                                   GFP_KERNEL);
 122         if (!etnaviv_domain->bad_page_cpu) {
 123                 ret = -ENOMEM;
 124                 goto fail_mem;
 125         }
 126         p = etnaviv_domain->bad_page_cpu;
 127         for (i = 0; i < SZ_4K / 4; i++)
 128                 *p++ = 0xdead55aa;
 129
 130         etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->dev,
 131                                                   SZ_4K,
 132                                                   &etnaviv_domain->mtlb_dma,
 133                                                   GFP_KERNEL);
 134         if (!etnaviv_domain->mtlb_cpu) {
 135                 ret = -ENOMEM;
 136                 goto fail_mem;
 137         }
 138
 139         /* pre-populate STLB pages (may want to switch to on-demand later) */
 140         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
 141                 etnaviv_domain->stlb_cpu[i] =
 142                                 dma_alloc_coherent(etnaviv_domain->dev,
 143                                                    SZ_4K,
 144                                                    &etnaviv_domain->stlb_dma[i],
 145                                                    GFP_KERNEL);
 146                 if (!etnaviv_domain->stlb_cpu[i]) {
 147                         ret = -ENOMEM;
 148                         goto fail_mem;
 149                 }
 150                 p = etnaviv_domain->stlb_cpu[i];
 151                 for (j = 0; j < SZ_4K / 4; j++)
 152                         *p++ = MMUv2_PTE_EXCEPTION;
 153
 154                 etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
 155                                               MMUv2_PTE_PRESENT;
 156         }
 157
 158         return 0;
 159
 160 fail_mem:
 161         if (etnaviv_domain->bad_page_cpu)
 162                 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
 163                                   etnaviv_domain->bad_page_cpu,
 164                                   etnaviv_domain->bad_page_dma);
 165
 166         if (etnaviv_domain->mtlb_cpu)
 167                 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
 168                                   etnaviv_domain->mtlb_cpu,
 169                                   etnaviv_domain->mtlb_dma);
 170
 171         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
 172                 if (etnaviv_domain->stlb_cpu[i])
 173                         dma_free_coherent(etnaviv_domain->dev, SZ_4K,
 174                                           etnaviv_domain->stlb_cpu[i],
 175                                           etnaviv_domain->stlb_dma[i]);
 176         }
 177
 178         return ret;
 179 }
 180
 181 static void etnaviv_iommuv2_domain_free(struct iommu_domain *domain)
 182 {
 183         struct etnaviv_iommuv2_domain *etnaviv_domain =
 184                         to_etnaviv_domain(domain);
 185         int i;
 186
 187         dma_free_coherent(etnaviv_domain->dev, SZ_4K,
 188                           etnaviv_domain->bad_page_cpu,
 189                           etnaviv_domain->bad_page_dma);
 190
 191         dma_free_coherent(etnaviv_domain->dev, SZ_4K,
 192                           etnaviv_domain->mtlb_cpu,
 193                           etnaviv_domain->mtlb_dma);
 194
 195         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
 196                 if (etnaviv_domain->stlb_cpu[i])
 197                         dma_free_coherent(etnaviv_domain->dev, SZ_4K,
 198                                           etnaviv_domain->stlb_cpu[i],
 199                                           etnaviv_domain->stlb_dma[i]);
 200         }
 201
 202         vfree(etnaviv_domain);
 203 }
 204
 205 static size_t etnaviv_iommuv2_dump_size(struct iommu_domain *domain)
 206 {
 207         struct etnaviv_iommuv2_domain *etnaviv_domain =
 208                         to_etnaviv_domain(domain);
 209         size_t dump_size = SZ_4K;
 210         int i;
 211
 212         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
 213                 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
 214                         dump_size += SZ_4K;
 215
 216         return dump_size;
 217 }
 218
 219 static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf)
 220 {
 221         struct etnaviv_iommuv2_domain *etnaviv_domain =
 222                         to_etnaviv_domain(domain);
 223         int i;
 224
 225         memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K);
 226         buf += SZ_4K;
 227         for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K)
 228                 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
 229                         memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K);
 230 }
 231
 232 static struct etnaviv_iommu_ops etnaviv_iommu_ops = {
 233         .ops = {
 234                 .domain_free = etnaviv_iommuv2_domain_free,
 235                 .map = etnaviv_iommuv2_map,
 236                 .unmap = etnaviv_iommuv2_unmap,
 237                 .iova_to_phys = etnaviv_iommuv2_iova_to_phys,
 238                 .pgsize_bitmap = SZ_4K,
 239         },
 240         .dump_size = etnaviv_iommuv2_dump_size,
 241         .dump = etnaviv_iommuv2_dump,
 242 };
 243
 244 void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
 245 {
 246         struct etnaviv_iommuv2_domain *etnaviv_domain =
 247                         to_etnaviv_domain(gpu->mmu->domain);
 248         u16 prefetch;
 249
 250         /* If the MMU is already enabled the state is still there. */
 251         if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
 252                 return;
 253
 254         prefetch = etnaviv_buffer_config_mmuv2(gpu,
 255                                 (u32)etnaviv_domain->mtlb_dma,
 256                                 (u32)etnaviv_domain->bad_page_dma);
 257         etnaviv_gpu_start_fe(gpu, gpu->buffer->paddr, prefetch);
 258         etnaviv_gpu_wait_idle(gpu, 100);
 259
 260         gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
 261 }
 262 struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu)
 263 {
 264         struct etnaviv_iommuv2_domain *etnaviv_domain;
 265         int ret;
 266
 267         etnaviv_domain = vzalloc(sizeof(*etnaviv_domain));
 268         if (!etnaviv_domain)
 269                 return NULL;
 270
 271         etnaviv_domain->dev = gpu->dev;
 272
 273         etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING;
 274         etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops;
 275         etnaviv_domain->domain.pgsize_bitmap = SZ_4K;
 276         etnaviv_domain->domain.geometry.aperture_start = 0;
 277         etnaviv_domain->domain.geometry.aperture_end = ~0UL & ~(SZ_4K - 1);
 278
 279         ret = etnaviv_iommuv2_init(etnaviv_domain);
 280         if (ret)
 281                 goto out_free;
 282
 283         return &etnaviv_domain->domain;
 284
 285 out_free:
 286         vfree(etnaviv_domain);
 287         return NULL;
 288 }