drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

   1 /*
   2  * Copyright 2017 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "vmm.h"
  23
  24 #include <subdev/fb.h>
  25 #include <subdev/ltc.h>
  26 #include <subdev/timer.h>
  27
  28 #include <nvif/if900d.h>
  29 #include <nvif/unpack.h>
  30
  31 static inline void
  32 gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  33                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
  34 {
  35         u64 base = (addr >> 8) | map->type;
  36         u64 data = base;
  37
  38         if (map->ctag && !(map->next & (1ULL << 44))) {
  39                 while (ptes--) {
  40                         data = base | ((map->ctag >> 1) << 44);
  41                         if (!(map->ctag++ & 1))
  42                                 data |= BIT_ULL(60);
  43
  44                         VMM_WO064(pt, vmm, ptei++ * 8, data);
  45                         base += map->next;
  46                 }
  47         } else {
  48                 map->type += ptes * map->ctag;
  49
  50                 while (ptes--) {
  51                         VMM_WO064(pt, vmm, ptei++ * 8, data);
  52                         data += map->next;
  53                 }
  54         }
  55 }
  56
  57 void
  58 gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  59                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  60 {
  61         VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  62 }
  63
  64 void
  65 gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  66                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  67 {
  68         if (map->page->shift == PAGE_SHIFT) {
  69                 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
  70                 nvkm_kmap(pt->memory);
  71                 while (ptes--) {
  72                         const u64 data = (*map->dma++ >> 8) | map->type;
  73                         VMM_WO064(pt, vmm, ptei++ * 8, data);
  74                         map->type += map->ctag;
  75                 }
  76                 nvkm_done(pt->memory);
  77                 return;
  78         }
  79
  80         VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  81 }
  82
  83 void
  84 gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  85                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  86 {
  87         VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  88 }
  89
  90 void
  91 gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,
  92                     struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
  93 {
  94         VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
  95 }
  96
  97 const struct nvkm_vmm_desc_func
  98 gf100_vmm_pgt = {
  99         .unmap = gf100_vmm_pgt_unmap,
 100         .mem = gf100_vmm_pgt_mem,
 101         .dma = gf100_vmm_pgt_dma,
 102         .sgl = gf100_vmm_pgt_sgl,
 103 };
 104
 105 void
 106 gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
 107 {
 108         struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 109         struct nvkm_mmu_pt *pd = pgd->pt[0];
 110         struct nvkm_mmu_pt *pt;
 111         u64 data = 0;
 112
 113         if ((pt = pgt->pt[0])) {
 114                 switch (nvkm_memory_target(pt->memory)) {
 115                 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;
 116                 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;
 117                         data |= BIT_ULL(35); /* VOL */
 118                         break;
 119                 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;
 120                 default:
 121                         WARN_ON(1);
 122                         return;
 123                 }
 124                 data |= pt->addr >> 8;
 125         }
 126
 127         if ((pt = pgt->pt[1])) {
 128                 switch (nvkm_memory_target(pt->memory)) {
 129                 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;
 130                 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;
 131                         data |= BIT_ULL(34); /* VOL */
 132                         break;
 133                 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;
 134                 default:
 135                         WARN_ON(1);
 136                         return;
 137                 }
 138                 data |= pt->addr << 24;
 139         }
 140
 141         nvkm_kmap(pd->memory);
 142         VMM_WO064(pd, vmm, pdei * 8, data);
 143         nvkm_done(pd->memory);
 144 }
 145
 146 const struct nvkm_vmm_desc_func
 147 gf100_vmm_pgd = {
 148         .unmap = gf100_vmm_pgt_unmap,
 149         .pde = gf100_vmm_pgd_pde,
 150 };
 151
 152 static const struct nvkm_vmm_desc
 153 gf100_vmm_desc_17_12[] = {
 154         { SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
 155         { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
 156         {}
 157 };
 158
 159 static const struct nvkm_vmm_desc
 160 gf100_vmm_desc_17_17[] = {
 161         { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
 162         { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
 163         {}
 164 };
 165
 166 static const struct nvkm_vmm_desc
 167 gf100_vmm_desc_16_12[] = {
 168         { SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
 169         { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
 170         {}
 171 };
 172
 173 static const struct nvkm_vmm_desc
 174 gf100_vmm_desc_16_16[] = {
 175         { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
 176         { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
 177         {}
 178 };
 179
 180 void
 181 gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr)
 182 {
 183         struct nvkm_device *device = vmm->mmu->subdev.device;
 184         nvkm_wr32(device, 0x100cb8, addr);
 185 }
 186
 187 void
 188 gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type)
 189 {
 190         struct nvkm_subdev *subdev = &vmm->mmu->subdev;
 191         struct nvkm_device *device = subdev->device;
 192         struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
 193         u64 addr = 0;
 194
 195         mutex_lock(&subdev->mutex);
 196         /* Looks like maybe a "free flush slots" counter, the
 197          * faster you write to 0x100cbc to more it decreases.
 198          */
 199         nvkm_msec(device, 2000,
 200                 if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
 201                         break;
 202         );
 203
 204         if (!(type & 0x00000002) /* ALL_PDB. */) {
 205                 switch (nvkm_memory_target(pd->memory)) {
 206                 case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break;
 207                 case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break;
 208                 case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break;
 209                 default:
 210                         WARN_ON(1);
 211                         break;
 212                 }
 213                 addr |= (vmm->pd->pt[0]->addr >> 12) << 4;
 214
 215                 vmm->func->invalidate_pdb(vmm, addr);
 216         }
 217
 218         nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
 219
 220         /* Wait for flush to be queued? */
 221         nvkm_msec(device, 2000,
 222                 if (nvkm_rd32(device, 0x100c80) & 0x00008000)
 223                         break;
 224         );
 225         mutex_unlock(&subdev->mutex);
 226 }
 227
 228 void
 229 gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)
 230 {
 231         u32 type = 0x00000001; /* PAGE_ALL */
 232         if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
 233                 type |= 0x00000004; /* HUB_ONLY */
 234         gf100_vmm_invalidate(vmm, type);
 235 }
 236
 237 int
 238 gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
 239                 struct nvkm_vmm_map *map)
 240 {
 241         const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
 242         const struct nvkm_vmm_page *page = map->page;
 243         const bool gm20x = page->desc->func->sparse != NULL;
 244         union {
 245                 struct gf100_vmm_map_vn vn;
 246                 struct gf100_vmm_map_v0 v0;
 247         } *args = argv;
 248         struct nvkm_device *device = vmm->mmu->subdev.device;
 249         struct nvkm_memory *memory = map->memory;
 250         u8  kind, kind_inv, priv, ro, vol;
 251         int kindn, aper, ret = -ENOSYS;
 252         const u8 *kindm;
 253
 254         map->next = (1 << page->shift) >> 8;
 255         map->type = map->ctag = 0;
 256
 257         if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 258                 vol  = !!args->v0.vol;
 259                 ro   = !!args->v0.ro;
 260                 priv = !!args->v0.priv;
 261                 kind =   args->v0.kind;
 262         } else
 263         if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
 264                 vol  = target == NVKM_MEM_TARGET_HOST;
 265                 ro   = 0;
 266                 priv = 0;
 267                 kind = 0x00;
 268         } else {
 269                 VMM_DEBUG(vmm, "args");
 270                 return ret;
 271         }
 272
 273         aper = vmm->func->aper(target);
 274         if (WARN_ON(aper < 0))
 275                 return aper;
 276
 277         kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);
 278         if (kind >= kindn || kindm[kind] == kind_inv) {
 279                 VMM_DEBUG(vmm, "kind %02x", kind);
 280                 return -EINVAL;
 281         }
 282
 283         if (kindm[kind] != kind) {
 284                 u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;
 285                 u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;
 286                 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
 287                         VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
 288                         return -EINVAL;
 289                 }
 290
 291                 ret = nvkm_memory_tags_get(memory, device, tags,
 292                                            nvkm_ltc_tags_clear,
 293                                            &map->tags);
 294                 if (ret) {
 295                         VMM_DEBUG(vmm, "comp %d", ret);
 296                         return ret;
 297                 }
 298
 299                 if (map->tags->mn) {
 300                         u64 tags = map->tags->mn->offset + (map->offset >> 17);
 301                         if (page->shift == 17 || !gm20x) {
 302                                 map->type |= tags << 44;
 303                                 map->ctag |= 1ULL << 44;
 304                                 map->next |= 1ULL << 44;
 305                         } else {
 306                                 map->ctag |= tags << 1 | 1;
 307                         }
 308                 } else {
 309                         kind = kindm[kind];
 310                 }
 311         }
 312
 313         map->type |= BIT(0);
 314         map->type |= (u64)priv << 1;
 315         map->type |= (u64)  ro << 2;
 316         map->type |= (u64) vol << 32;
 317         map->type |= (u64)aper << 33;
 318         map->type |= (u64)kind << 36;
 319         return 0;
 320 }
 321
 322 int
 323 gf100_vmm_aper(enum nvkm_memory_target target)
 324 {
 325         switch (target) {
 326         case NVKM_MEM_TARGET_VRAM: return 0;
 327         case NVKM_MEM_TARGET_HOST: return 2;
 328         case NVKM_MEM_TARGET_NCOH: return 3;
 329         default:
 330                 return -EINVAL;
 331         }
 332 }
 333
 334 void
 335 gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 336 {
 337         nvkm_fo64(inst, 0x0200, 0x00000000, 2);
 338 }
 339
 340 int
 341 gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
 342 {
 343         struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
 344
 345         switch (nvkm_memory_target(pd->memory)) {
 346         case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;
 347         case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;
 348                 base |= BIT_ULL(2) /* VOL. */;
 349                 break;
 350         case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;
 351         default:
 352                 WARN_ON(1);
 353                 return -EINVAL;
 354         }
 355         base |= pd->addr;
 356
 357         nvkm_kmap(inst);
 358         nvkm_wo64(inst, 0x0200, base);
 359         nvkm_wo64(inst, 0x0208, vmm->limit - 1);
 360         nvkm_done(inst);
 361         return 0;
 362 }
 363
 364 int
 365 gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 366 {
 367         return gf100_vmm_join_(vmm, inst, 0);
 368 }
 369
 370 static const struct nvkm_vmm_func
 371 gf100_vmm_17 = {
 372         .join = gf100_vmm_join,
 373         .part = gf100_vmm_part,
 374         .aper = gf100_vmm_aper,
 375         .valid = gf100_vmm_valid,
 376         .flush = gf100_vmm_flush,
 377         .invalidate_pdb = gf100_vmm_invalidate_pdb,
 378         .page = {
 379                 { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
 380                 { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
 381                 {}
 382         }
 383 };
 384
 385 static const struct nvkm_vmm_func
 386 gf100_vmm_16 = {
 387         .join = gf100_vmm_join,
 388         .part = gf100_vmm_part,
 389         .aper = gf100_vmm_aper,
 390         .valid = gf100_vmm_valid,
 391         .flush = gf100_vmm_flush,
 392         .invalidate_pdb = gf100_vmm_invalidate_pdb,
 393         .page = {
 394                 { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
 395                 { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
 396                 {}
 397         }
 398 };
 399
 400 int
 401 gf100_vmm_new_(const struct nvkm_vmm_func *func_16,
 402                const struct nvkm_vmm_func *func_17,
 403                struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
 404                void *argv, u32 argc, struct lock_class_key *key,
 405                const char *name, struct nvkm_vmm **pvmm)
 406 {
 407         switch (mmu->subdev.device->fb->page) {
 408         case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size,
 409                                       argv, argc, key, name, pvmm);
 410         case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size,
 411                                       argv, argc, key, name, pvmm);
 412         default:
 413                 WARN_ON(1);
 414                 return -EINVAL;
 415         }
 416 }
 417
 418 int
 419 gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
 420               void *argv, u32 argc, struct lock_class_key *key,
 421               const char *name, struct nvkm_vmm **pvmm)
 422 {
 423         return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr,
 424                               size, argv, argc, key, name, pvmm);
 425 }