drivers/vfio/vfio_iommu_spapr_tce.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO: IOMMU DMA mapping support for TCE on POWER
   4  *
   5  * Copyright (C) 2013 IBM Corp.  All rights reserved.
   6  *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
   7  * Copyright Gavin Shan, IBM Corporation 2014.
   8  *
   9  * Derived from original vfio_iommu_type1.c:
  10  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
  11  *     Author: Alex Williamson <alex.williamson@redhat.com>
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/pci.h>
  16 #include <linux/slab.h>
  17 #include <linux/uaccess.h>
  18 #include <linux/err.h>
  19 #include <linux/vfio.h>
  20 #include <linux/vmalloc.h>
  21 #include <linux/sched/mm.h>
  22 #include <linux/sched/signal.h>
  23 #include <linux/mm.h>
  24 #include "vfio.h"
  25
  26 #include <asm/iommu.h>
  27 #include <asm/tce.h>
  28 #include <asm/mmu_context.h>
  29
  30 #define DRIVER_VERSION  "0.1"
  31 #define DRIVER_AUTHOR   "aik@ozlabs.ru"
  32 #define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"
  33
  34 static void tce_iommu_detach_group(void *iommu_data,
  35                 struct iommu_group *iommu_group);
  36
  37 /*
  38  * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
  39  *
  40  * This code handles mapping and unmapping of user data buffers
  41  * into DMA'ble space using the IOMMU
  42  */
  43
  44 struct tce_iommu_group {
  45         struct list_head next;
  46         struct iommu_group *grp;
  47 };
  48
  49 /*
  50  * A container needs to remember which preregistered region  it has
  51  * referenced to do proper cleanup at the userspace process exit.
  52  */
  53 struct tce_iommu_prereg {
  54         struct list_head next;
  55         struct mm_iommu_table_group_mem_t *mem;
  56 };
  57
  58 /*
  59  * The container descriptor supports only a single group per container.
  60  * Required by the API as the container is not supplied with the IOMMU group
  61  * at the moment of initialization.
  62  */
  63 struct tce_container {
  64         struct mutex lock;
  65         bool enabled;
  66         bool v2;
  67         bool def_window_pending;
  68         unsigned long locked_pages;
  69         struct mm_struct *mm;
  70         struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
  71         struct list_head group_list;
  72         struct list_head prereg_list;
  73 };
  74
  75 static long tce_iommu_mm_set(struct tce_container *container)
  76 {
  77         if (container->mm) {
  78                 if (container->mm == current->mm)
  79                         return 0;
  80                 return -EPERM;
  81         }
  82         BUG_ON(!current->mm);
  83         container->mm = current->mm;
  84         mmgrab(container->mm);
  85
  86         return 0;
  87 }
  88
  89 static long tce_iommu_prereg_free(struct tce_container *container,
  90                 struct tce_iommu_prereg *tcemem)
  91 {
  92         long ret;
  93
  94         ret = mm_iommu_put(container->mm, tcemem->mem);
  95         if (ret)
  96                 return ret;
  97
  98         list_del(&tcemem->next);
  99         kfree(tcemem);
 100
 101         return 0;
 102 }
 103
 104 static long tce_iommu_unregister_pages(struct tce_container *container,
 105                 __u64 vaddr, __u64 size)
 106 {
 107         struct mm_iommu_table_group_mem_t *mem;
 108         struct tce_iommu_prereg *tcemem;
 109         bool found = false;
 110         long ret;
 111
 112         if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
 113                 return -EINVAL;
 114
 115         mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
 116         if (!mem)
 117                 return -ENOENT;
 118
 119         list_for_each_entry(tcemem, &container->prereg_list, next) {
 120                 if (tcemem->mem == mem) {
 121                         found = true;
 122                         break;
 123                 }
 124         }
 125
 126         if (!found)
 127                 ret = -ENOENT;
 128         else
 129                 ret = tce_iommu_prereg_free(container, tcemem);
 130
 131         mm_iommu_put(container->mm, mem);
 132
 133         return ret;
 134 }
 135
 136 static long tce_iommu_register_pages(struct tce_container *container,
 137                 __u64 vaddr, __u64 size)
 138 {
 139         long ret = 0;
 140         struct mm_iommu_table_group_mem_t *mem = NULL;
 141         struct tce_iommu_prereg *tcemem;
 142         unsigned long entries = size >> PAGE_SHIFT;
 143
 144         if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
 145                         ((vaddr + size) < vaddr))
 146                 return -EINVAL;
 147
 148         mem = mm_iommu_get(container->mm, vaddr, entries);
 149         if (mem) {
 150                 list_for_each_entry(tcemem, &container->prereg_list, next) {
 151                         if (tcemem->mem == mem) {
 152                                 ret = -EBUSY;
 153                                 goto put_exit;
 154                         }
 155                 }
 156         } else {
 157                 ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
 158                 if (ret)
 159                         return ret;
 160         }
 161
 162         tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
 163         if (!tcemem) {
 164                 ret = -ENOMEM;
 165                 goto put_exit;
 166         }
 167
 168         tcemem->mem = mem;
 169         list_add(&tcemem->next, &container->prereg_list);
 170
 171         container->enabled = true;
 172
 173         return 0;
 174
 175 put_exit:
 176         mm_iommu_put(container->mm, mem);
 177         return ret;
 178 }
 179
 180 static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
 181                 unsigned int it_page_shift)
 182 {
 183         struct page *page;
 184         unsigned long size = 0;
 185
 186         if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
 187                 return size == (1UL << it_page_shift);
 188
 189         page = pfn_to_page(hpa >> PAGE_SHIFT);
 190         /*
 191          * Check that the TCE table granularity is not bigger than the size of
 192          * a page we just found. Otherwise the hardware can get access to
 193          * a bigger memory chunk that it should.
 194          */
 195         return page_shift(compound_head(page)) >= it_page_shift;
 196 }
 197
 198 static inline bool tce_groups_attached(struct tce_container *container)
 199 {
 200         return !list_empty(&container->group_list);
 201 }
 202
 203 static long tce_iommu_find_table(struct tce_container *container,
 204                 phys_addr_t ioba, struct iommu_table **ptbl)
 205 {
 206         long i;
 207
 208         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
 209                 struct iommu_table *tbl = container->tables[i];
 210
 211                 if (tbl) {
 212                         unsigned long entry = ioba >> tbl->it_page_shift;
 213                         unsigned long start = tbl->it_offset;
 214                         unsigned long end = start + tbl->it_size;
 215
 216                         if ((start <= entry) && (entry < end)) {
 217                                 *ptbl = tbl;
 218                                 return i;
 219                         }
 220                 }
 221         }
 222
 223         return -1;
 224 }
 225
 226 static int tce_iommu_find_free_table(struct tce_container *container)
 227 {
 228         int i;
 229
 230         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
 231                 if (!container->tables[i])
 232                         return i;
 233         }
 234
 235         return -ENOSPC;
 236 }
 237
 238 static int tce_iommu_enable(struct tce_container *container)
 239 {
 240         int ret = 0;
 241         unsigned long locked;
 242         struct iommu_table_group *table_group;
 243         struct tce_iommu_group *tcegrp;
 244
 245         if (container->enabled)
 246                 return -EBUSY;
 247
 248         /*
 249          * When userspace pages are mapped into the IOMMU, they are effectively
 250          * locked memory, so, theoretically, we need to update the accounting
 251          * of locked pages on each map and unmap.  For powerpc, the map unmap
 252          * paths can be very hot, though, and the accounting would kill
 253          * performance, especially since it would be difficult to impossible
 254          * to handle the accounting in real mode only.
 255          *
 256          * To address that, rather than precisely accounting every page, we
 257          * instead account for a worst case on locked memory when the iommu is
 258          * enabled and disabled.  The worst case upper bound on locked memory
 259          * is the size of the whole iommu window, which is usually relatively
 260          * small (compared to total memory sizes) on POWER hardware.
 261          *
 262          * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
 263          * that would effectively kill the guest at random points, much better
 264          * enforcing the limit based on the max that the guest can map.
 265          *
 266          * Unfortunately at the moment it counts whole tables, no matter how
 267          * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
 268          * each with 2GB DMA window, 8GB will be counted here. The reason for
 269          * this is that we cannot tell here the amount of RAM used by the guest
 270          * as this information is only available from KVM and VFIO is
 271          * KVM agnostic.
 272          *
 273          * So we do not allow enabling a container without a group attached
 274          * as there is no way to know how much we should increment
 275          * the locked_vm counter.
 276          */
 277         if (!tce_groups_attached(container))
 278                 return -ENODEV;
 279
 280         tcegrp = list_first_entry(&container->group_list,
 281                         struct tce_iommu_group, next);
 282         table_group = iommu_group_get_iommudata(tcegrp->grp);
 283         if (!table_group)
 284                 return -ENODEV;
 285
 286         if (!table_group->tce32_size)
 287                 return -EPERM;
 288
 289         ret = tce_iommu_mm_set(container);
 290         if (ret)
 291                 return ret;
 292
 293         locked = table_group->tce32_size >> PAGE_SHIFT;
 294         ret = account_locked_vm(container->mm, locked, true);
 295         if (ret)
 296                 return ret;
 297
 298         container->locked_pages = locked;
 299
 300         container->enabled = true;
 301
 302         return ret;
 303 }
 304
 305 static void tce_iommu_disable(struct tce_container *container)
 306 {
 307         if (!container->enabled)
 308                 return;
 309
 310         container->enabled = false;
 311
 312         BUG_ON(!container->mm);
 313         account_locked_vm(container->mm, container->locked_pages, false);
 314 }
 315
 316 static void *tce_iommu_open(unsigned long arg)
 317 {
 318         struct tce_container *container;
 319
 320         if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
 321                 pr_err("tce_vfio: Wrong IOMMU type\n");
 322                 return ERR_PTR(-EINVAL);
 323         }
 324
 325         container = kzalloc(sizeof(*container), GFP_KERNEL);
 326         if (!container)
 327                 return ERR_PTR(-ENOMEM);
 328
 329         mutex_init(&container->lock);
 330         INIT_LIST_HEAD_RCU(&container->group_list);
 331         INIT_LIST_HEAD_RCU(&container->prereg_list);
 332
 333         container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
 334
 335         return container;
 336 }
 337
 338 static int tce_iommu_clear(struct tce_container *container,
 339                 struct iommu_table *tbl,
 340                 unsigned long entry, unsigned long pages);
 341 static void tce_iommu_free_table(struct tce_container *container,
 342                 struct iommu_table *tbl);
 343
 344 static void tce_iommu_release(void *iommu_data)
 345 {
 346         struct tce_container *container = iommu_data;
 347         struct tce_iommu_group *tcegrp;
 348         struct tce_iommu_prereg *tcemem, *tmtmp;
 349         long i;
 350
 351         while (tce_groups_attached(container)) {
 352                 tcegrp = list_first_entry(&container->group_list,
 353                                 struct tce_iommu_group, next);
 354                 tce_iommu_detach_group(iommu_data, tcegrp->grp);
 355         }
 356
 357         /*
 358          * If VFIO created a table, it was not disposed
 359          * by tce_iommu_detach_group() so do it now.
 360          */
 361         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
 362                 struct iommu_table *tbl = container->tables[i];
 363
 364                 if (!tbl)
 365                         continue;
 366
 367                 tce_iommu_free_table(container, tbl);
 368         }
 369
 370         list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
 371                 WARN_ON(tce_iommu_prereg_free(container, tcemem));
 372
 373         tce_iommu_disable(container);
 374         if (container->mm)
 375                 mmdrop(container->mm);
 376         mutex_destroy(&container->lock);
 377
 378         kfree(container);
 379 }
 380
 381 static void tce_iommu_unuse_page(unsigned long hpa)
 382 {
 383         struct page *page;
 384
 385         page = pfn_to_page(hpa >> PAGE_SHIFT);
 386         unpin_user_page(page);
 387 }
 388
 389 static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
 390                 unsigned long tce, unsigned long shift,
 391                 unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
 392 {
 393         long ret = 0;
 394         struct mm_iommu_table_group_mem_t *mem;
 395
 396         mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
 397         if (!mem)
 398                 return -EINVAL;
 399
 400         ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
 401         if (ret)
 402                 return -EINVAL;
 403
 404         *pmem = mem;
 405
 406         return 0;
 407 }
 408
 409 static void tce_iommu_unuse_page_v2(struct tce_container *container,
 410                 struct iommu_table *tbl, unsigned long entry)
 411 {
 412         struct mm_iommu_table_group_mem_t *mem = NULL;
 413         int ret;
 414         unsigned long hpa = 0;
 415         __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
 416
 417         if (!pua)
 418                 return;
 419
 420         ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
 421                         tbl->it_page_shift, &hpa, &mem);
 422         if (ret)
 423                 pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
 424                                 __func__, be64_to_cpu(*pua), entry, ret);
 425         if (mem)
 426                 mm_iommu_mapped_dec(mem);
 427
 428         *pua = cpu_to_be64(0);
 429 }
 430
 431 static int tce_iommu_clear(struct tce_container *container,
 432                 struct iommu_table *tbl,
 433                 unsigned long entry, unsigned long pages)
 434 {
 435         unsigned long oldhpa;
 436         long ret;
 437         enum dma_data_direction direction;
 438         unsigned long lastentry = entry + pages, firstentry = entry;
 439
 440         for ( ; entry < lastentry; ++entry) {
 441                 if (tbl->it_indirect_levels && tbl->it_userspace) {
 442                         /*
 443                          * For multilevel tables, we can take a shortcut here
 444                          * and skip some TCEs as we know that the userspace
 445                          * addresses cache is a mirror of the real TCE table
 446                          * and if it is missing some indirect levels, then
 447                          * the hardware table does not have them allocated
 448                          * either and therefore does not require updating.
 449                          */
 450                         __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
 451                                         entry);
 452                         if (!pua) {
 453                                 /* align to level_size which is power of two */
 454                                 entry |= tbl->it_level_size - 1;
 455                                 continue;
 456                         }
 457                 }
 458
 459                 cond_resched();
 460
 461                 direction = DMA_NONE;
 462                 oldhpa = 0;
 463                 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
 464                                 &direction);
 465                 if (ret)
 466                         continue;
 467
 468                 if (direction == DMA_NONE)
 469                         continue;
 470
 471                 if (container->v2) {
 472                         tce_iommu_unuse_page_v2(container, tbl, entry);
 473                         continue;
 474                 }
 475
 476                 tce_iommu_unuse_page(oldhpa);
 477         }
 478
 479         iommu_tce_kill(tbl, firstentry, pages);
 480
 481         return 0;
 482 }
 483
 484 static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
 485 {
 486         struct page *page = NULL;
 487         enum dma_data_direction direction = iommu_tce_direction(tce);
 488
 489         if (pin_user_pages_fast(tce & PAGE_MASK, 1,
 490                         direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
 491                         &page) != 1)
 492                 return -EFAULT;
 493
 494         *hpa = __pa((unsigned long) page_address(page));
 495
 496         return 0;
 497 }
 498
 499 static long tce_iommu_build(struct tce_container *container,
 500                 struct iommu_table *tbl,
 501                 unsigned long entry, unsigned long tce, unsigned long pages,
 502                 enum dma_data_direction direction)
 503 {
 504         long i, ret = 0;
 505         unsigned long hpa;
 506         enum dma_data_direction dirtmp;
 507
 508         for (i = 0; i < pages; ++i) {
 509                 unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
 510
 511                 ret = tce_iommu_use_page(tce, &hpa);
 512                 if (ret)
 513                         break;
 514
 515                 if (!tce_page_is_contained(container->mm, hpa,
 516                                 tbl->it_page_shift)) {
 517                         ret = -EPERM;
 518                         break;
 519                 }
 520
 521                 hpa |= offset;
 522                 dirtmp = direction;
 523                 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
 524                                 &hpa, &dirtmp);
 525                 if (ret) {
 526                         tce_iommu_unuse_page(hpa);
 527                         pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
 528                                         __func__, entry << tbl->it_page_shift,
 529                                         tce, ret);
 530                         break;
 531                 }
 532
 533                 if (dirtmp != DMA_NONE)
 534                         tce_iommu_unuse_page(hpa);
 535
 536                 tce += IOMMU_PAGE_SIZE(tbl);
 537         }
 538
 539         if (ret)
 540                 tce_iommu_clear(container, tbl, entry, i);
 541         else
 542                 iommu_tce_kill(tbl, entry, pages);
 543
 544         return ret;
 545 }
 546
 547 static long tce_iommu_build_v2(struct tce_container *container,
 548                 struct iommu_table *tbl,
 549                 unsigned long entry, unsigned long tce, unsigned long pages,
 550                 enum dma_data_direction direction)
 551 {
 552         long i, ret = 0;
 553         unsigned long hpa;
 554         enum dma_data_direction dirtmp;
 555
 556         for (i = 0; i < pages; ++i) {
 557                 struct mm_iommu_table_group_mem_t *mem = NULL;
 558                 __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
 559
 560                 ret = tce_iommu_prereg_ua_to_hpa(container,
 561                                 tce, tbl->it_page_shift, &hpa, &mem);
 562                 if (ret)
 563                         break;
 564
 565                 if (!tce_page_is_contained(container->mm, hpa,
 566                                 tbl->it_page_shift)) {
 567                         ret = -EPERM;
 568                         break;
 569                 }
 570
 571                 /* Preserve offset within IOMMU page */
 572                 hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
 573                 dirtmp = direction;
 574
 575                 /* The registered region is being unregistered */
 576                 if (mm_iommu_mapped_inc(mem))
 577                         break;
 578
 579                 ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
 580                                 &hpa, &dirtmp);
 581                 if (ret) {
 582                         /* dirtmp cannot be DMA_NONE here */
 583                         tce_iommu_unuse_page_v2(container, tbl, entry + i);
 584                         pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
 585                                         __func__, entry << tbl->it_page_shift,
 586                                         tce, ret);
 587                         break;
 588                 }
 589
 590                 if (dirtmp != DMA_NONE)
 591                         tce_iommu_unuse_page_v2(container, tbl, entry + i);
 592
 593                 *pua = cpu_to_be64(tce);
 594
 595                 tce += IOMMU_PAGE_SIZE(tbl);
 596         }
 597
 598         if (ret)
 599                 tce_iommu_clear(container, tbl, entry, i);
 600         else
 601                 iommu_tce_kill(tbl, entry, pages);
 602
 603         return ret;
 604 }
 605
 606 static long tce_iommu_create_table(struct tce_container *container,
 607                         struct iommu_table_group *table_group,
 608                         int num,
 609                         __u32 page_shift,
 610                         __u64 window_size,
 611                         __u32 levels,
 612                         struct iommu_table **ptbl)
 613 {
 614         long ret, table_size;
 615
 616         table_size = table_group->ops->get_table_size(page_shift, window_size,
 617                         levels);
 618         if (!table_size)
 619                 return -EINVAL;
 620
 621         ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
 622         if (ret)
 623                 return ret;
 624
 625         ret = table_group->ops->create_table(table_group, num,
 626                         page_shift, window_size, levels, ptbl);
 627
 628         WARN_ON(!ret && !(*ptbl)->it_ops->free);
 629         WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
 630
 631         return ret;
 632 }
 633
 634 static void tce_iommu_free_table(struct tce_container *container,
 635                 struct iommu_table *tbl)
 636 {
 637         unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
 638
 639         iommu_tce_table_put(tbl);
 640         account_locked_vm(container->mm, pages, false);
 641 }
 642
 643 static long tce_iommu_create_window(struct tce_container *container,
 644                 __u32 page_shift, __u64 window_size, __u32 levels,
 645                 __u64 *start_addr)
 646 {
 647         struct tce_iommu_group *tcegrp;
 648         struct iommu_table_group *table_group;
 649         struct iommu_table *tbl = NULL;
 650         long ret, num;
 651
 652         num = tce_iommu_find_free_table(container);
 653         if (num < 0)
 654                 return num;
 655
 656         /* Get the first group for ops::create_table */
 657         tcegrp = list_first_entry(&container->group_list,
 658                         struct tce_iommu_group, next);
 659         table_group = iommu_group_get_iommudata(tcegrp->grp);
 660         if (!table_group)
 661                 return -EFAULT;
 662
 663         if (!(table_group->pgsizes & (1ULL << page_shift)))
 664                 return -EINVAL;
 665
 666         if (!table_group->ops->set_window || !table_group->ops->unset_window ||
 667                         !table_group->ops->get_table_size ||
 668                         !table_group->ops->create_table)
 669                 return -EPERM;
 670
 671         /* Create TCE table */
 672         ret = tce_iommu_create_table(container, table_group, num,
 673                         page_shift, window_size, levels, &tbl);
 674         if (ret)
 675                 return ret;
 676
 677         BUG_ON(!tbl->it_ops->free);
 678
 679         /*
 680          * Program the table to every group.
 681          * Groups have been tested for compatibility at the attach time.
 682          */
 683         list_for_each_entry(tcegrp, &container->group_list, next) {
 684                 table_group = iommu_group_get_iommudata(tcegrp->grp);
 685
 686                 ret = table_group->ops->set_window(table_group, num, tbl);
 687                 if (ret)
 688                         goto unset_exit;
 689         }
 690
 691         container->tables[num] = tbl;
 692
 693         /* Return start address assigned by platform in create_table() */
 694         *start_addr = tbl->it_offset << tbl->it_page_shift;
 695
 696         return 0;
 697
 698 unset_exit:
 699         list_for_each_entry(tcegrp, &container->group_list, next) {
 700                 table_group = iommu_group_get_iommudata(tcegrp->grp);
 701                 table_group->ops->unset_window(table_group, num);
 702         }
 703         tce_iommu_free_table(container, tbl);
 704
 705         return ret;
 706 }
 707
 708 static long tce_iommu_remove_window(struct tce_container *container,
 709                 __u64 start_addr)
 710 {
 711         struct iommu_table_group *table_group = NULL;
 712         struct iommu_table *tbl;
 713         struct tce_iommu_group *tcegrp;
 714         int num;
 715
 716         num = tce_iommu_find_table(container, start_addr, &tbl);
 717         if (num < 0)
 718                 return -EINVAL;
 719
 720         BUG_ON(!tbl->it_size);
 721
 722         tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
 723
 724         /* Detach groups from IOMMUs */
 725         list_for_each_entry(tcegrp, &container->group_list, next) {
 726                 table_group = iommu_group_get_iommudata(tcegrp->grp);
 727
 728                 /*
 729                  * SPAPR TCE IOMMU exposes the default DMA window to
 730                  * the guest via dma32_window_start/size of
 731                  * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
 732                  * the userspace to remove this window, some do not so
 733                  * here we check for the platform capability.
 734                  */
 735                 if (!table_group->ops || !table_group->ops->unset_window)
 736                         return -EPERM;
 737
 738                 table_group->ops->unset_window(table_group, num);
 739         }
 740
 741         /* Free table */
 742         tce_iommu_free_table(container, tbl);
 743         container->tables[num] = NULL;
 744
 745         return 0;
 746 }
 747
 748 static long tce_iommu_create_default_window(struct tce_container *container)
 749 {
 750         long ret;
 751         __u64 start_addr = 0;
 752         struct tce_iommu_group *tcegrp;
 753         struct iommu_table_group *table_group;
 754
 755         if (!container->def_window_pending)
 756                 return 0;
 757
 758         if (!tce_groups_attached(container))
 759                 return -ENODEV;
 760
 761         tcegrp = list_first_entry(&container->group_list,
 762                         struct tce_iommu_group, next);
 763         table_group = iommu_group_get_iommudata(tcegrp->grp);
 764         if (!table_group)
 765                 return -ENODEV;
 766
 767         ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
 768                         table_group->tce32_size, 1, &start_addr);
 769         WARN_ON_ONCE(!ret && start_addr);
 770
 771         if (!ret)
 772                 container->def_window_pending = false;
 773
 774         return ret;
 775 }
 776
 777 static long vfio_spapr_ioctl_eeh_pe_op(struct iommu_group *group,
 778                                        unsigned long arg)
 779 {
 780         struct eeh_pe *pe;
 781         struct vfio_eeh_pe_op op;
 782         unsigned long minsz;
 783
 784         pe = eeh_iommu_group_to_pe(group);
 785         if (!pe)
 786                 return -ENODEV;
 787
 788         minsz = offsetofend(struct vfio_eeh_pe_op, op);
 789         if (copy_from_user(&op, (void __user *)arg, minsz))
 790                 return -EFAULT;
 791         if (op.argsz < minsz || op.flags)
 792                 return -EINVAL;
 793
 794         switch (op.op) {
 795         case VFIO_EEH_PE_DISABLE:
 796                 return eeh_pe_set_option(pe, EEH_OPT_DISABLE);
 797         case VFIO_EEH_PE_ENABLE:
 798                 return eeh_pe_set_option(pe, EEH_OPT_ENABLE);
 799         case VFIO_EEH_PE_UNFREEZE_IO:
 800                 return eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
 801         case VFIO_EEH_PE_UNFREEZE_DMA:
 802                 return eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
 803         case VFIO_EEH_PE_GET_STATE:
 804                 return eeh_pe_get_state(pe);
 805                 break;
 806         case VFIO_EEH_PE_RESET_DEACTIVATE:
 807                 return eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, true);
 808         case VFIO_EEH_PE_RESET_HOT:
 809                 return eeh_pe_reset(pe, EEH_RESET_HOT, true);
 810         case VFIO_EEH_PE_RESET_FUNDAMENTAL:
 811                 return eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL, true);
 812         case VFIO_EEH_PE_CONFIGURE:
 813                 return eeh_pe_configure(pe);
 814         case VFIO_EEH_PE_INJECT_ERR:
 815                 minsz = offsetofend(struct vfio_eeh_pe_op, err.mask);
 816                 if (op.argsz < minsz)
 817                         return -EINVAL;
 818                 if (copy_from_user(&op, (void __user *)arg, minsz))
 819                         return -EFAULT;
 820
 821                 return eeh_pe_inject_err(pe, op.err.type, op.err.func,
 822                                          op.err.addr, op.err.mask);
 823         default:
 824                 return -EINVAL;
 825         }
 826 }
 827
 828 static long tce_iommu_ioctl(void *iommu_data,
 829                                  unsigned int cmd, unsigned long arg)
 830 {
 831         struct tce_container *container = iommu_data;
 832         unsigned long minsz, ddwsz;
 833         long ret;
 834
 835         switch (cmd) {
 836         case VFIO_CHECK_EXTENSION:
 837                 switch (arg) {
 838                 case VFIO_SPAPR_TCE_IOMMU:
 839                 case VFIO_SPAPR_TCE_v2_IOMMU:
 840                         return 1;
 841                 case VFIO_EEH:
 842                         return eeh_enabled();
 843                 default:
 844                         return 0;
 845                 }
 846         }
 847
 848         /*
 849          * Sanity check to prevent one userspace from manipulating
 850          * another userspace mm.
 851          */
 852         BUG_ON(!container);
 853         if (container->mm && container->mm != current->mm)
 854                 return -EPERM;
 855
 856         switch (cmd) {
 857         case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
 858                 struct vfio_iommu_spapr_tce_info info;
 859                 struct tce_iommu_group *tcegrp;
 860                 struct iommu_table_group *table_group;
 861
 862                 if (!tce_groups_attached(container))
 863                         return -ENXIO;
 864
 865                 tcegrp = list_first_entry(&container->group_list,
 866                                 struct tce_iommu_group, next);
 867                 table_group = iommu_group_get_iommudata(tcegrp->grp);
 868
 869                 if (!table_group)
 870                         return -ENXIO;
 871
 872                 minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
 873                                 dma32_window_size);
 874
 875                 if (copy_from_user(&info, (void __user *)arg, minsz))
 876                         return -EFAULT;
 877
 878                 if (info.argsz < minsz)
 879                         return -EINVAL;
 880
 881                 info.dma32_window_start = table_group->tce32_start;
 882                 info.dma32_window_size = table_group->tce32_size;
 883                 info.flags = 0;
 884                 memset(&info.ddw, 0, sizeof(info.ddw));
 885
 886                 if (table_group->max_dynamic_windows_supported &&
 887                                 container->v2) {
 888                         info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
 889                         info.ddw.pgsizes = table_group->pgsizes;
 890                         info.ddw.max_dynamic_windows_supported =
 891                                 table_group->max_dynamic_windows_supported;
 892                         info.ddw.levels = table_group->max_levels;
 893                 }
 894
 895                 ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
 896
 897                 if (info.argsz >= ddwsz)
 898                         minsz = ddwsz;
 899
 900                 if (copy_to_user((void __user *)arg, &info, minsz))
 901                         return -EFAULT;
 902
 903                 return 0;
 904         }
 905         case VFIO_IOMMU_MAP_DMA: {
 906                 struct vfio_iommu_type1_dma_map param;
 907                 struct iommu_table *tbl = NULL;
 908                 long num;
 909                 enum dma_data_direction direction;
 910
 911                 if (!container->enabled)
 912                         return -EPERM;
 913
 914                 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
 915
 916                 if (copy_from_user(&param, (void __user *)arg, minsz))
 917                         return -EFAULT;
 918
 919                 if (param.argsz < minsz)
 920                         return -EINVAL;
 921
 922                 if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
 923                                 VFIO_DMA_MAP_FLAG_WRITE))
 924                         return -EINVAL;
 925
 926                 ret = tce_iommu_create_default_window(container);
 927                 if (ret)
 928                         return ret;
 929
 930                 num = tce_iommu_find_table(container, param.iova, &tbl);
 931                 if (num < 0)
 932                         return -ENXIO;
 933
 934                 if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
 935                                 (param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
 936                         return -EINVAL;
 937
 938                 /* iova is checked by the IOMMU API */
 939                 if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
 940                         if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
 941                                 direction = DMA_BIDIRECTIONAL;
 942                         else
 943                                 direction = DMA_TO_DEVICE;
 944                 } else {
 945                         if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
 946                                 direction = DMA_FROM_DEVICE;
 947                         else
 948                                 return -EINVAL;
 949                 }
 950
 951                 ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
 952                 if (ret)
 953                         return ret;
 954
 955                 if (container->v2)
 956                         ret = tce_iommu_build_v2(container, tbl,
 957                                         param.iova >> tbl->it_page_shift,
 958                                         param.vaddr,
 959                                         param.size >> tbl->it_page_shift,
 960                                         direction);
 961                 else
 962                         ret = tce_iommu_build(container, tbl,
 963                                         param.iova >> tbl->it_page_shift,
 964                                         param.vaddr,
 965                                         param.size >> tbl->it_page_shift,
 966                                         direction);
 967
 968                 iommu_flush_tce(tbl);
 969
 970                 return ret;
 971         }
 972         case VFIO_IOMMU_UNMAP_DMA: {
 973                 struct vfio_iommu_type1_dma_unmap param;
 974                 struct iommu_table *tbl = NULL;
 975                 long num;
 976
 977                 if (!container->enabled)
 978                         return -EPERM;
 979
 980                 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
 981                                 size);
 982
 983                 if (copy_from_user(&param, (void __user *)arg, minsz))
 984                         return -EFAULT;
 985
 986                 if (param.argsz < minsz)
 987                         return -EINVAL;
 988
 989                 /* No flag is supported now */
 990                 if (param.flags)
 991                         return -EINVAL;
 992
 993                 ret = tce_iommu_create_default_window(container);
 994                 if (ret)
 995                         return ret;
 996
 997                 num = tce_iommu_find_table(container, param.iova, &tbl);
 998                 if (num < 0)
 999                         return -ENXIO;
1000
1001                 if (param.size & ~IOMMU_PAGE_MASK(tbl))
1002                         return -EINVAL;
1003
1004                 ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
1005                                 param.size >> tbl->it_page_shift);
1006                 if (ret)
1007                         return ret;
1008
1009                 ret = tce_iommu_clear(container, tbl,
1010                                 param.iova >> tbl->it_page_shift,
1011                                 param.size >> tbl->it_page_shift);
1012                 iommu_flush_tce(tbl);
1013
1014                 return ret;
1015         }
1016         case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
1017                 struct vfio_iommu_spapr_register_memory param;
1018
1019                 if (!container->v2)
1020                         break;
1021
1022                 minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1023                                 size);
1024
1025                 ret = tce_iommu_mm_set(container);
1026                 if (ret)
1027                         return ret;
1028
1029                 if (copy_from_user(&param, (void __user *)arg, minsz))
1030                         return -EFAULT;
1031
1032                 if (param.argsz < minsz)
1033                         return -EINVAL;
1034
1035                 /* No flag is supported now */
1036                 if (param.flags)
1037                         return -EINVAL;
1038
1039                 mutex_lock(&container->lock);
1040                 ret = tce_iommu_register_pages(container, param.vaddr,
1041                                 param.size);
1042                 mutex_unlock(&container->lock);
1043
1044                 return ret;
1045         }
1046         case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
1047                 struct vfio_iommu_spapr_register_memory param;
1048
1049                 if (!container->v2)
1050                         break;
1051
1052                 if (!container->mm)
1053                         return -EPERM;
1054
1055                 minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1056                                 size);
1057
1058                 if (copy_from_user(&param, (void __user *)arg, minsz))
1059                         return -EFAULT;
1060
1061                 if (param.argsz < minsz)
1062                         return -EINVAL;
1063
1064                 /* No flag is supported now */
1065                 if (param.flags)
1066                         return -EINVAL;
1067
1068                 mutex_lock(&container->lock);
1069                 ret = tce_iommu_unregister_pages(container, param.vaddr,
1070                                 param.size);
1071                 mutex_unlock(&container->lock);
1072
1073                 return ret;
1074         }
1075         case VFIO_IOMMU_ENABLE:
1076                 if (container->v2)
1077                         break;
1078
1079                 mutex_lock(&container->lock);
1080                 ret = tce_iommu_enable(container);
1081                 mutex_unlock(&container->lock);
1082                 return ret;
1083
1084
1085         case VFIO_IOMMU_DISABLE:
1086                 if (container->v2)
1087                         break;
1088
1089                 mutex_lock(&container->lock);
1090                 tce_iommu_disable(container);
1091                 mutex_unlock(&container->lock);
1092                 return 0;
1093
1094         case VFIO_EEH_PE_OP: {
1095                 struct tce_iommu_group *tcegrp;
1096
1097                 ret = 0;
1098                 list_for_each_entry(tcegrp, &container->group_list, next) {
1099                         ret = vfio_spapr_ioctl_eeh_pe_op(tcegrp->grp, arg);
1100                         if (ret)
1101                                 return ret;
1102                 }
1103                 return ret;
1104         }
1105
1106         case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1107                 struct vfio_iommu_spapr_tce_create create;
1108
1109                 if (!container->v2)
1110                         break;
1111
1112                 ret = tce_iommu_mm_set(container);
1113                 if (ret)
1114                         return ret;
1115
1116                 if (!tce_groups_attached(container))
1117                         return -ENXIO;
1118
1119                 minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1120                                 start_addr);
1121
1122                 if (copy_from_user(&create, (void __user *)arg, minsz))
1123                         return -EFAULT;
1124
1125                 if (create.argsz < minsz)
1126                         return -EINVAL;
1127
1128                 if (create.flags)
1129                         return -EINVAL;
1130
1131                 mutex_lock(&container->lock);
1132
1133                 ret = tce_iommu_create_default_window(container);
1134                 if (!ret)
1135                         ret = tce_iommu_create_window(container,
1136                                         create.page_shift,
1137                                         create.window_size, create.levels,
1138                                         &create.start_addr);
1139
1140                 mutex_unlock(&container->lock);
1141
1142                 if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1143                         ret = -EFAULT;
1144
1145                 return ret;
1146         }
1147         case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1148                 struct vfio_iommu_spapr_tce_remove remove;
1149
1150                 if (!container->v2)
1151                         break;
1152
1153                 ret = tce_iommu_mm_set(container);
1154                 if (ret)
1155                         return ret;
1156
1157                 if (!tce_groups_attached(container))
1158                         return -ENXIO;
1159
1160                 minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1161                                 start_addr);
1162
1163                 if (copy_from_user(&remove, (void __user *)arg, minsz))
1164                         return -EFAULT;
1165
1166                 if (remove.argsz < minsz)
1167                         return -EINVAL;
1168
1169                 if (remove.flags)
1170                         return -EINVAL;
1171
1172                 if (container->def_window_pending && !remove.start_addr) {
1173                         container->def_window_pending = false;
1174                         return 0;
1175                 }
1176
1177                 mutex_lock(&container->lock);
1178
1179                 ret = tce_iommu_remove_window(container, remove.start_addr);
1180
1181                 mutex_unlock(&container->lock);
1182
1183                 return ret;
1184         }
1185         }
1186
1187         return -ENOTTY;
1188 }
1189
1190 static void tce_iommu_release_ownership(struct tce_container *container,
1191                 struct iommu_table_group *table_group)
1192 {
1193         long i;
1194
1195         if (!table_group->ops->unset_window) {
1196                 WARN_ON_ONCE(1);
1197                 return;
1198         }
1199
1200         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1201                 if (container->tables[i]) {
1202                         tce_iommu_clear(container, container->tables[i],
1203                                         container->tables[i]->it_offset,
1204                                         container->tables[i]->it_size);
1205                         table_group->ops->unset_window(table_group, i);
1206                 }
1207         }
1208 }
1209
1210 static long tce_iommu_take_ownership(struct tce_container *container,
1211                 struct iommu_table_group *table_group)
1212 {
1213         long i, ret = 0;
1214
1215         /* Set all windows to the new group */
1216         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1217                 struct iommu_table *tbl = container->tables[i];
1218
1219                 if (!tbl)
1220                         continue;
1221
1222                 ret = table_group->ops->set_window(table_group, i, tbl);
1223                 if (ret)
1224                         goto release_exit;
1225         }
1226
1227         return 0;
1228
1229 release_exit:
1230         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1231                 table_group->ops->unset_window(table_group, i);
1232
1233         return ret;
1234 }
1235
1236 static int tce_iommu_attach_group(void *iommu_data,
1237                 struct iommu_group *iommu_group, enum vfio_group_type type)
1238 {
1239         int ret = 0;
1240         struct tce_container *container = iommu_data;
1241         struct iommu_table_group *table_group;
1242         struct tce_iommu_group *tcegrp = NULL;
1243
1244         if (type == VFIO_EMULATED_IOMMU)
1245                 return -EINVAL;
1246
1247         mutex_lock(&container->lock);
1248
1249         /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
1250                         iommu_group_id(iommu_group), iommu_group); */
1251         table_group = iommu_group_get_iommudata(iommu_group);
1252         if (!table_group) {
1253                 ret = -ENODEV;
1254                 goto unlock_exit;
1255         }
1256
1257         /* v2 requires full support of dynamic DMA windows */
1258         if (container->v2 && table_group->max_dynamic_windows_supported == 0) {
1259                 ret = -EINVAL;
1260                 goto unlock_exit;
1261         }
1262
1263         /* v1 reuses TCE tables and does not share them among PEs */
1264         if (!container->v2 && tce_groups_attached(container)) {
1265                 ret = -EBUSY;
1266                 goto unlock_exit;
1267         }
1268
1269         /*
1270          * Check if new group has the same iommu_table_group_ops
1271          * (i.e. compatible)
1272          */
1273         list_for_each_entry(tcegrp, &container->group_list, next) {
1274                 struct iommu_table_group *table_group_tmp;
1275
1276                 if (tcegrp->grp == iommu_group) {
1277                         pr_warn("tce_vfio: Group %d is already attached\n",
1278                                         iommu_group_id(iommu_group));
1279                         ret = -EBUSY;
1280                         goto unlock_exit;
1281                 }
1282                 table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
1283                 if (table_group_tmp->ops->create_table !=
1284                                 table_group->ops->create_table) {
1285                         pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
1286                                         iommu_group_id(iommu_group),
1287                                         iommu_group_id(tcegrp->grp));
1288                         ret = -EPERM;
1289                         goto unlock_exit;
1290                 }
1291         }
1292
1293         tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
1294         if (!tcegrp) {
1295                 ret = -ENOMEM;
1296                 goto unlock_exit;
1297         }
1298
1299         ret = tce_iommu_take_ownership(container, table_group);
1300         if (!tce_groups_attached(container) && !container->tables[0])
1301                 container->def_window_pending = true;
1302
1303         if (!ret) {
1304                 tcegrp->grp = iommu_group;
1305                 list_add(&tcegrp->next, &container->group_list);
1306         }
1307
1308         if (ret && tcegrp)
1309                 kfree(tcegrp);
1310
1311 unlock_exit:
1312         mutex_unlock(&container->lock);
1313
1314         return ret;
1315 }
1316
1317 static void tce_iommu_detach_group(void *iommu_data,
1318                 struct iommu_group *iommu_group)
1319 {
1320         struct tce_container *container = iommu_data;
1321         struct iommu_table_group *table_group;
1322         bool found = false;
1323         struct tce_iommu_group *tcegrp;
1324
1325         mutex_lock(&container->lock);
1326
1327         list_for_each_entry(tcegrp, &container->group_list, next) {
1328                 if (tcegrp->grp == iommu_group) {
1329                         found = true;
1330                         break;
1331                 }
1332         }
1333
1334         if (!found) {
1335                 pr_warn("tce_vfio: detaching unattached group #%u\n",
1336                                 iommu_group_id(iommu_group));
1337                 goto unlock_exit;
1338         }
1339
1340         list_del(&tcegrp->next);
1341         kfree(tcegrp);
1342
1343         table_group = iommu_group_get_iommudata(iommu_group);
1344         BUG_ON(!table_group);
1345
1346         tce_iommu_release_ownership(container, table_group);
1347
1348 unlock_exit:
1349         mutex_unlock(&container->lock);
1350 }
1351
1352 static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
1353         .name           = "iommu-vfio-powerpc",
1354         .owner          = THIS_MODULE,
1355         .open           = tce_iommu_open,
1356         .release        = tce_iommu_release,
1357         .ioctl          = tce_iommu_ioctl,
1358         .attach_group   = tce_iommu_attach_group,
1359         .detach_group   = tce_iommu_detach_group,
1360 };
1361
1362 static int __init tce_iommu_init(void)
1363 {
1364         return vfio_register_iommu_driver(&tce_iommu_driver_ops);
1365 }
1366
1367 static void __exit tce_iommu_cleanup(void)
1368 {
1369         vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
1370 }
1371
1372 module_init(tce_iommu_init);
1373 module_exit(tce_iommu_cleanup);
1374
1375 MODULE_VERSION(DRIVER_VERSION);
1376 MODULE_LICENSE("GPL v2");
1377 MODULE_AUTHOR(DRIVER_AUTHOR);
1378 MODULE_DESCRIPTION(DRIVER_DESC);
1379