drivers/iommu/iommufd/io_pagetable.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES.
   3  *
   4  */
   5 #ifndef __IO_PAGETABLE_H
   6 #define __IO_PAGETABLE_H
   7
   8 #include <linux/interval_tree.h>
   9 #include <linux/kref.h>
  10 #include <linux/mutex.h>
  11 #include <linux/xarray.h>
  12
  13 #include "iommufd_private.h"
  14
  15 struct iommu_domain;
  16
  17 /*
  18  * Each io_pagetable is composed of intervals of areas which cover regions of
  19  * the iova that are backed by something. iova not covered by areas is not
  20  * populated in the page table. Each area is fully populated with pages.
  21  *
  22  * iovas are in byte units, but must be iopt->iova_alignment aligned.
  23  *
  24  * pages can be NULL, this means some other thread is still working on setting
  25  * up or tearing down the area. When observed under the write side of the
  26  * domain_rwsem a NULL pages must mean the area is still being setup and no
  27  * domains are filled.
  28  *
  29  * storage_domain points at an arbitrary iommu_domain that is holding the PFNs
  30  * for this area. It is locked by the pages->mutex. This simplifies the locking
  31  * as the pages code can rely on the storage_domain without having to get the
  32  * iopt->domains_rwsem.
  33  *
  34  * The io_pagetable::iova_rwsem protects node
  35  * The iopt_pages::mutex protects pages_node
  36  * iopt and iommu_prot are immutable
  37  * The pages::mutex protects num_accesses
  38  */
  39 struct iopt_area {
  40         struct interval_tree_node node;
  41         struct interval_tree_node pages_node;
  42         struct io_pagetable *iopt;
  43         struct iopt_pages *pages;
  44         struct iommu_domain *storage_domain;
  45         /* How many bytes into the first page the area starts */
  46         unsigned int page_offset;
  47         /* IOMMU_READ, IOMMU_WRITE, etc */
  48         int iommu_prot;
  49         bool prevent_access : 1;
  50         unsigned int num_accesses;
  51 };
  52
  53 struct iopt_allowed {
  54         struct interval_tree_node node;
  55 };
  56
  57 struct iopt_reserved {
  58         struct interval_tree_node node;
  59         void *owner;
  60 };
  61
  62 int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages);
  63 void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages);
  64
  65 int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain);
  66 void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages,
  67                              struct iommu_domain *domain);
  68 void iopt_area_unmap_domain(struct iopt_area *area,
  69                             struct iommu_domain *domain);
  70
  71 static inline unsigned long iopt_area_index(struct iopt_area *area)
  72 {
  73         return area->pages_node.start;
  74 }
  75
  76 static inline unsigned long iopt_area_last_index(struct iopt_area *area)
  77 {
  78         return area->pages_node.last;
  79 }
  80
  81 static inline unsigned long iopt_area_iova(struct iopt_area *area)
  82 {
  83         return area->node.start;
  84 }
  85
  86 static inline unsigned long iopt_area_last_iova(struct iopt_area *area)
  87 {
  88         return area->node.last;
  89 }
  90
  91 static inline size_t iopt_area_length(struct iopt_area *area)
  92 {
  93         return (area->node.last - area->node.start) + 1;
  94 }
  95
  96 /*
  97  * Number of bytes from the start of the iopt_pages that the iova begins.
  98  * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index
  99  * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page
 100  */
 101 static inline unsigned long iopt_area_start_byte(struct iopt_area *area,
 102                                                  unsigned long iova)
 103 {
 104         if (IS_ENABLED(CONFIG_IOMMUFD_TEST))
 105                 WARN_ON(iova < iopt_area_iova(area) ||
 106                         iova > iopt_area_last_iova(area));
 107         return (iova - iopt_area_iova(area)) + area->page_offset +
 108                iopt_area_index(area) * PAGE_SIZE;
 109 }
 110
 111 static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area,
 112                                                     unsigned long iova)
 113 {
 114         return iopt_area_start_byte(area, iova) / PAGE_SIZE;
 115 }
 116
 117 #define __make_iopt_iter(name)                                                 \
 118         static inline struct iopt_##name *iopt_##name##_iter_first(            \
 119                 struct io_pagetable *iopt, unsigned long start,                \
 120                 unsigned long last)                                            \
 121         {                                                                      \
 122                 struct interval_tree_node *node;                               \
 123                                                                                \
 124                 lockdep_assert_held(&iopt->iova_rwsem);                        \
 125                 node = interval_tree_iter_first(&iopt->name##_itree, start,    \
 126                                                 last);                         \
 127                 if (!node)                                                     \
 128                         return NULL;                                           \
 129                 return container_of(node, struct iopt_##name, node);           \
 130         }                                                                      \
 131         static inline struct iopt_##name *iopt_##name##_iter_next(             \
 132                 struct iopt_##name *last_node, unsigned long start,            \
 133                 unsigned long last)                                            \
 134         {                                                                      \
 135                 struct interval_tree_node *node;                               \
 136                                                                                \
 137                 node = interval_tree_iter_next(&last_node->node, start, last); \
 138                 if (!node)                                                     \
 139                         return NULL;                                           \
 140                 return container_of(node, struct iopt_##name, node);           \
 141         }
 142
 143 __make_iopt_iter(area)
 144 __make_iopt_iter(allowed)
 145 __make_iopt_iter(reserved)
 146
 147 struct iopt_area_contig_iter {
 148         unsigned long cur_iova;
 149         unsigned long last_iova;
 150         struct iopt_area *area;
 151 };
 152 struct iopt_area *iopt_area_contig_init(struct iopt_area_contig_iter *iter,
 153                                         struct io_pagetable *iopt,
 154                                         unsigned long iova,
 155                                         unsigned long last_iova);
 156 struct iopt_area *iopt_area_contig_next(struct iopt_area_contig_iter *iter);
 157
 158 static inline bool iopt_area_contig_done(struct iopt_area_contig_iter *iter)
 159 {
 160         return iter->area && iter->last_iova <= iopt_area_last_iova(iter->area);
 161 }
 162
 163 /*
 164  * Iterate over a contiguous list of areas that span the iova,last_iova range.
 165  * The caller must check iopt_area_contig_done() after the loop to see if
 166  * contiguous areas existed.
 167  */
 168 #define iopt_for_each_contig_area(iter, area, iopt, iova, last_iova)          \
 169         for (area = iopt_area_contig_init(iter, iopt, iova, last_iova); area; \
 170              area = iopt_area_contig_next(iter))
 171
 172 enum {
 173         IOPT_PAGES_ACCOUNT_NONE = 0,
 174         IOPT_PAGES_ACCOUNT_USER = 1,
 175         IOPT_PAGES_ACCOUNT_MM = 2,
 176 };
 177
 178 /*
 179  * This holds a pinned page list for multiple areas of IO address space. The
 180  * pages always originate from a linear chunk of userspace VA. Multiple
 181  * io_pagetable's, through their iopt_area's, can share a single iopt_pages
 182  * which avoids multi-pinning and double accounting of page consumption.
 183  *
 184  * indexes in this structure are measured in PAGE_SIZE units, are 0 based from
 185  * the start of the uptr and extend to npages. pages are pinned dynamically
 186  * according to the intervals in the access_itree and domains_itree, npinned
 187  * records the current number of pages pinned.
 188  */
 189 struct iopt_pages {
 190         struct kref kref;
 191         struct mutex mutex;
 192         size_t npages;
 193         size_t npinned;
 194         size_t last_npinned;
 195         struct task_struct *source_task;
 196         struct mm_struct *source_mm;
 197         struct user_struct *source_user;
 198         void __user *uptr;
 199         bool writable:1;
 200         u8 account_mode;
 201
 202         struct xarray pinned_pfns;
 203         /* Of iopt_pages_access::node */
 204         struct rb_root_cached access_itree;
 205         /* Of iopt_area::pages_node */
 206         struct rb_root_cached domains_itree;
 207 };
 208
 209 struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length,
 210                                     bool writable);
 211 void iopt_release_pages(struct kref *kref);
 212 static inline void iopt_put_pages(struct iopt_pages *pages)
 213 {
 214         kref_put(&pages->kref, iopt_release_pages);
 215 }
 216
 217 void iopt_pages_fill_from_xarray(struct iopt_pages *pages, unsigned long start,
 218                                  unsigned long last, struct page **out_pages);
 219 int iopt_pages_fill_xarray(struct iopt_pages *pages, unsigned long start,
 220                            unsigned long last, struct page **out_pages);
 221 void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start,
 222                               unsigned long last);
 223
 224 int iopt_area_add_access(struct iopt_area *area, unsigned long start,
 225                          unsigned long last, struct page **out_pages,
 226                          unsigned int flags);
 227 void iopt_area_remove_access(struct iopt_area *area, unsigned long start,
 228                             unsigned long last);
 229 int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte,
 230                          void *data, unsigned long length, unsigned int flags);
 231
 232 /*
 233  * Each interval represents an active iopt_access_pages(), it acts as an
 234  * interval lock that keeps the PFNs pinned and stored in the xarray.
 235  */
 236 struct iopt_pages_access {
 237         struct interval_tree_node node;
 238         unsigned int users;
 239 };
 240
 241 #endif