mm/frame_vector.c

   1 #include <linux/kernel.h>
   2 #include <linux/errno.h>
   3 #include <linux/err.h>
   4 #include <linux/mm.h>
   5 #include <linux/slab.h>
   6 #include <linux/vmalloc.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/sched.h>
   9
  10 /**
  11  * get_vaddr_frames() - map virtual addresses to pfns
  12  * @start:      starting user address
  13  * @nr_frames:  number of pages / pfns from start to map
  14  * @gup_flags:  flags modifying lookup behaviour
  15  * @vec:        structure which receives pages / pfns of the addresses mapped.
  16  *              It should have space for at least nr_frames entries.
  17  *
  18  * This function maps virtual addresses from @start and fills @vec structure
  19  * with page frame numbers or page pointers to corresponding pages (choice
  20  * depends on the type of the vma underlying the virtual address). If @start
  21  * belongs to a normal vma, the function grabs reference to each of the pages
  22  * to pin them in memory. If @start belongs to VM_IO | VM_PFNMAP vma, we don't
  23  * touch page structures and the caller must make sure pfns aren't reused for
  24  * anything else while he is using them.
  25  *
  26  * The function returns number of pages mapped which may be less than
  27  * @nr_frames. In particular we stop mapping if there are more vmas of
  28  * different type underlying the specified range of virtual addresses.
  29  * When the function isn't able to map a single page, it returns error.
  30  *
  31  * This function takes care of grabbing mmap_sem as necessary.
  32  */
  33 int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
  34                      unsigned int gup_flags, struct frame_vector *vec)
  35 {
  36         struct mm_struct *mm = current->mm;
  37         struct vm_area_struct *vma;
  38         int ret = 0;
  39         int err;
  40         int locked;
  41
  42         if (nr_frames == 0)
  43                 return 0;
  44
  45         if (WARN_ON_ONCE(nr_frames > vec->nr_allocated))
  46                 nr_frames = vec->nr_allocated;
  47
  48         down_read(&mm->mmap_sem);
  49         locked = 1;
  50         vma = find_vma_intersection(mm, start, start + 1);
  51         if (!vma) {
  52                 ret = -EFAULT;
  53                 goto out;
  54         }
  55         if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) {
  56                 vec->got_ref = true;
  57                 vec->is_pfns = false;
  58                 ret = get_user_pages_locked(current, mm, start, nr_frames,
  59                         gup_flags, (struct page **)(vec->ptrs), &locked);
  60                 goto out;
  61         }
  62
  63         vec->got_ref = false;
  64         vec->is_pfns = true;
  65         do {
  66                 unsigned long *nums = frame_vector_pfns(vec);
  67
  68                 while (ret < nr_frames && start + PAGE_SIZE <= vma->vm_end) {
  69                         err = follow_pfn(vma, start, &nums[ret]);
  70                         if (err) {
  71                                 if (ret == 0)
  72                                         ret = err;
  73                                 goto out;
  74                         }
  75                         start += PAGE_SIZE;
  76                         ret++;
  77                 }
  78                 /*
  79                  * We stop if we have enough pages or if VMA doesn't completely
  80                  * cover the tail page.
  81                  */
  82                 if (ret >= nr_frames || start < vma->vm_end)
  83                         break;
  84                 vma = find_vma_intersection(mm, start, start + 1);
  85         } while (vma && vma->vm_flags & (VM_IO | VM_PFNMAP));
  86 out:
  87         if (locked)
  88                 up_read(&mm->mmap_sem);
  89         if (!ret)
  90                 ret = -EFAULT;
  91         if (ret > 0)
  92                 vec->nr_frames = ret;
  93         return ret;
  94 }
  95 EXPORT_SYMBOL(get_vaddr_frames);
  96
  97 /**
  98  * put_vaddr_frames() - drop references to pages if get_vaddr_frames() acquired
  99  *                      them
 100  * @vec:        frame vector to put
 101  *
 102  * Drop references to pages if get_vaddr_frames() acquired them. We also
 103  * invalidate the frame vector so that it is prepared for the next call into
 104  * get_vaddr_frames().
 105  */
 106 void put_vaddr_frames(struct frame_vector *vec)
 107 {
 108         int i;
 109         struct page **pages;
 110
 111         if (!vec->got_ref)
 112                 goto out;
 113         pages = frame_vector_pages(vec);
 114         /*
 115          * frame_vector_pages() might needed to do a conversion when
 116          * get_vaddr_frames() got pages but vec was later converted to pfns.
 117          * But it shouldn't really fail to convert pfns back...
 118          */
 119         if (WARN_ON(IS_ERR(pages)))
 120                 goto out;
 121         for (i = 0; i < vec->nr_frames; i++)
 122                 put_page(pages[i]);
 123         vec->got_ref = false;
 124 out:
 125         vec->nr_frames = 0;
 126 }
 127 EXPORT_SYMBOL(put_vaddr_frames);
 128
 129 /**
 130  * frame_vector_to_pages - convert frame vector to contain page pointers
 131  * @vec:        frame vector to convert
 132  *
 133  * Convert @vec to contain array of page pointers.  If the conversion is
 134  * successful, return 0. Otherwise return an error. Note that we do not grab
 135  * page references for the page structures.
 136  */
 137 int frame_vector_to_pages(struct frame_vector *vec)
 138 {
 139         int i;
 140         unsigned long *nums;
 141         struct page **pages;
 142
 143         if (!vec->is_pfns)
 144                 return 0;
 145         nums = frame_vector_pfns(vec);
 146         for (i = 0; i < vec->nr_frames; i++)
 147                 if (!pfn_valid(nums[i]))
 148                         return -EINVAL;
 149         pages = (struct page **)nums;
 150         for (i = 0; i < vec->nr_frames; i++)
 151                 pages[i] = pfn_to_page(nums[i]);
 152         vec->is_pfns = false;
 153         return 0;
 154 }
 155 EXPORT_SYMBOL(frame_vector_to_pages);
 156
 157 /**
 158  * frame_vector_to_pfns - convert frame vector to contain pfns
 159  * @vec:        frame vector to convert
 160  *
 161  * Convert @vec to contain array of pfns.
 162  */
 163 void frame_vector_to_pfns(struct frame_vector *vec)
 164 {
 165         int i;
 166         unsigned long *nums;
 167         struct page **pages;
 168
 169         if (vec->is_pfns)
 170                 return;
 171         pages = (struct page **)(vec->ptrs);
 172         nums = (unsigned long *)pages;
 173         for (i = 0; i < vec->nr_frames; i++)
 174                 nums[i] = page_to_pfn(pages[i]);
 175         vec->is_pfns = true;
 176 }
 177 EXPORT_SYMBOL(frame_vector_to_pfns);
 178
 179 /**
 180  * frame_vector_create() - allocate & initialize structure for pinned pfns
 181  * @nr_frames:  number of pfns slots we should reserve
 182  *
 183  * Allocate and initialize struct pinned_pfns to be able to hold @nr_pfns
 184  * pfns.
 185  */
 186 struct frame_vector *frame_vector_create(unsigned int nr_frames)
 187 {
 188         struct frame_vector *vec;
 189         int size = sizeof(struct frame_vector) + sizeof(void *) * nr_frames;
 190
 191         if (WARN_ON_ONCE(nr_frames == 0))
 192                 return NULL;
 193         /*
 194          * This is absurdly high. It's here just to avoid strange effects when
 195          * arithmetics overflows.
 196          */
 197         if (WARN_ON_ONCE(nr_frames > INT_MAX / sizeof(void *) / 2))
 198                 return NULL;
 199         /*
 200          * Avoid higher order allocations, use vmalloc instead. It should
 201          * be rare anyway.
 202          */
 203         if (size <= PAGE_SIZE)
 204                 vec = kmalloc(size, GFP_KERNEL);
 205         else
 206                 vec = vmalloc(size);
 207         if (!vec)
 208                 return NULL;
 209         vec->nr_allocated = nr_frames;
 210         vec->nr_frames = 0;
 211         return vec;
 212 }
 213 EXPORT_SYMBOL(frame_vector_create);
 214
 215 /**
 216  * frame_vector_destroy() - free memory allocated to carry frame vector
 217  * @vec:        Frame vector to free
 218  *
 219  * Free structure allocated by frame_vector_create() to carry frames.
 220  */
 221 void frame_vector_destroy(struct frame_vector *vec)
 222 {
 223         /* Make sure put_vaddr_frames() got called properly... */
 224         VM_BUG_ON(vec->nr_frames > 0);
 225         kvfree(vec);
 226 }
 227 EXPORT_SYMBOL(frame_vector_destroy);