net/ceph/pagevec.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/ceph/ceph_debug.h>
   3
   4 #include <linux/module.h>
   5 #include <linux/sched.h>
   6 #include <linux/slab.h>
   7 #include <linux/file.h>
   8 #include <linux/namei.h>
   9 #include <linux/writeback.h>
  10
  11 #include <linux/ceph/libceph.h>
  12
  13 /*
  14  * build a vector of user pages
  15  */
  16 struct page **ceph_get_direct_page_vector(const void __user *data,
  17                                           int num_pages, bool write_page)
  18 {
  19         struct page **pages;
  20         int got = 0;
  21         int rc = 0;
  22
  23         pages = kmalloc_array(num_pages, sizeof(*pages), GFP_NOFS);
  24         if (!pages)
  25                 return ERR_PTR(-ENOMEM);
  26
  27         while (got < num_pages) {
  28                 rc = get_user_pages_fast(
  29                     (unsigned long)data + ((unsigned long)got * PAGE_SIZE),
  30                     num_pages - got, write_page, pages + got);
  31                 if (rc < 0)
  32                         break;
  33                 BUG_ON(rc == 0);
  34                 got += rc;
  35         }
  36         if (rc < 0)
  37                 goto fail;
  38         return pages;
  39
  40 fail:
  41         ceph_put_page_vector(pages, got, false);
  42         return ERR_PTR(rc);
  43 }
  44 EXPORT_SYMBOL(ceph_get_direct_page_vector);
  45
  46 void ceph_put_page_vector(struct page **pages, int num_pages, bool dirty)
  47 {
  48         int i;
  49
  50         for (i = 0; i < num_pages; i++) {
  51                 if (dirty)
  52                         set_page_dirty_lock(pages[i]);
  53                 put_page(pages[i]);
  54         }
  55         kvfree(pages);
  56 }
  57 EXPORT_SYMBOL(ceph_put_page_vector);
  58
  59 void ceph_release_page_vector(struct page **pages, int num_pages)
  60 {
  61         int i;
  62
  63         for (i = 0; i < num_pages; i++)
  64                 __free_pages(pages[i], 0);
  65         kfree(pages);
  66 }
  67 EXPORT_SYMBOL(ceph_release_page_vector);
  68
  69 /*
  70  * allocate a vector new pages
  71  */
  72 struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags)
  73 {
  74         struct page **pages;
  75         int i;
  76
  77         pages = kmalloc_array(num_pages, sizeof(*pages), flags);
  78         if (!pages)
  79                 return ERR_PTR(-ENOMEM);
  80         for (i = 0; i < num_pages; i++) {
  81                 pages[i] = __page_cache_alloc(flags);
  82                 if (pages[i] == NULL) {
  83                         ceph_release_page_vector(pages, i);
  84                         return ERR_PTR(-ENOMEM);
  85                 }
  86         }
  87         return pages;
  88 }
  89 EXPORT_SYMBOL(ceph_alloc_page_vector);
  90
  91 /*
  92  * copy user data into a page vector
  93  */
  94 int ceph_copy_user_to_page_vector(struct page **pages,
  95                                          const void __user *data,
  96                                          loff_t off, size_t len)
  97 {
  98         int i = 0;
  99         int po = off & ~PAGE_MASK;
 100         int left = len;
 101         int l, bad;
 102
 103         while (left > 0) {
 104                 l = min_t(int, PAGE_SIZE-po, left);
 105                 bad = copy_from_user(page_address(pages[i]) + po, data, l);
 106                 if (bad == l)
 107                         return -EFAULT;
 108                 data += l - bad;
 109                 left -= l - bad;
 110                 po += l - bad;
 111                 if (po == PAGE_SIZE) {
 112                         po = 0;
 113                         i++;
 114                 }
 115         }
 116         return len;
 117 }
 118 EXPORT_SYMBOL(ceph_copy_user_to_page_vector);
 119
 120 void ceph_copy_to_page_vector(struct page **pages,
 121                                     const void *data,
 122                                     loff_t off, size_t len)
 123 {
 124         int i = 0;
 125         size_t po = off & ~PAGE_MASK;
 126         size_t left = len;
 127
 128         while (left > 0) {
 129                 size_t l = min_t(size_t, PAGE_SIZE-po, left);
 130
 131                 memcpy(page_address(pages[i]) + po, data, l);
 132                 data += l;
 133                 left -= l;
 134                 po += l;
 135                 if (po == PAGE_SIZE) {
 136                         po = 0;
 137                         i++;
 138                 }
 139         }
 140 }
 141 EXPORT_SYMBOL(ceph_copy_to_page_vector);
 142
 143 void ceph_copy_from_page_vector(struct page **pages,
 144                                     void *data,
 145                                     loff_t off, size_t len)
 146 {
 147         int i = 0;
 148         size_t po = off & ~PAGE_MASK;
 149         size_t left = len;
 150
 151         while (left > 0) {
 152                 size_t l = min_t(size_t, PAGE_SIZE-po, left);
 153
 154                 memcpy(data, page_address(pages[i]) + po, l);
 155                 data += l;
 156                 left -= l;
 157                 po += l;
 158                 if (po == PAGE_SIZE) {
 159                         po = 0;
 160                         i++;
 161                 }
 162         }
 163 }
 164 EXPORT_SYMBOL(ceph_copy_from_page_vector);
 165
 166 /*
 167  * Zero an extent within a page vector.  Offset is relative to the
 168  * start of the first page.
 169  */
 170 void ceph_zero_page_vector_range(int off, int len, struct page **pages)
 171 {
 172         int i = off >> PAGE_SHIFT;
 173
 174         off &= ~PAGE_MASK;
 175
 176         dout("zero_page_vector_page %u~%u\n", off, len);
 177
 178         /* leading partial page? */
 179         if (off) {
 180                 int end = min((int)PAGE_SIZE, off + len);
 181                 dout("zeroing %d %p head from %d\n", i, pages[i],
 182                      (int)off);
 183                 zero_user_segment(pages[i], off, end);
 184                 len -= (end - off);
 185                 i++;
 186         }
 187         while (len >= PAGE_SIZE) {
 188                 dout("zeroing %d %p len=%d\n", i, pages[i], len);
 189                 zero_user_segment(pages[i], 0, PAGE_SIZE);
 190                 len -= PAGE_SIZE;
 191                 i++;
 192         }
 193         /* trailing partial page? */
 194         if (len) {
 195                 dout("zeroing %d %p tail to %d\n", i, pages[i], (int)len);
 196                 zero_user_segment(pages[i], 0, len);
 197         }
 198 }
 199 EXPORT_SYMBOL(ceph_zero_page_vector_range);