mm/gup_benchmark.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7
   8 #define GUP_FAST_BENCHMARK      _IOWR('g', 1, struct gup_benchmark)
   9 #define GUP_LONGTERM_BENCHMARK  _IOWR('g', 2, struct gup_benchmark)
  10 #define GUP_BENCHMARK           _IOWR('g', 3, struct gup_benchmark)
  11 #define PIN_FAST_BENCHMARK      _IOWR('g', 4, struct gup_benchmark)
  12 #define PIN_BENCHMARK           _IOWR('g', 5, struct gup_benchmark)
  13
  14 struct gup_benchmark {
  15         __u64 get_delta_usec;
  16         __u64 put_delta_usec;
  17         __u64 addr;
  18         __u64 size;
  19         __u32 nr_pages_per_call;
  20         __u32 flags;
  21         __u64 expansion[10];    /* For future use */
  22 };
  23
  24 static void put_back_pages(unsigned int cmd, struct page **pages,
  25                            unsigned long nr_pages)
  26 {
  27         unsigned long i;
  28
  29         switch (cmd) {
  30         case GUP_FAST_BENCHMARK:
  31         case GUP_LONGTERM_BENCHMARK:
  32         case GUP_BENCHMARK:
  33                 for (i = 0; i < nr_pages; i++)
  34                         put_page(pages[i]);
  35                 break;
  36
  37         case PIN_FAST_BENCHMARK:
  38         case PIN_BENCHMARK:
  39                 unpin_user_pages(pages, nr_pages);
  40                 break;
  41         }
  42 }
  43
  44 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  45                               unsigned long nr_pages)
  46 {
  47         unsigned long i;
  48         struct page *page;
  49
  50         switch (cmd) {
  51         case PIN_FAST_BENCHMARK:
  52         case PIN_BENCHMARK:
  53                 for (i = 0; i < nr_pages; i++) {
  54                         page = pages[i];
  55                         if (WARN(!page_maybe_dma_pinned(page),
  56                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  57
  58                                 dump_page(page, "gup_benchmark failure");
  59                                 break;
  60                         }
  61                 }
  62                 break;
  63         }
  64 }
  65
  66 static int __gup_benchmark_ioctl(unsigned int cmd,
  67                 struct gup_benchmark *gup)
  68 {
  69         ktime_t start_time, end_time;
  70         unsigned long i, nr_pages, addr, next;
  71         int nr;
  72         struct page **pages;
  73         int ret = 0;
  74
  75         if (gup->size > ULONG_MAX)
  76                 return -EINVAL;
  77
  78         nr_pages = gup->size / PAGE_SIZE;
  79         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
  80         if (!pages)
  81                 return -ENOMEM;
  82
  83         i = 0;
  84         nr = gup->nr_pages_per_call;
  85         start_time = ktime_get();
  86         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
  87                 if (nr != gup->nr_pages_per_call)
  88                         break;
  89
  90                 next = addr + nr * PAGE_SIZE;
  91                 if (next > gup->addr + gup->size) {
  92                         next = gup->addr + gup->size;
  93                         nr = (next - addr) / PAGE_SIZE;
  94                 }
  95
  96                 /* Filter out most gup flags: only allow a tiny subset here: */
  97                 gup->flags &= FOLL_WRITE;
  98
  99                 switch (cmd) {
 100                 case GUP_FAST_BENCHMARK:
 101                         nr = get_user_pages_fast(addr, nr, gup->flags,
 102                                                  pages + i);
 103                         break;
 104                 case GUP_LONGTERM_BENCHMARK:
 105                         nr = get_user_pages(addr, nr,
 106                                             gup->flags | FOLL_LONGTERM,
 107                                             pages + i, NULL);
 108                         break;
 109                 case GUP_BENCHMARK:
 110                         nr = get_user_pages(addr, nr, gup->flags, pages + i,
 111                                             NULL);
 112                         break;
 113                 case PIN_FAST_BENCHMARK:
 114                         nr = pin_user_pages_fast(addr, nr, gup->flags,
 115                                                  pages + i);
 116                         break;
 117                 case PIN_BENCHMARK:
 118                         nr = pin_user_pages(addr, nr, gup->flags, pages + i,
 119                                             NULL);
 120                         break;
 121                 default:
 122                         kvfree(pages);
 123                         ret = -EINVAL;
 124                         goto out;
 125                 }
 126
 127                 if (nr <= 0)
 128                         break;
 129                 i += nr;
 130         }
 131         end_time = ktime_get();
 132
 133         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 134         nr_pages = i;
 135
 136         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 137         gup->size = addr - gup->addr;
 138
 139         /*
 140          * Take an un-benchmark-timed moment to verify DMA pinned
 141          * state: print a warning if any non-dma-pinned pages are found:
 142          */
 143         verify_dma_pinned(cmd, pages, nr_pages);
 144
 145         start_time = ktime_get();
 146
 147         put_back_pages(cmd, pages, nr_pages);
 148
 149         end_time = ktime_get();
 150         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 151
 152         kvfree(pages);
 153 out:
 154         return ret;
 155 }
 156
 157 static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd,
 158                 unsigned long arg)
 159 {
 160         struct gup_benchmark gup;
 161         int ret;
 162
 163         switch (cmd) {
 164         case GUP_FAST_BENCHMARK:
 165         case GUP_LONGTERM_BENCHMARK:
 166         case GUP_BENCHMARK:
 167         case PIN_FAST_BENCHMARK:
 168         case PIN_BENCHMARK:
 169                 break;
 170         default:
 171                 return -EINVAL;
 172         }
 173
 174         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 175                 return -EFAULT;
 176
 177         ret = __gup_benchmark_ioctl(cmd, &gup);
 178         if (ret)
 179                 return ret;
 180
 181         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 182                 return -EFAULT;
 183
 184         return 0;
 185 }
 186
 187 static const struct file_operations gup_benchmark_fops = {
 188         .open = nonseekable_open,
 189         .unlocked_ioctl = gup_benchmark_ioctl,
 190 };
 191
 192 static int gup_benchmark_init(void)
 193 {
 194         debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL,
 195                                    &gup_benchmark_fops);
 196
 197         return 0;
 198 }
 199
 200 late_initcall(gup_benchmark_init);