tools/testing/selftests/mm/gup_test.c

   1 #define __SANE_USERSPACE_TYPES__ // Use ll64
   2 #include <fcntl.h>
   3 #include <errno.h>
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6 #include <unistd.h>
   7 #include <dirent.h>
   8 #include <sys/ioctl.h>
   9 #include <sys/mman.h>
  10 #include <sys/stat.h>
  11 #include <sys/types.h>
  12 #include <pthread.h>
  13 #include <assert.h>
  14 #include <mm/gup_test.h>
  15 #include "../kselftest.h"
  16 #include "vm_util.h"
  17
  18 #define MB (1UL << 20)
  19
  20 /* Just the flags we need, copied from mm.h: */
  21 #define FOLL_WRITE      0x01    /* check pte is writable */
  22 #define FOLL_TOUCH      0x02    /* mark page accessed */
  23
  24 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
  25
  26 static unsigned long cmd = GUP_FAST_BENCHMARK;
  27 static int gup_fd, repeats = 1;
  28 static unsigned long size = 128 * MB;
  29 /* Serialize prints */
  30 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
  31
  32 static char *cmd_to_str(unsigned long cmd)
  33 {
  34         switch (cmd) {
  35         case GUP_FAST_BENCHMARK:
  36                 return "GUP_FAST_BENCHMARK";
  37         case PIN_FAST_BENCHMARK:
  38                 return "PIN_FAST_BENCHMARK";
  39         case PIN_LONGTERM_BENCHMARK:
  40                 return "PIN_LONGTERM_BENCHMARK";
  41         case GUP_BASIC_TEST:
  42                 return "GUP_BASIC_TEST";
  43         case PIN_BASIC_TEST:
  44                 return "PIN_BASIC_TEST";
  45         case DUMP_USER_PAGES_TEST:
  46                 return "DUMP_USER_PAGES_TEST";
  47         }
  48         return "Unknown command";
  49 }
  50
  51 void *gup_thread(void *data)
  52 {
  53         struct gup_test gup = *(struct gup_test *)data;
  54         int i, status;
  55
  56         /* Only report timing information on the *_BENCHMARK commands: */
  57         if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
  58              (cmd == PIN_LONGTERM_BENCHMARK)) {
  59                 for (i = 0; i < repeats; i++) {
  60                         gup.size = size;
  61                         status = ioctl(gup_fd, cmd, &gup);
  62                         if (status)
  63                                 break;
  64
  65                         pthread_mutex_lock(&print_mutex);
  66                         ksft_print_msg("%s: Time: get:%lld put:%lld us",
  67                                        cmd_to_str(cmd), gup.get_delta_usec,
  68                                        gup.put_delta_usec);
  69                         if (gup.size != size)
  70                                 ksft_print_msg(", truncated (size: %lld)", gup.size);
  71                         ksft_print_msg("\n");
  72                         pthread_mutex_unlock(&print_mutex);
  73                 }
  74         } else {
  75                 gup.size = size;
  76                 status = ioctl(gup_fd, cmd, &gup);
  77                 if (status)
  78                         goto return_;
  79
  80                 pthread_mutex_lock(&print_mutex);
  81                 ksft_print_msg("%s: done\n", cmd_to_str(cmd));
  82                 if (gup.size != size)
  83                         ksft_print_msg("Truncated (size: %lld)\n", gup.size);
  84                 pthread_mutex_unlock(&print_mutex);
  85         }
  86
  87 return_:
  88         ksft_test_result(!status, "ioctl status %d\n", status);
  89         return NULL;
  90 }
  91
  92 int main(int argc, char **argv)
  93 {
  94         struct gup_test gup = { 0 };
  95         int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
  96         int flags = MAP_PRIVATE, touch = 0;
  97         char *file = "/dev/zero";
  98         pthread_t *tid;
  99         char *p;
 100
 101         while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
 102                 switch (opt) {
 103                 case 'a':
 104                         cmd = PIN_FAST_BENCHMARK;
 105                         break;
 106                 case 'b':
 107                         cmd = PIN_BASIC_TEST;
 108                         break;
 109                 case 'L':
 110                         cmd = PIN_LONGTERM_BENCHMARK;
 111                         break;
 112                 case 'c':
 113                         cmd = DUMP_USER_PAGES_TEST;
 114                         /*
 115                          * Dump page 0 (index 1). May be overridden later, by
 116                          * user's non-option arguments.
 117                          *
 118                          * .which_pages is zero-based, so that zero can mean "do
 119                          * nothing".
 120                          */
 121                         gup.which_pages[0] = 1;
 122                         break;
 123                 case 'p':
 124                         /* works only with DUMP_USER_PAGES_TEST */
 125                         gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
 126                         break;
 127                 case 'F':
 128                         /* strtol, so you can pass flags in hex form */
 129                         gup.gup_flags = strtol(optarg, 0, 0);
 130                         break;
 131                 case 'j':
 132                         nthreads = atoi(optarg);
 133                         break;
 134                 case 'm':
 135                         size = atoi(optarg) * MB;
 136                         break;
 137                 case 'r':
 138                         repeats = atoi(optarg);
 139                         break;
 140                 case 'n':
 141                         nr_pages = atoi(optarg);
 142                         break;
 143                 case 't':
 144                         thp = 1;
 145                         break;
 146                 case 'T':
 147                         thp = 0;
 148                         break;
 149                 case 'U':
 150                         cmd = GUP_BASIC_TEST;
 151                         break;
 152                 case 'u':
 153                         cmd = GUP_FAST_BENCHMARK;
 154                         break;
 155                 case 'w':
 156                         write = 1;
 157                         break;
 158                 case 'W':
 159                         write = 0;
 160                         break;
 161                 case 'f':
 162                         file = optarg;
 163                         break;
 164                 case 'S':
 165                         flags &= ~MAP_PRIVATE;
 166                         flags |= MAP_SHARED;
 167                         break;
 168                 case 'H':
 169                         flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
 170                         break;
 171                 case 'z':
 172                         /* fault pages in gup, do not fault in userland */
 173                         touch = 1;
 174                         break;
 175                 default:
 176                         ksft_exit_fail_msg("Wrong argument\n");
 177                 }
 178         }
 179
 180         if (optind < argc) {
 181                 int extra_arg_count = 0;
 182                 /*
 183                  * For example:
 184                  *
 185                  *   ./gup_test -c 0 1 0x1001
 186                  *
 187                  * ...to dump pages 0, 1, and 4097
 188                  */
 189
 190                 while ((optind < argc) &&
 191                        (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
 192                         /*
 193                          * Do the 1-based indexing here, so that the user can
 194                          * use normal 0-based indexing on the command line.
 195                          */
 196                         long page_index = strtol(argv[optind], 0, 0) + 1;
 197
 198                         gup.which_pages[extra_arg_count] = page_index;
 199                         extra_arg_count++;
 200                         optind++;
 201                 }
 202         }
 203
 204         ksft_print_header();
 205         ksft_set_plan(nthreads);
 206
 207         filed = open(file, O_RDWR|O_CREAT, 0664);
 208         if (filed < 0)
 209                 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno));
 210
 211         gup.nr_pages_per_call = nr_pages;
 212         if (write)
 213                 gup.gup_flags |= FOLL_WRITE;
 214
 215         gup_fd = open(GUP_TEST_FILE, O_RDWR);
 216         if (gup_fd == -1) {
 217                 switch (errno) {
 218                 case EACCES:
 219                         if (getuid())
 220                                 ksft_print_msg("Please run this test as root\n");
 221                         break;
 222                 case ENOENT:
 223                         if (opendir("/sys/kernel/debug") == NULL)
 224                                 ksft_print_msg("mount debugfs at /sys/kernel/debug\n");
 225                         ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n");
 226                         break;
 227                 default:
 228                         ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno));
 229                         break;
 230                 }
 231                 ksft_test_result_skip("Please run this test as root\n");
 232                 ksft_exit_pass();
 233         }
 234
 235         p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
 236         if (p == MAP_FAILED)
 237                 ksft_exit_fail_msg("mmap: %s\n", strerror(errno));
 238         gup.addr = (unsigned long)p;
 239
 240         if (thp == 1)
 241                 madvise(p, size, MADV_HUGEPAGE);
 242         else if (thp == 0)
 243                 madvise(p, size, MADV_NOHUGEPAGE);
 244
 245         /*
 246          * FOLL_TOUCH, in gup_test, is used as an either/or case: either
 247          * fault pages in from the kernel via FOLL_TOUCH, or fault them
 248          * in here, from user space. This allows comparison of performance
 249          * between those two cases.
 250          */
 251         if (touch) {
 252                 gup.gup_flags |= FOLL_TOUCH;
 253         } else {
 254                 for (; (unsigned long)p < gup.addr + size; p += psize())
 255                         p[0] = 0;
 256         }
 257
 258         tid = malloc(sizeof(pthread_t) * nthreads);
 259         assert(tid);
 260         for (i = 0; i < nthreads; i++) {
 261                 ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
 262                 assert(ret == 0);
 263         }
 264         for (i = 0; i < nthreads; i++) {
 265                 ret = pthread_join(tid[i], NULL);
 266                 assert(ret == 0);
 267         }
 268
 269         free(tid);
 270
 271         ksft_exit_pass();
 272 }