mm/util.c

   1 #include <linux/mm.h>
   2 #include <linux/slab.h>
   3 #include <linux/string.h>
   4 #include <linux/module.h>
   5 #include <linux/err.h>
   6 #include <linux/sched.h>
   7 #include <linux/hugetlb.h>
   8 #include <linux/syscalls.h>
   9 #include <linux/mman.h>
  10 #include <linux/file.h>
  11 #include <asm/uaccess.h>
  12
  13 #define CREATE_TRACE_POINTS
  14 #include <trace/events/kmem.h>
  15
  16 /**
  17  * kstrdup - allocate space for and copy an existing string
  18  * @s: the string to duplicate
  19  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  20  */
  21 char *kstrdup(const char *s, gfp_t gfp)
  22 {
  23         size_t len;
  24         char *buf;
  25
  26         if (!s)
  27                 return NULL;
  28
  29         len = strlen(s) + 1;
  30         buf = kmalloc_track_caller(len, gfp);
  31         if (buf)
  32                 memcpy(buf, s, len);
  33         return buf;
  34 }
  35 EXPORT_SYMBOL(kstrdup);
  36
  37 /**
  38  * kstrndup - allocate space for and copy an existing string
  39  * @s: the string to duplicate
  40  * @max: read at most @max chars from @s
  41  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  42  */
  43 char *kstrndup(const char *s, size_t max, gfp_t gfp)
  44 {
  45         size_t len;
  46         char *buf;
  47
  48         if (!s)
  49                 return NULL;
  50
  51         len = strnlen(s, max);
  52         buf = kmalloc_track_caller(len+1, gfp);
  53         if (buf) {
  54                 memcpy(buf, s, len);
  55                 buf[len] = '\0';
  56         }
  57         return buf;
  58 }
  59 EXPORT_SYMBOL(kstrndup);
  60
  61 /**
  62  * kmemdup - duplicate region of memory
  63  *
  64  * @src: memory region to duplicate
  65  * @len: memory region length
  66  * @gfp: GFP mask to use
  67  */
  68 void *kmemdup(const void *src, size_t len, gfp_t gfp)
  69 {
  70         void *p;
  71
  72         p = kmalloc_track_caller(len, gfp);
  73         if (p)
  74                 memcpy(p, src, len);
  75         return p;
  76 }
  77 EXPORT_SYMBOL(kmemdup);
  78
  79 /**
  80  * memdup_user - duplicate memory region from user space
  81  *
  82  * @src: source address in user space
  83  * @len: number of bytes to copy
  84  *
  85  * Returns an ERR_PTR() on failure.
  86  */
  87 void *memdup_user(const void __user *src, size_t len)
  88 {
  89         void *p;
  90
  91         /*
  92          * Always use GFP_KERNEL, since copy_from_user() can sleep and
  93          * cause pagefault, which makes it pointless to use GFP_NOFS
  94          * or GFP_ATOMIC.
  95          */
  96         p = kmalloc_track_caller(len, GFP_KERNEL);
  97         if (!p)
  98                 return ERR_PTR(-ENOMEM);
  99
 100         if (copy_from_user(p, src, len)) {
 101                 kfree(p);
 102                 return ERR_PTR(-EFAULT);
 103         }
 104
 105         return p;
 106 }
 107 EXPORT_SYMBOL(memdup_user);
 108
 109 /**
 110  * __krealloc - like krealloc() but don't free @p.
 111  * @p: object to reallocate memory for.
 112  * @new_size: how many bytes of memory are required.
 113  * @flags: the type of memory to allocate.
 114  *
 115  * This function is like krealloc() except it never frees the originally
 116  * allocated buffer. Use this if you don't want to free the buffer immediately
 117  * like, for example, with RCU.
 118  */
 119 void *__krealloc(const void *p, size_t new_size, gfp_t flags)
 120 {
 121         void *ret;
 122         size_t ks = 0;
 123
 124         if (unlikely(!new_size))
 125                 return ZERO_SIZE_PTR;
 126
 127         if (p)
 128                 ks = ksize(p);
 129
 130         if (ks >= new_size)
 131                 return (void *)p;
 132
 133         ret = kmalloc_track_caller(new_size, flags);
 134         if (ret && p)
 135                 memcpy(ret, p, ks);
 136
 137         return ret;
 138 }
 139 EXPORT_SYMBOL(__krealloc);
 140
 141 /**
 142  * krealloc - reallocate memory. The contents will remain unchanged.
 143  * @p: object to reallocate memory for.
 144  * @new_size: how many bytes of memory are required.
 145  * @flags: the type of memory to allocate.
 146  *
 147  * The contents of the object pointed to are preserved up to the
 148  * lesser of the new and old sizes.  If @p is %NULL, krealloc()
 149  * behaves exactly like kmalloc().  If @size is 0 and @p is not a
 150  * %NULL pointer, the object pointed to is freed.
 151  */
 152 void *krealloc(const void *p, size_t new_size, gfp_t flags)
 153 {
 154         void *ret;
 155
 156         if (unlikely(!new_size)) {
 157                 kfree(p);
 158                 return ZERO_SIZE_PTR;
 159         }
 160
 161         ret = __krealloc(p, new_size, flags);
 162         if (ret && p != ret)
 163                 kfree(p);
 164
 165         return ret;
 166 }
 167 EXPORT_SYMBOL(krealloc);
 168
 169 /**
 170  * kzfree - like kfree but zero memory
 171  * @p: object to free memory of
 172  *
 173  * The memory of the object @p points to is zeroed before freed.
 174  * If @p is %NULL, kzfree() does nothing.
 175  *
 176  * Note: this function zeroes the whole allocated buffer which can be a good
 177  * deal bigger than the requested buffer size passed to kmalloc(). So be
 178  * careful when using this function in performance sensitive code.
 179  */
 180 void kzfree(const void *p)
 181 {
 182         size_t ks;
 183         void *mem = (void *)p;
 184
 185         if (unlikely(ZERO_OR_NULL_PTR(mem)))
 186                 return;
 187         ks = ksize(mem);
 188         memset(mem, 0, ks);
 189         kfree(mem);
 190 }
 191 EXPORT_SYMBOL(kzfree);
 192
 193 /*
 194  * strndup_user - duplicate an existing string from user space
 195  * @s: The string to duplicate
 196  * @n: Maximum number of bytes to copy, including the trailing NUL.
 197  */
 198 char *strndup_user(const char __user *s, long n)
 199 {
 200         char *p;
 201         long length;
 202
 203         length = strnlen_user(s, n);
 204
 205         if (!length)
 206                 return ERR_PTR(-EFAULT);
 207
 208         if (length > n)
 209                 return ERR_PTR(-EINVAL);
 210
 211         p = kmalloc(length, GFP_KERNEL);
 212
 213         if (!p)
 214                 return ERR_PTR(-ENOMEM);
 215
 216         if (copy_from_user(p, s, length)) {
 217                 kfree(p);
 218                 return ERR_PTR(-EFAULT);
 219         }
 220
 221         p[length - 1] = '\0';
 222
 223         return p;
 224 }
 225 EXPORT_SYMBOL(strndup_user);
 226
 227 #ifndef HAVE_ARCH_PICK_MMAP_LAYOUT
 228 void arch_pick_mmap_layout(struct mm_struct *mm)
 229 {
 230         mm->mmap_base = TASK_UNMAPPED_BASE;
 231         mm->get_unmapped_area = arch_get_unmapped_area;
 232         mm->unmap_area = arch_unmap_area;
 233 }
 234 #endif
 235
 236 /**
 237  * get_user_pages_fast() - pin user pages in memory
 238  * @start:      starting user address
 239  * @nr_pages:   number of pages from start to pin
 240  * @write:      whether pages will be written to
 241  * @pages:      array that receives pointers to the pages pinned.
 242  *              Should be at least nr_pages long.
 243  *
 244  * Returns number of pages pinned. This may be fewer than the number
 245  * requested. If nr_pages is 0 or negative, returns 0. If no pages
 246  * were pinned, returns -errno.
 247  *
 248  * get_user_pages_fast provides equivalent functionality to get_user_pages,
 249  * operating on current and current->mm, with force=0 and vma=NULL. However
 250  * unlike get_user_pages, it must be called without mmap_sem held.
 251  *
 252  * get_user_pages_fast may take mmap_sem and page table locks, so no
 253  * assumptions can be made about lack of locking. get_user_pages_fast is to be
 254  * implemented in a way that is advantageous (vs get_user_pages()) when the
 255  * user memory area is already faulted in and present in ptes. However if the
 256  * pages have to be faulted in, it may turn out to be slightly slower so
 257  * callers need to carefully consider what to use. On many architectures,
 258  * get_user_pages_fast simply falls back to get_user_pages.
 259  */
 260 int __attribute__((weak)) get_user_pages_fast(unsigned long start,
 261                                 int nr_pages, int write, struct page **pages)
 262 {
 263         struct mm_struct *mm = current->mm;
 264         int ret;
 265
 266         down_read(&mm->mmap_sem);
 267         ret = get_user_pages(current, mm, start, nr_pages,
 268                                         write, 0, pages, NULL);
 269         up_read(&mm->mmap_sem);
 270
 271         return ret;
 272 }
 273 EXPORT_SYMBOL_GPL(get_user_pages_fast);
 274
 275 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
 276                 unsigned long, prot, unsigned long, flags,
 277                 unsigned long, fd, unsigned long, pgoff)
 278 {
 279         struct file * file = NULL;
 280         unsigned long retval = -EBADF;
 281
 282         if (!(flags & MAP_ANONYMOUS)) {
 283                 if (unlikely(flags & MAP_HUGETLB))
 284                         return -EINVAL;
 285                 file = fget(fd);
 286                 if (!file)
 287                         goto out;
 288         } else if (flags & MAP_HUGETLB) {
 289                 struct user_struct *user = NULL;
 290                 /*
 291                  * VM_NORESERVE is used because the reservations will be
 292                  * taken when vm_ops->mmap() is called
 293                  * A dummy user value is used because we are not locking
 294                  * memory so no accounting is necessary
 295                  */
 296                 len = ALIGN(len, huge_page_size(&default_hstate));
 297                 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE,
 298                                                 &user, HUGETLB_ANONHUGE_INODE);
 299                 if (IS_ERR(file))
 300                         return PTR_ERR(file);
 301         }
 302
 303         flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
 304
 305         down_write(&current->mm->mmap_sem);
 306         retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff);
 307         up_write(&current->mm->mmap_sem);
 308
 309         if (file)
 310                 fput(file);
 311 out:
 312         return retval;
 313 }
 314
 315 /* Tracepoints definitions. */
 316 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
 317 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
 318 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
 319 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
 320 EXPORT_TRACEPOINT_SYMBOL(kfree);
 321 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);