mm/maccess.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Access kernel or user memory without faulting.
   4  */
   5 #include <linux/export.h>
   6 #include <linux/mm.h>
   7 #include <linux/uaccess.h>
   8 #include <asm/tlb.h>
   9
  10 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
  11                 size_t size)
  12 {
  13         return true;
  14 }
  15
  16 /*
  17  * The below only uses kmsan_check_memory() to ensure uninitialized kernel
  18  * memory isn't leaked.
  19  */
  20 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)   \
  21         while (len >= sizeof(type)) {                                   \
  22                 __get_kernel_nofault(dst, src, type, err_label);        \
  23                 kmsan_check_memory(src, sizeof(type));                  \
  24                 dst += sizeof(type);                                    \
  25                 src += sizeof(type);                                    \
  26                 len -= sizeof(type);                                    \
  27         }
  28
  29 long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
  30 {
  31         unsigned long align = 0;
  32
  33         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
  34                 align = (unsigned long)dst | (unsigned long)src;
  35
  36         if (!copy_from_kernel_nofault_allowed(src, size))
  37                 return -ERANGE;
  38
  39         pagefault_disable();
  40         if (!(align & 7))
  41                 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
  42         if (!(align & 3))
  43                 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
  44         if (!(align & 1))
  45                 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
  46         copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
  47         pagefault_enable();
  48         return 0;
  49 Efault:
  50         pagefault_enable();
  51         return -EFAULT;
  52 }
  53 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
  54
  55 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)     \
  56         while (len >= sizeof(type)) {                                   \
  57                 __put_kernel_nofault(dst, src, type, err_label);        \
  58                 instrument_write(dst, sizeof(type));                    \
  59                 dst += sizeof(type);                                    \
  60                 src += sizeof(type);                                    \
  61                 len -= sizeof(type);                                    \
  62         }
  63
  64 long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
  65 {
  66         unsigned long align = 0;
  67
  68         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
  69                 align = (unsigned long)dst | (unsigned long)src;
  70
  71         pagefault_disable();
  72         if (!(align & 7))
  73                 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
  74         if (!(align & 3))
  75                 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
  76         if (!(align & 1))
  77                 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
  78         copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
  79         pagefault_enable();
  80         return 0;
  81 Efault:
  82         pagefault_enable();
  83         return -EFAULT;
  84 }
  85 EXPORT_SYMBOL_GPL(copy_to_kernel_nofault);
  86
  87 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
  88 {
  89         const void *src = unsafe_addr;
  90
  91         if (unlikely(count <= 0))
  92                 return 0;
  93         if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
  94                 return -ERANGE;
  95
  96         pagefault_disable();
  97         do {
  98                 __get_kernel_nofault(dst, src, u8, Efault);
  99                 dst++;
 100                 src++;
 101         } while (dst[-1] && src - unsafe_addr < count);
 102         pagefault_enable();
 103
 104         dst[-1] = '\0';
 105         return src - unsafe_addr;
 106 Efault:
 107         pagefault_enable();
 108         dst[0] = '\0';
 109         return -EFAULT;
 110 }
 111
 112 /**
 113  * copy_from_user_nofault(): safely attempt to read from a user-space location
 114  * @dst: pointer to the buffer that shall take the data
 115  * @src: address to read from. This must be a user address.
 116  * @size: size of the data chunk
 117  *
 118  * Safely read from user address @src to the buffer at @dst. If a kernel fault
 119  * happens, handle that and return -EFAULT.
 120  */
 121 long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
 122 {
 123         long ret = -EFAULT;
 124
 125         if (!__access_ok(src, size))
 126                 return ret;
 127
 128         if (!nmi_uaccess_okay())
 129                 return ret;
 130
 131         pagefault_disable();
 132         ret = __copy_from_user_inatomic(dst, src, size);
 133         pagefault_enable();
 134
 135         if (ret)
 136                 return -EFAULT;
 137         return 0;
 138 }
 139 EXPORT_SYMBOL_GPL(copy_from_user_nofault);
 140
 141 /**
 142  * copy_to_user_nofault(): safely attempt to write to a user-space location
 143  * @dst: address to write to
 144  * @src: pointer to the data that shall be written
 145  * @size: size of the data chunk
 146  *
 147  * Safely write to address @dst from the buffer at @src.  If a kernel fault
 148  * happens, handle that and return -EFAULT.
 149  */
 150 long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
 151 {
 152         long ret = -EFAULT;
 153
 154         if (access_ok(dst, size)) {
 155                 pagefault_disable();
 156                 ret = __copy_to_user_inatomic(dst, src, size);
 157                 pagefault_enable();
 158         }
 159
 160         if (ret)
 161                 return -EFAULT;
 162         return 0;
 163 }
 164 EXPORT_SYMBOL_GPL(copy_to_user_nofault);
 165
 166 /**
 167  * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
 168  *                              address.
 169  * @dst:   Destination address, in kernel space.  This buffer must be at
 170  *         least @count bytes long.
 171  * @unsafe_addr: Unsafe user address.
 172  * @count: Maximum number of bytes to copy, including the trailing NUL.
 173  *
 174  * Copies a NUL-terminated string from unsafe user address to kernel buffer.
 175  *
 176  * On success, returns the length of the string INCLUDING the trailing NUL.
 177  *
 178  * If access fails, returns -EFAULT (some data may have been copied
 179  * and the trailing NUL added).
 180  *
 181  * If @count is smaller than the length of the string, copies @count-1 bytes,
 182  * sets the last byte of @dst buffer to NUL and returns @count.
 183  */
 184 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
 185                               long count)
 186 {
 187         long ret;
 188
 189         if (unlikely(count <= 0))
 190                 return 0;
 191
 192         pagefault_disable();
 193         ret = strncpy_from_user(dst, unsafe_addr, count);
 194         pagefault_enable();
 195
 196         if (ret >= count) {
 197                 ret = count;
 198                 dst[ret - 1] = '\0';
 199         } else if (ret > 0) {
 200                 ret++;
 201         }
 202
 203         return ret;
 204 }
 205
 206 /**
 207  * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
 208  * @unsafe_addr: The string to measure.
 209  * @count: Maximum count (including NUL)
 210  *
 211  * Get the size of a NUL-terminated string in user space without pagefault.
 212  *
 213  * Returns the size of the string INCLUDING the terminating NUL.
 214  *
 215  * If the string is too long, returns a number larger than @count. User
 216  * has to check the return value against "> count".
 217  * On exception (or invalid count), returns 0.
 218  *
 219  * Unlike strnlen_user, this can be used from IRQ handler etc. because
 220  * it disables pagefaults.
 221  */
 222 long strnlen_user_nofault(const void __user *unsafe_addr, long count)
 223 {
 224         int ret;
 225
 226         pagefault_disable();
 227         ret = strnlen_user(unsafe_addr, count);
 228         pagefault_enable();
 229
 230         return ret;
 231 }
 232
 233 void __copy_overflow(int size, unsigned long count)
 234 {
 235         WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
 236 }
 237 EXPORT_SYMBOL(__copy_overflow);