?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
uwb: Use kcalloc instead of kzalloc to allocate array
[zen-stable.git] / arch / x86 / include / asm / uaccess.h
blob36361bf6fdd1ed2e977d87a6c0332410a742c658
1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
3 /*
4 * User space memory access functions
5 */
6 #include <linux/errno.h>
7 #include <linux/compiler.h>
8 #include <linux/thread_info.h>
9 #include <linux/string.h>
10 #include <asm/asm.h>
11 #include <asm/page.h>
12
13 #define VERIFY_READ 0
14 #define VERIFY_WRITE 1
15
16 /*
17 * The fs value determines whether argument validity checking should be
18 * performed or not. If get_fs() == USER_DS, checking is performed, with
19 * get_fs() == KERNEL_DS, checking is bypassed.
20 *
21 * For historical reasons, these macros are grossly misnamed.
22 */
23
24 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
25
26 #define KERNEL_DS MAKE_MM_SEG(-1UL)
27 #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
28
29 #define get_ds() (KERNEL_DS)
30 #define get_fs() (current_thread_info()->addr_limit)
31 #define set_fs(x) (current_thread_info()->addr_limit = (x))
32
33 #define segment_eq(a, b) ((a).seg == (b).seg)
34
35 #define __addr_ok(addr) \
36 ((unsigned long __force)(addr) < \
37 (current_thread_info()->addr_limit.seg))
38
39 /*
40 * Test whether a block of memory is a valid user space address.
41 * Returns 0 if the range is valid, nonzero otherwise.
42 *
43 * This is equivalent to the following test:
44 * (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64)
45 *
46 * This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry...
47 */
48
49 #define __range_not_ok(addr, size) \
50 ({ \
51 unsigned long flag, roksum; \
52 __chk_user_ptr(addr); \
53 asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \
54 : "=&r" (flag), "=r" (roksum) \
55 : "1" (addr), "g" ((long)(size)), \
56 "rm" (current_thread_info()->addr_limit.seg)); \
57 flag; \
58 })
59
60 /**
61 * access_ok: - Checks if a user space pointer is valid
62 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that
63 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
64 * to write to a block, it is always safe to read from it.
65 * @addr: User space pointer to start of block to check
66 * @size: Size of block to check
67 *
68 * Context: User context only. This function may sleep.
69 *
70 * Checks if a pointer to a block of memory in user space is valid.
71 *
72 * Returns true (nonzero) if the memory block may be valid, false (zero)
73 * if it is definitely invalid.
74 *
75 * Note that, depending on architecture, this function probably just
76 * checks that the pointer is in the user space range - after calling
77 * this function, memory access functions may still return -EFAULT.
78 */
79 #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
80
81 /*
82 * The exception table consists of pairs of addresses: the first is the
83 * address of an instruction that is allowed to fault, and the second is
84 * the address at which the program should continue. No registers are
85 * modified, so it is entirely up to the continuation code to figure out
86 * what to do.
87 *
88 * All the routines below use bits of fixup code that are out of line
89 * with the main instruction path. This means when everything is well,
90 * we don't even have to jump over them. Further, they do not intrude
91 * on our cache or tlb entries.
92 */
93
94 struct exception_table_entry {
95 unsigned long insn, fixup;
96 };
97
98 extern int fixup_exception(struct pt_regs *regs);
99
100 /*
101 * These are the main single-value transfer routines. They automatically
102 * use the right size if we just have the right pointer type.
103 *
104 * This gets kind of ugly. We want to return _two_ values in "get_user()"
105 * and yet we don't want to do any pointers, because that is too much
106 * of a performance impact. Thus we have a few rather ugly macros here,
107 * and hide all the ugliness from the user.
108 *
109 * The "__xxx" versions of the user access functions are versions that
110 * do not verify the address space, that must have been done previously
111 * with a separate "access_ok()" call (this is used when we do multiple
112 * accesses to the same area of user memory).
113 */
114
115 extern int __get_user_1(void);
116 extern int __get_user_2(void);
117 extern int __get_user_4(void);
118 extern int __get_user_8(void);
119 extern int __get_user_bad(void);
120
121 #define __get_user_x(size, ret, x, ptr) \
122 asm volatile("call __get_user_" #size \
123 : "=a" (ret), "=d" (x) \
124 : "0" (ptr)) \
125
126 /* Careful: we have to cast the result to the type of the pointer
127 * for sign reasons */
128
129 /**
130 * get_user: - Get a simple variable from user space.
131 * @x: Variable to store result.
132 * @ptr: Source address, in user space.
133 *
134 * Context: User context only. This function may sleep.
135 *
136 * This macro copies a single simple variable from user space to kernel
137 * space. It supports simple types like char and int, but not larger
138 * data types like structures or arrays.
139 *
140 * @ptr must have pointer-to-simple-variable type, and the result of
141 * dereferencing @ptr must be assignable to @x without a cast.
142 *
143 * Returns zero on success, or -EFAULT on error.
144 * On error, the variable @x is set to zero.
145 */
146 #ifdef CONFIG_X86_32
147 #define __get_user_8(__ret_gu, __val_gu, ptr) \
148 __get_user_x(X, __ret_gu, __val_gu, ptr)
149 #else
150 #define __get_user_8(__ret_gu, __val_gu, ptr) \
151 __get_user_x(8, __ret_gu, __val_gu, ptr)
152 #endif
153
154 #define get_user(x, ptr) \
155 ({ \
156 int __ret_gu; \
157 unsigned long __val_gu; \
158 __chk_user_ptr(ptr); \
159 might_fault(); \
160 switch (sizeof(*(ptr))) { \
161 case 1: \
162 __get_user_x(1, __ret_gu, __val_gu, ptr); \
163 break; \
164 case 2: \
165 __get_user_x(2, __ret_gu, __val_gu, ptr); \
166 break; \
167 case 4: \
168 __get_user_x(4, __ret_gu, __val_gu, ptr); \
169 break; \
170 case 8: \
171 __get_user_8(__ret_gu, __val_gu, ptr); \
172 break; \
173 default: \
174 __get_user_x(X, __ret_gu, __val_gu, ptr); \
175 break; \
176 } \
177 (x) = (__typeof__(*(ptr)))__val_gu; \
178 __ret_gu; \
179 })
180
181 #define __put_user_x(size, x, ptr, __ret_pu) \
182 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
183 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
184
185
186
187 #ifdef CONFIG_X86_32
188 #define __put_user_asm_u64(x, addr, err, errret) \
189 asm volatile("1: movl %%eax,0(%2)\n" \
190 "2: movl %%edx,4(%2)\n" \
191 "3:\n" \
192 ".section .fixup,\"ax\"\n" \
193 "4: movl %3,%0\n" \
194 " jmp 3b\n" \
195 ".previous\n" \
196 _ASM_EXTABLE(1b, 4b) \
197 _ASM_EXTABLE(2b, 4b) \
198 : "=r" (err) \
199 : "A" (x), "r" (addr), "i" (errret), "0" (err))
200
201 #define __put_user_asm_ex_u64(x, addr) \
202 asm volatile("1: movl %%eax,0(%1)\n" \
203 "2: movl %%edx,4(%1)\n" \
204 "3:\n" \
205 _ASM_EXTABLE(1b, 2b - 1b) \
206 _ASM_EXTABLE(2b, 3b - 2b) \
207 : : "A" (x), "r" (addr))
208
209 #define __put_user_x8(x, ptr, __ret_pu) \
210 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
211 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
212 #else
213 #define __put_user_asm_u64(x, ptr, retval, errret) \
214 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
215 #define __put_user_asm_ex_u64(x, addr) \
216 __put_user_asm_ex(x, addr, "q", "", "er")
217 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
218 #endif
219
220 extern void __put_user_bad(void);
221
222 /*
223 * Strange magic calling convention: pointer in %ecx,
224 * value in %eax(:%edx), return value in %eax. clobbers %rbx
225 */
226 extern void __put_user_1(void);
227 extern void __put_user_2(void);
228 extern void __put_user_4(void);
229 extern void __put_user_8(void);
230
231 #ifdef CONFIG_X86_WP_WORKS_OK
232
233 /**
234 * put_user: - Write a simple value into user space.
235 * @x: Value to copy to user space.
236 * @ptr: Destination address, in user space.
237 *
238 * Context: User context only. This function may sleep.
239 *
240 * This macro copies a single simple value from kernel space to user
241 * space. It supports simple types like char and int, but not larger
242 * data types like structures or arrays.
243 *
244 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
245 * to the result of dereferencing @ptr.
246 *
247 * Returns zero on success, or -EFAULT on error.
248 */
249 #define put_user(x, ptr) \
250 ({ \
251 int __ret_pu; \
252 __typeof__(*(ptr)) __pu_val; \
253 __chk_user_ptr(ptr); \
254 might_fault(); \
255 __pu_val = x; \
256 switch (sizeof(*(ptr))) { \
257 case 1: \
258 __put_user_x(1, __pu_val, ptr, __ret_pu); \
259 break; \
260 case 2: \
261 __put_user_x(2, __pu_val, ptr, __ret_pu); \
262 break; \
263 case 4: \
264 __put_user_x(4, __pu_val, ptr, __ret_pu); \
265 break; \
266 case 8: \
267 __put_user_x8(__pu_val, ptr, __ret_pu); \
268 break; \
269 default: \
270 __put_user_x(X, __pu_val, ptr, __ret_pu); \
271 break; \
272 } \
273 __ret_pu; \
274 })
275
276 #define __put_user_size(x, ptr, size, retval, errret) \
277 do { \
278 retval = 0; \
279 __chk_user_ptr(ptr); \
280 switch (size) { \
281 case 1: \
282 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
283 break; \
284 case 2: \
285 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
286 break; \
287 case 4: \
288 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
289 break; \
290 case 8: \
291 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
292 errret); \
293 break; \
294 default: \
295 __put_user_bad(); \
296 } \
297 } while (0)
298
299 #define __put_user_size_ex(x, ptr, size) \
300 do { \
301 __chk_user_ptr(ptr); \
302 switch (size) { \
303 case 1: \
304 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
305 break; \
306 case 2: \
307 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
308 break; \
309 case 4: \
310 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
311 break; \
312 case 8: \
313 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
314 break; \
315 default: \
316 __put_user_bad(); \
317 } \
318 } while (0)
319
320 #else
321
322 #define __put_user_size(x, ptr, size, retval, errret) \
323 do { \
324 __typeof__(*(ptr))__pus_tmp = x; \
325 retval = 0; \
326 \
327 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \
328 retval = errret; \
329 } while (0)
330
331 #define put_user(x, ptr) \
332 ({ \
333 int __ret_pu; \
334 __typeof__(*(ptr))__pus_tmp = x; \
335 __ret_pu = 0; \
336 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
337 sizeof(*(ptr))) != 0)) \
338 __ret_pu = -EFAULT; \
339 __ret_pu; \
340 })
341 #endif
342
343 #ifdef CONFIG_X86_32
344 #define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad()
345 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
346 #else
347 #define __get_user_asm_u64(x, ptr, retval, errret) \
348 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
349 #define __get_user_asm_ex_u64(x, ptr) \
350 __get_user_asm_ex(x, ptr, "q", "", "=r")
351 #endif
352
353 #define __get_user_size(x, ptr, size, retval, errret) \
354 do { \
355 retval = 0; \
356 __chk_user_ptr(ptr); \
357 switch (size) { \
358 case 1: \
359 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
360 break; \
361 case 2: \
362 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
363 break; \
364 case 4: \
365 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
366 break; \
367 case 8: \
368 __get_user_asm_u64(x, ptr, retval, errret); \
369 break; \
370 default: \
371 (x) = __get_user_bad(); \
372 } \
373 } while (0)
374
375 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
376 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
377 "2:\n" \
378 ".section .fixup,\"ax\"\n" \
379 "3: mov %3,%0\n" \
380 " xor"itype" %"rtype"1,%"rtype"1\n" \
381 " jmp 2b\n" \
382 ".previous\n" \
383 _ASM_EXTABLE(1b, 3b) \
384 : "=r" (err), ltype(x) \
385 : "m" (__m(addr)), "i" (errret), "0" (err))
386
387 #define __get_user_size_ex(x, ptr, size) \
388 do { \
389 __chk_user_ptr(ptr); \
390 switch (size) { \
391 case 1: \
392 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
393 break; \
394 case 2: \
395 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
396 break; \
397 case 4: \
398 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
399 break; \
400 case 8: \
401 __get_user_asm_ex_u64(x, ptr); \
402 break; \
403 default: \
404 (x) = __get_user_bad(); \
405 } \
406 } while (0)
407
408 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
409 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
410 "2:\n" \
411 _ASM_EXTABLE(1b, 2b - 1b) \
412 : ltype(x) : "m" (__m(addr)))
413
414 #define __put_user_nocheck(x, ptr, size) \
415 ({ \
416 int __pu_err; \
417 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
418 __pu_err; \
419 })
420
421 #define __get_user_nocheck(x, ptr, size) \
422 ({ \
423 int __gu_err; \
424 unsigned long __gu_val; \
425 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
426 (x) = (__force __typeof__(*(ptr)))__gu_val; \
427 __gu_err; \
428 })
429
430 /* FIXME: this hack is definitely wrong -AK */
431 struct __large_struct { unsigned long buf[100]; };
432 #define __m(x) (*(struct __large_struct __user *)(x))
433
434 /*
435 * Tell gcc we read from memory instead of writing: this is because
436 * we do not write to any memory gcc knows about, so there are no
437 * aliasing issues.
438 */
439 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
440 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
441 "2:\n" \
442 ".section .fixup,\"ax\"\n" \
443 "3: mov %3,%0\n" \
444 " jmp 2b\n" \
445 ".previous\n" \
446 _ASM_EXTABLE(1b, 3b) \
447 : "=r"(err) \
448 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
449
450 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
451 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
452 "2:\n" \
453 _ASM_EXTABLE(1b, 2b - 1b) \
454 : : ltype(x), "m" (__m(addr)))
455
456 /*
457 * uaccess_try and catch
458 */
459 #define uaccess_try do { \
460 int prev_err = current_thread_info()->uaccess_err; \
461 current_thread_info()->uaccess_err = 0; \
462 barrier();
463
464 #define uaccess_catch(err) \
465 (err) |= current_thread_info()->uaccess_err; \
466 current_thread_info()->uaccess_err = prev_err; \
467 } while (0)
468
469 /**
470 * __get_user: - Get a simple variable from user space, with less checking.
471 * @x: Variable to store result.
472 * @ptr: Source address, in user space.
473 *
474 * Context: User context only. This function may sleep.
475 *
476 * This macro copies a single simple variable from user space to kernel
477 * space. It supports simple types like char and int, but not larger
478 * data types like structures or arrays.
479 *
480 * @ptr must have pointer-to-simple-variable type, and the result of
481 * dereferencing @ptr must be assignable to @x without a cast.
482 *
483 * Caller must check the pointer with access_ok() before calling this
484 * function.
485 *
486 * Returns zero on success, or -EFAULT on error.
487 * On error, the variable @x is set to zero.
488 */
489
490 #define __get_user(x, ptr) \
491 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
492
493 /**
494 * __put_user: - Write a simple value into user space, with less checking.
495 * @x: Value to copy to user space.
496 * @ptr: Destination address, in user space.
497 *
498 * Context: User context only. This function may sleep.
499 *
500 * This macro copies a single simple value from kernel space to user
501 * space. It supports simple types like char and int, but not larger
502 * data types like structures or arrays.
503 *
504 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
505 * to the result of dereferencing @ptr.
506 *
507 * Caller must check the pointer with access_ok() before calling this
508 * function.
509 *
510 * Returns zero on success, or -EFAULT on error.
511 */
512
513 #define __put_user(x, ptr) \
514 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
515
516 #define __get_user_unaligned __get_user
517 #define __put_user_unaligned __put_user
518
519 /*
520 * {get|put}_user_try and catch
521 *
522 * get_user_try {
523 * get_user_ex(...);
524 * } get_user_catch(err)
525 */
526 #define get_user_try uaccess_try
527 #define get_user_catch(err) uaccess_catch(err)
528
529 #define get_user_ex(x, ptr) do { \
530 unsigned long __gue_val; \
531 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
532 (x) = (__force __typeof__(*(ptr)))__gue_val; \
533 } while (0)
534
535 #ifdef CONFIG_X86_WP_WORKS_OK
536
537 #define put_user_try uaccess_try
538 #define put_user_catch(err) uaccess_catch(err)
539
540 #define put_user_ex(x, ptr) \
541 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
542
543 #else /* !CONFIG_X86_WP_WORKS_OK */
544
545 #define put_user_try do { \
546 int __uaccess_err = 0;
547
548 #define put_user_catch(err) \
549 (err) |= __uaccess_err; \
550 } while (0)
551
552 #define put_user_ex(x, ptr) do { \
553 __uaccess_err |= __put_user(x, ptr); \
554 } while (0)
555
556 #endif /* CONFIG_X86_WP_WORKS_OK */
557
558 extern unsigned long
559 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
560
561 /*
562 * movsl can be slow when source and dest are not both 8-byte aligned
563 */
564 #ifdef CONFIG_X86_INTEL_USERCOPY
565 extern struct movsl_mask {
566 int mask;
567 } ____cacheline_aligned_in_smp movsl_mask;
568 #endif
569
570 #define ARCH_HAS_NOCACHE_UACCESS 1
571
572 #ifdef CONFIG_X86_32
573 # include "uaccess_32.h"
574 #else
575 # include "uaccess_64.h"
576 #endif
577
578 #endif /* _ASM_X86_UACCESS_H */
579