?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'pm-4.13-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
[linux/fpc-iii.git] / arch / x86 / include / asm / uaccess.h
blob30269dafec47916e0f46866fa9cb3c19e93f9828
1 #ifndef _ASM_X86_UACCESS_H
2 #define _ASM_X86_UACCESS_H
3 /*
4 * User space memory access functions
5 */
6 #include <linux/compiler.h>
7 #include <linux/kasan-checks.h>
8 #include <linux/string.h>
9 #include <asm/asm.h>
10 #include <asm/page.h>
11 #include <asm/smap.h>
12 #include <asm/extable.h>
13
14 /*
15 * The fs value determines whether argument validity checking should be
16 * performed or not. If get_fs() == USER_DS, checking is performed, with
17 * get_fs() == KERNEL_DS, checking is bypassed.
18 *
19 * For historical reasons, these macros are grossly misnamed.
20 */
21
22 #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
23
24 #define KERNEL_DS MAKE_MM_SEG(-1UL)
25 #define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
26
27 #define get_ds() (KERNEL_DS)
28 #define get_fs() (current->thread.addr_limit)
29 #define set_fs(x) (current->thread.addr_limit = (x))
30
31 #define segment_eq(a, b) ((a).seg == (b).seg)
32
33 #define user_addr_max() (current->thread.addr_limit.seg)
34 #define __addr_ok(addr) \
35 ((unsigned long __force)(addr) < user_addr_max())
36
37 /*
38 * Test whether a block of memory is a valid user space address.
39 * Returns 0 if the range is valid, nonzero otherwise.
40 */
41 static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
42 {
43 /*
44 * If we have used "sizeof()" for the size,
45 * we know it won't overflow the limit (but
46 * it might overflow the 'addr', so it's
47 * important to subtract the size from the
48 * limit, not add it to the address).
49 */
50 if (__builtin_constant_p(size))
51 return unlikely(addr > limit - size);
52
53 /* Arbitrary sizes? Be careful about overflow */
54 addr += size;
55 if (unlikely(addr < size))
56 return true;
57 return unlikely(addr > limit);
58 }
59
60 #define __range_not_ok(addr, size, limit) \
61 ({ \
62 __chk_user_ptr(addr); \
63 __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
64 })
65
66 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
67 # define WARN_ON_IN_IRQ() WARN_ON_ONCE(!in_task())
68 #else
69 # define WARN_ON_IN_IRQ()
70 #endif
71
72 /**
73 * access_ok: - Checks if a user space pointer is valid
74 * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that
75 * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
76 * to write to a block, it is always safe to read from it.
77 * @addr: User space pointer to start of block to check
78 * @size: Size of block to check
79 *
80 * Context: User context only. This function may sleep if pagefaults are
81 * enabled.
82 *
83 * Checks if a pointer to a block of memory in user space is valid.
84 *
85 * Returns true (nonzero) if the memory block may be valid, false (zero)
86 * if it is definitely invalid.
87 *
88 * Note that, depending on architecture, this function probably just
89 * checks that the pointer is in the user space range - after calling
90 * this function, memory access functions may still return -EFAULT.
91 */
92 #define access_ok(type, addr, size) \
93 ({ \
94 WARN_ON_IN_IRQ(); \
95 likely(!__range_not_ok(addr, size, user_addr_max())); \
96 })
97
98 /*
99 * These are the main single-value transfer routines. They automatically
100 * use the right size if we just have the right pointer type.
101 *
102 * This gets kind of ugly. We want to return _two_ values in "get_user()"
103 * and yet we don't want to do any pointers, because that is too much
104 * of a performance impact. Thus we have a few rather ugly macros here,
105 * and hide all the ugliness from the user.
106 *
107 * The "__xxx" versions of the user access functions are versions that
108 * do not verify the address space, that must have been done previously
109 * with a separate "access_ok()" call (this is used when we do multiple
110 * accesses to the same area of user memory).
111 */
112
113 extern int __get_user_1(void);
114 extern int __get_user_2(void);
115 extern int __get_user_4(void);
116 extern int __get_user_8(void);
117 extern int __get_user_bad(void);
118
119 #define __uaccess_begin() stac()
120 #define __uaccess_end() clac()
121
122 /*
123 * This is a type: either unsigned long, if the argument fits into
124 * that type, or otherwise unsigned long long.
125 */
126 #define __inttype(x) \
127 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
128
129 /**
130 * get_user: - Get a simple variable from user space.
131 * @x: Variable to store result.
132 * @ptr: Source address, in user space.
133 *
134 * Context: User context only. This function may sleep if pagefaults are
135 * enabled.
136 *
137 * This macro copies a single simple variable from user space to kernel
138 * space. It supports simple types like char and int, but not larger
139 * data types like structures or arrays.
140 *
141 * @ptr must have pointer-to-simple-variable type, and the result of
142 * dereferencing @ptr must be assignable to @x without a cast.
143 *
144 * Returns zero on success, or -EFAULT on error.
145 * On error, the variable @x is set to zero.
146 */
147 /*
148 * Careful: we have to cast the result to the type of the pointer
149 * for sign reasons.
150 *
151 * The use of _ASM_DX as the register specifier is a bit of a
152 * simplification, as gcc only cares about it as the starting point
153 * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
154 * (%ecx being the next register in gcc's x86 register sequence), and
155 * %rdx on 64 bits.
156 *
157 * Clang/LLVM cares about the size of the register, but still wants
158 * the base register for something that ends up being a pair.
159 */
160 #define get_user(x, ptr) \
161 ({ \
162 int __ret_gu; \
163 register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \
164 register void *__sp asm(_ASM_SP); \
165 __chk_user_ptr(ptr); \
166 might_fault(); \
167 asm volatile("call __get_user_%P4" \
168 : "=a" (__ret_gu), "=r" (__val_gu), "+r" (__sp) \
169 : "0" (ptr), "i" (sizeof(*(ptr)))); \
170 (x) = (__force __typeof__(*(ptr))) __val_gu; \
171 __builtin_expect(__ret_gu, 0); \
172 })
173
174 #define __put_user_x(size, x, ptr, __ret_pu) \
175 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
176 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
177
178
179
180 #ifdef CONFIG_X86_32
181 #define __put_user_asm_u64(x, addr, err, errret) \
182 asm volatile("\n" \
183 "1: movl %%eax,0(%2)\n" \
184 "2: movl %%edx,4(%2)\n" \
185 "3:" \
186 ".section .fixup,\"ax\"\n" \
187 "4: movl %3,%0\n" \
188 " jmp 3b\n" \
189 ".previous\n" \
190 _ASM_EXTABLE(1b, 4b) \
191 _ASM_EXTABLE(2b, 4b) \
192 : "=r" (err) \
193 : "A" (x), "r" (addr), "i" (errret), "0" (err))
194
195 #define __put_user_asm_ex_u64(x, addr) \
196 asm volatile("\n" \
197 "1: movl %%eax,0(%1)\n" \
198 "2: movl %%edx,4(%1)\n" \
199 "3:" \
200 _ASM_EXTABLE_EX(1b, 2b) \
201 _ASM_EXTABLE_EX(2b, 3b) \
202 : : "A" (x), "r" (addr))
203
204 #define __put_user_x8(x, ptr, __ret_pu) \
205 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
206 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
207 #else
208 #define __put_user_asm_u64(x, ptr, retval, errret) \
209 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
210 #define __put_user_asm_ex_u64(x, addr) \
211 __put_user_asm_ex(x, addr, "q", "", "er")
212 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
213 #endif
214
215 extern void __put_user_bad(void);
216
217 /*
218 * Strange magic calling convention: pointer in %ecx,
219 * value in %eax(:%edx), return value in %eax. clobbers %rbx
220 */
221 extern void __put_user_1(void);
222 extern void __put_user_2(void);
223 extern void __put_user_4(void);
224 extern void __put_user_8(void);
225
226 /**
227 * put_user: - Write a simple value into user space.
228 * @x: Value to copy to user space.
229 * @ptr: Destination address, in user space.
230 *
231 * Context: User context only. This function may sleep if pagefaults are
232 * enabled.
233 *
234 * This macro copies a single simple value from kernel space to user
235 * space. It supports simple types like char and int, but not larger
236 * data types like structures or arrays.
237 *
238 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
239 * to the result of dereferencing @ptr.
240 *
241 * Returns zero on success, or -EFAULT on error.
242 */
243 #define put_user(x, ptr) \
244 ({ \
245 int __ret_pu; \
246 __typeof__(*(ptr)) __pu_val; \
247 __chk_user_ptr(ptr); \
248 might_fault(); \
249 __pu_val = x; \
250 switch (sizeof(*(ptr))) { \
251 case 1: \
252 __put_user_x(1, __pu_val, ptr, __ret_pu); \
253 break; \
254 case 2: \
255 __put_user_x(2, __pu_val, ptr, __ret_pu); \
256 break; \
257 case 4: \
258 __put_user_x(4, __pu_val, ptr, __ret_pu); \
259 break; \
260 case 8: \
261 __put_user_x8(__pu_val, ptr, __ret_pu); \
262 break; \
263 default: \
264 __put_user_x(X, __pu_val, ptr, __ret_pu); \
265 break; \
266 } \
267 __builtin_expect(__ret_pu, 0); \
268 })
269
270 #define __put_user_size(x, ptr, size, retval, errret) \
271 do { \
272 retval = 0; \
273 __chk_user_ptr(ptr); \
274 switch (size) { \
275 case 1: \
276 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
277 break; \
278 case 2: \
279 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
280 break; \
281 case 4: \
282 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
283 break; \
284 case 8: \
285 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
286 errret); \
287 break; \
288 default: \
289 __put_user_bad(); \
290 } \
291 } while (0)
292
293 /*
294 * This doesn't do __uaccess_begin/end - the exception handling
295 * around it must do that.
296 */
297 #define __put_user_size_ex(x, ptr, size) \
298 do { \
299 __chk_user_ptr(ptr); \
300 switch (size) { \
301 case 1: \
302 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
303 break; \
304 case 2: \
305 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
306 break; \
307 case 4: \
308 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
309 break; \
310 case 8: \
311 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
312 break; \
313 default: \
314 __put_user_bad(); \
315 } \
316 } while (0)
317
318 #ifdef CONFIG_X86_32
319 #define __get_user_asm_u64(x, ptr, retval, errret) \
320 ({ \
321 __typeof__(ptr) __ptr = (ptr); \
322 asm volatile("\n" \
323 "1: movl %2,%%eax\n" \
324 "2: movl %3,%%edx\n" \
325 "3:\n" \
326 ".section .fixup,\"ax\"\n" \
327 "4: mov %4,%0\n" \
328 " xorl %%eax,%%eax\n" \
329 " xorl %%edx,%%edx\n" \
330 " jmp 3b\n" \
331 ".previous\n" \
332 _ASM_EXTABLE(1b, 4b) \
333 _ASM_EXTABLE(2b, 4b) \
334 : "=r" (retval), "=&A"(x) \
335 : "m" (__m(__ptr)), "m" __m(((u32 *)(__ptr)) + 1), \
336 "i" (errret), "0" (retval)); \
337 })
338
339 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
340 #else
341 #define __get_user_asm_u64(x, ptr, retval, errret) \
342 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
343 #define __get_user_asm_ex_u64(x, ptr) \
344 __get_user_asm_ex(x, ptr, "q", "", "=r")
345 #endif
346
347 #define __get_user_size(x, ptr, size, retval, errret) \
348 do { \
349 retval = 0; \
350 __chk_user_ptr(ptr); \
351 switch (size) { \
352 case 1: \
353 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
354 break; \
355 case 2: \
356 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
357 break; \
358 case 4: \
359 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
360 break; \
361 case 8: \
362 __get_user_asm_u64(x, ptr, retval, errret); \
363 break; \
364 default: \
365 (x) = __get_user_bad(); \
366 } \
367 } while (0)
368
369 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
370 asm volatile("\n" \
371 "1: mov"itype" %2,%"rtype"1\n" \
372 "2:\n" \
373 ".section .fixup,\"ax\"\n" \
374 "3: mov %3,%0\n" \
375 " xor"itype" %"rtype"1,%"rtype"1\n" \
376 " jmp 2b\n" \
377 ".previous\n" \
378 _ASM_EXTABLE(1b, 3b) \
379 : "=r" (err), ltype(x) \
380 : "m" (__m(addr)), "i" (errret), "0" (err))
381
382 #define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \
383 asm volatile("\n" \
384 "1: mov"itype" %2,%"rtype"1\n" \
385 "2:\n" \
386 ".section .fixup,\"ax\"\n" \
387 "3: mov %3,%0\n" \
388 " jmp 2b\n" \
389 ".previous\n" \
390 _ASM_EXTABLE(1b, 3b) \
391 : "=r" (err), ltype(x) \
392 : "m" (__m(addr)), "i" (errret), "0" (err))
393
394 /*
395 * This doesn't do __uaccess_begin/end - the exception handling
396 * around it must do that.
397 */
398 #define __get_user_size_ex(x, ptr, size) \
399 do { \
400 __chk_user_ptr(ptr); \
401 switch (size) { \
402 case 1: \
403 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
404 break; \
405 case 2: \
406 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
407 break; \
408 case 4: \
409 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
410 break; \
411 case 8: \
412 __get_user_asm_ex_u64(x, ptr); \
413 break; \
414 default: \
415 (x) = __get_user_bad(); \
416 } \
417 } while (0)
418
419 #define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
420 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
421 "2:\n" \
422 ".section .fixup,\"ax\"\n" \
423 "3:xor"itype" %"rtype"0,%"rtype"0\n" \
424 " jmp 2b\n" \
425 ".previous\n" \
426 _ASM_EXTABLE_EX(1b, 3b) \
427 : ltype(x) : "m" (__m(addr)))
428
429 #define __put_user_nocheck(x, ptr, size) \
430 ({ \
431 int __pu_err; \
432 __uaccess_begin(); \
433 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
434 __uaccess_end(); \
435 __builtin_expect(__pu_err, 0); \
436 })
437
438 #define __get_user_nocheck(x, ptr, size) \
439 ({ \
440 int __gu_err; \
441 __inttype(*(ptr)) __gu_val; \
442 __uaccess_begin(); \
443 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
444 __uaccess_end(); \
445 (x) = (__force __typeof__(*(ptr)))__gu_val; \
446 __builtin_expect(__gu_err, 0); \
447 })
448
449 /* FIXME: this hack is definitely wrong -AK */
450 struct __large_struct { unsigned long buf[100]; };
451 #define __m(x) (*(struct __large_struct __user *)(x))
452
453 /*
454 * Tell gcc we read from memory instead of writing: this is because
455 * we do not write to any memory gcc knows about, so there are no
456 * aliasing issues.
457 */
458 #define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
459 asm volatile("\n" \
460 "1: mov"itype" %"rtype"1,%2\n" \
461 "2:\n" \
462 ".section .fixup,\"ax\"\n" \
463 "3: mov %3,%0\n" \
464 " jmp 2b\n" \
465 ".previous\n" \
466 _ASM_EXTABLE(1b, 3b) \
467 : "=r"(err) \
468 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
469
470 #define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
471 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
472 "2:\n" \
473 _ASM_EXTABLE_EX(1b, 2b) \
474 : : ltype(x), "m" (__m(addr)))
475
476 /*
477 * uaccess_try and catch
478 */
479 #define uaccess_try do { \
480 current->thread.uaccess_err = 0; \
481 __uaccess_begin(); \
482 barrier();
483
484 #define uaccess_catch(err) \
485 __uaccess_end(); \
486 (err) |= (current->thread.uaccess_err ? -EFAULT : 0); \
487 } while (0)
488
489 /**
490 * __get_user: - Get a simple variable from user space, with less checking.
491 * @x: Variable to store result.
492 * @ptr: Source address, in user space.
493 *
494 * Context: User context only. This function may sleep if pagefaults are
495 * enabled.
496 *
497 * This macro copies a single simple variable from user space to kernel
498 * space. It supports simple types like char and int, but not larger
499 * data types like structures or arrays.
500 *
501 * @ptr must have pointer-to-simple-variable type, and the result of
502 * dereferencing @ptr must be assignable to @x without a cast.
503 *
504 * Caller must check the pointer with access_ok() before calling this
505 * function.
506 *
507 * Returns zero on success, or -EFAULT on error.
508 * On error, the variable @x is set to zero.
509 */
510
511 #define __get_user(x, ptr) \
512 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
513
514 /**
515 * __put_user: - Write a simple value into user space, with less checking.
516 * @x: Value to copy to user space.
517 * @ptr: Destination address, in user space.
518 *
519 * Context: User context only. This function may sleep if pagefaults are
520 * enabled.
521 *
522 * This macro copies a single simple value from kernel space to user
523 * space. It supports simple types like char and int, but not larger
524 * data types like structures or arrays.
525 *
526 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
527 * to the result of dereferencing @ptr.
528 *
529 * Caller must check the pointer with access_ok() before calling this
530 * function.
531 *
532 * Returns zero on success, or -EFAULT on error.
533 */
534
535 #define __put_user(x, ptr) \
536 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
537
538 /*
539 * {get|put}_user_try and catch
540 *
541 * get_user_try {
542 * get_user_ex(...);
543 * } get_user_catch(err)
544 */
545 #define get_user_try uaccess_try
546 #define get_user_catch(err) uaccess_catch(err)
547
548 #define get_user_ex(x, ptr) do { \
549 unsigned long __gue_val; \
550 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
551 (x) = (__force __typeof__(*(ptr)))__gue_val; \
552 } while (0)
553
554 #define put_user_try uaccess_try
555 #define put_user_catch(err) uaccess_catch(err)
556
557 #define put_user_ex(x, ptr) \
558 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
559
560 extern unsigned long
561 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
562 extern __must_check long
563 strncpy_from_user(char *dst, const char __user *src, long count);
564
565 extern __must_check long strnlen_user(const char __user *str, long n);
566
567 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
568 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
569
570 extern void __cmpxchg_wrong_size(void)
571 __compiletime_error("Bad argument size for cmpxchg");
572
573 #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \
574 ({ \
575 int __ret = 0; \
576 __typeof__(ptr) __uval = (uval); \
577 __typeof__(*(ptr)) __old = (old); \
578 __typeof__(*(ptr)) __new = (new); \
579 __uaccess_begin(); \
580 switch (size) { \
581 case 1: \
582 { \
583 asm volatile("\n" \
584 "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \
585 "2:\n" \
586 "\t.section .fixup, \"ax\"\n" \
587 "3:\tmov %3, %0\n" \
588 "\tjmp 2b\n" \
589 "\t.previous\n" \
590 _ASM_EXTABLE(1b, 3b) \
591 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
592 : "i" (-EFAULT), "q" (__new), "1" (__old) \
593 : "memory" \
594 ); \
595 break; \
596 } \
597 case 2: \
598 { \
599 asm volatile("\n" \
600 "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \
601 "2:\n" \
602 "\t.section .fixup, \"ax\"\n" \
603 "3:\tmov %3, %0\n" \
604 "\tjmp 2b\n" \
605 "\t.previous\n" \
606 _ASM_EXTABLE(1b, 3b) \
607 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
608 : "i" (-EFAULT), "r" (__new), "1" (__old) \
609 : "memory" \
610 ); \
611 break; \
612 } \
613 case 4: \
614 { \
615 asm volatile("\n" \
616 "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \
617 "2:\n" \
618 "\t.section .fixup, \"ax\"\n" \
619 "3:\tmov %3, %0\n" \
620 "\tjmp 2b\n" \
621 "\t.previous\n" \
622 _ASM_EXTABLE(1b, 3b) \
623 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
624 : "i" (-EFAULT), "r" (__new), "1" (__old) \
625 : "memory" \
626 ); \
627 break; \
628 } \
629 case 8: \
630 { \
631 if (!IS_ENABLED(CONFIG_X86_64)) \
632 __cmpxchg_wrong_size(); \
633 \
634 asm volatile("\n" \
635 "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \
636 "2:\n" \
637 "\t.section .fixup, \"ax\"\n" \
638 "3:\tmov %3, %0\n" \
639 "\tjmp 2b\n" \
640 "\t.previous\n" \
641 _ASM_EXTABLE(1b, 3b) \
642 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
643 : "i" (-EFAULT), "r" (__new), "1" (__old) \
644 : "memory" \
645 ); \
646 break; \
647 } \
648 default: \
649 __cmpxchg_wrong_size(); \
650 } \
651 __uaccess_end(); \
652 *__uval = __old; \
653 __ret; \
654 })
655
656 #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \
657 ({ \
658 access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \
659 __user_atomic_cmpxchg_inatomic((uval), (ptr), \
660 (old), (new), sizeof(*(ptr))) : \
661 -EFAULT; \
662 })
663
664 /*
665 * movsl can be slow when source and dest are not both 8-byte aligned
666 */
667 #ifdef CONFIG_X86_INTEL_USERCOPY
668 extern struct movsl_mask {
669 int mask;
670 } ____cacheline_aligned_in_smp movsl_mask;
671 #endif
672
673 #define ARCH_HAS_NOCACHE_UACCESS 1
674
675 #ifdef CONFIG_X86_32
676 # include <asm/uaccess_32.h>
677 #else
678 # include <asm/uaccess_64.h>
679 #endif
680
681 /*
682 * We rely on the nested NMI work to allow atomic faults from the NMI path; the
683 * nested NMI paths are careful to preserve CR2.
684 *
685 * Caller must use pagefault_enable/disable, or run in interrupt context,
686 * and also do a uaccess_ok() check
687 */
688 #define __copy_from_user_nmi __copy_from_user_inatomic
689
690 /*
691 * The "unsafe" user accesses aren't really "unsafe", but the naming
692 * is a big fat warning: you have to not only do the access_ok()
693 * checking before using them, but you have to surround them with the
694 * user_access_begin/end() pair.
695 */
696 #define user_access_begin() __uaccess_begin()
697 #define user_access_end() __uaccess_end()
698
699 #define unsafe_put_user(x, ptr, err_label) \
700 do { \
701 int __pu_err; \
702 __typeof__(*(ptr)) __pu_val = (x); \
703 __put_user_size(__pu_val, (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \
704 if (unlikely(__pu_err)) goto err_label; \
705 } while (0)
706
707 #define unsafe_get_user(x, ptr, err_label) \
708 do { \
709 int __gu_err; \
710 __inttype(*(ptr)) __gu_val; \
711 __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \
712 (x) = (__force __typeof__(*(ptr)))__gu_val; \
713 if (unlikely(__gu_err)) goto err_label; \
714 } while (0)
715
716 #endif /* _ASM_X86_UACCESS_H */
717
Linux with added FPC-III support