2 * User address space access functions.
3 * The non inlined parts of asm-i386/uaccess.h are here.
5 * Copyright 1997 Andi Kleen <ak@muc.de>
6 * Copyright 1997 Linus Torvalds
9 #include <linux/highmem.h>
10 #include <linux/blkdev.h>
11 #include <linux/module.h>
12 #include <linux/backing-dev.h>
13 #include <linux/interrupt.h>
14 #include <asm/uaccess.h>
18 #ifdef CONFIG_X86_INTEL_USERCOPY
20 * Alignment at which movsl is preferred for bulk memory copies.
22 struct movsl_mask movsl_mask __read_mostly
;
25 static inline int __movsl_is_ok(unsigned long a1
, unsigned long a2
, unsigned long n
)
27 #ifdef CONFIG_X86_INTEL_USERCOPY
28 if (n
>= 64 && ((a1
^ a2
) & movsl_mask
.mask
))
33 #define movsl_is_ok(a1, a2, n) \
34 __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
40 #define __do_clear_user(addr,size) \
44 __asm__ __volatile__( \
50 ".section .fixup,\"ax\"\n" \
51 "3: lea 0(%2,%0,4),%0\n" \
56 : "=&c"(size), "=&D" (__d0) \
57 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \
61 * clear_user: - Zero a block of memory in user space.
62 * @to: Destination address, in user space.
63 * @n: Number of bytes to zero.
65 * Zero a block of memory in user space.
67 * Returns number of bytes that could not be cleared.
68 * On success, this will be zero.
71 clear_user(void __user
*to
, unsigned long n
)
74 if (access_ok(VERIFY_WRITE
, to
, n
))
75 __do_clear_user(to
, n
);
78 EXPORT_SYMBOL(clear_user
);
81 * __clear_user: - Zero a block of memory in user space, with less checking.
82 * @to: Destination address, in user space.
83 * @n: Number of bytes to zero.
85 * Zero a block of memory in user space. Caller must check
86 * the specified block with access_ok() before calling this function.
88 * Returns number of bytes that could not be cleared.
89 * On success, this will be zero.
92 __clear_user(void __user
*to
, unsigned long n
)
94 __do_clear_user(to
, n
);
97 EXPORT_SYMBOL(__clear_user
);
99 #ifdef CONFIG_X86_INTEL_USERCOPY
101 __copy_user_intel(void __user
*to
, const void *from
, unsigned long size
)
104 __asm__
__volatile__(
106 "1: movl 32(%4), %%eax\n"
109 "2: movl 64(%4), %%eax\n"
111 "3: movl 0(%4), %%eax\n"
112 "4: movl 4(%4), %%edx\n"
113 "5: movl %%eax, 0(%3)\n"
114 "6: movl %%edx, 4(%3)\n"
115 "7: movl 8(%4), %%eax\n"
116 "8: movl 12(%4),%%edx\n"
117 "9: movl %%eax, 8(%3)\n"
118 "10: movl %%edx, 12(%3)\n"
119 "11: movl 16(%4), %%eax\n"
120 "12: movl 20(%4), %%edx\n"
121 "13: movl %%eax, 16(%3)\n"
122 "14: movl %%edx, 20(%3)\n"
123 "15: movl 24(%4), %%eax\n"
124 "16: movl 28(%4), %%edx\n"
125 "17: movl %%eax, 24(%3)\n"
126 "18: movl %%edx, 28(%3)\n"
127 "19: movl 32(%4), %%eax\n"
128 "20: movl 36(%4), %%edx\n"
129 "21: movl %%eax, 32(%3)\n"
130 "22: movl %%edx, 36(%3)\n"
131 "23: movl 40(%4), %%eax\n"
132 "24: movl 44(%4), %%edx\n"
133 "25: movl %%eax, 40(%3)\n"
134 "26: movl %%edx, 44(%3)\n"
135 "27: movl 48(%4), %%eax\n"
136 "28: movl 52(%4), %%edx\n"
137 "29: movl %%eax, 48(%3)\n"
138 "30: movl %%edx, 52(%3)\n"
139 "31: movl 56(%4), %%eax\n"
140 "32: movl 60(%4), %%edx\n"
141 "33: movl %%eax, 56(%3)\n"
142 "34: movl %%edx, 60(%3)\n"
148 "35: movl %0, %%eax\n"
153 "36: movl %%eax, %0\n"
156 ".section .fixup,\"ax\"\n"
157 "101: lea 0(%%eax,%0,4),%0\n"
160 _ASM_EXTABLE(1b
,100b
)
161 _ASM_EXTABLE(2b
,100b
)
162 _ASM_EXTABLE(3b
,100b
)
163 _ASM_EXTABLE(4b
,100b
)
164 _ASM_EXTABLE(5b
,100b
)
165 _ASM_EXTABLE(6b
,100b
)
166 _ASM_EXTABLE(7b
,100b
)
167 _ASM_EXTABLE(8b
,100b
)
168 _ASM_EXTABLE(9b
,100b
)
169 _ASM_EXTABLE(10b
,100b
)
170 _ASM_EXTABLE(11b
,100b
)
171 _ASM_EXTABLE(12b
,100b
)
172 _ASM_EXTABLE(13b
,100b
)
173 _ASM_EXTABLE(14b
,100b
)
174 _ASM_EXTABLE(15b
,100b
)
175 _ASM_EXTABLE(16b
,100b
)
176 _ASM_EXTABLE(17b
,100b
)
177 _ASM_EXTABLE(18b
,100b
)
178 _ASM_EXTABLE(19b
,100b
)
179 _ASM_EXTABLE(20b
,100b
)
180 _ASM_EXTABLE(21b
,100b
)
181 _ASM_EXTABLE(22b
,100b
)
182 _ASM_EXTABLE(23b
,100b
)
183 _ASM_EXTABLE(24b
,100b
)
184 _ASM_EXTABLE(25b
,100b
)
185 _ASM_EXTABLE(26b
,100b
)
186 _ASM_EXTABLE(27b
,100b
)
187 _ASM_EXTABLE(28b
,100b
)
188 _ASM_EXTABLE(29b
,100b
)
189 _ASM_EXTABLE(30b
,100b
)
190 _ASM_EXTABLE(31b
,100b
)
191 _ASM_EXTABLE(32b
,100b
)
192 _ASM_EXTABLE(33b
,100b
)
193 _ASM_EXTABLE(34b
,100b
)
194 _ASM_EXTABLE(35b
,100b
)
195 _ASM_EXTABLE(36b
,100b
)
196 _ASM_EXTABLE(37b
,100b
)
197 _ASM_EXTABLE(99b
,101b
)
198 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
199 : "1"(to
), "2"(from
), "0"(size
)
200 : "eax", "edx", "memory");
205 __copy_user_zeroing_intel(void *to
, const void __user
*from
, unsigned long size
)
208 __asm__
__volatile__(
210 "0: movl 32(%4), %%eax\n"
213 "1: movl 64(%4), %%eax\n"
215 "2: movl 0(%4), %%eax\n"
216 "21: movl 4(%4), %%edx\n"
217 " movl %%eax, 0(%3)\n"
218 " movl %%edx, 4(%3)\n"
219 "3: movl 8(%4), %%eax\n"
220 "31: movl 12(%4),%%edx\n"
221 " movl %%eax, 8(%3)\n"
222 " movl %%edx, 12(%3)\n"
223 "4: movl 16(%4), %%eax\n"
224 "41: movl 20(%4), %%edx\n"
225 " movl %%eax, 16(%3)\n"
226 " movl %%edx, 20(%3)\n"
227 "10: movl 24(%4), %%eax\n"
228 "51: movl 28(%4), %%edx\n"
229 " movl %%eax, 24(%3)\n"
230 " movl %%edx, 28(%3)\n"
231 "11: movl 32(%4), %%eax\n"
232 "61: movl 36(%4), %%edx\n"
233 " movl %%eax, 32(%3)\n"
234 " movl %%edx, 36(%3)\n"
235 "12: movl 40(%4), %%eax\n"
236 "71: movl 44(%4), %%edx\n"
237 " movl %%eax, 40(%3)\n"
238 " movl %%edx, 44(%3)\n"
239 "13: movl 48(%4), %%eax\n"
240 "81: movl 52(%4), %%edx\n"
241 " movl %%eax, 48(%3)\n"
242 " movl %%edx, 52(%3)\n"
243 "14: movl 56(%4), %%eax\n"
244 "91: movl 60(%4), %%edx\n"
245 " movl %%eax, 56(%3)\n"
246 " movl %%edx, 60(%3)\n"
252 "5: movl %0, %%eax\n"
260 ".section .fixup,\"ax\"\n"
261 "9: lea 0(%%eax,%0,4),%0\n"
264 " xorl %%eax,%%eax\n"
273 _ASM_EXTABLE(21b
,16b
)
275 _ASM_EXTABLE(31b
,16b
)
277 _ASM_EXTABLE(41b
,16b
)
278 _ASM_EXTABLE(10b
,16b
)
279 _ASM_EXTABLE(51b
,16b
)
280 _ASM_EXTABLE(11b
,16b
)
281 _ASM_EXTABLE(61b
,16b
)
282 _ASM_EXTABLE(12b
,16b
)
283 _ASM_EXTABLE(71b
,16b
)
284 _ASM_EXTABLE(13b
,16b
)
285 _ASM_EXTABLE(81b
,16b
)
286 _ASM_EXTABLE(14b
,16b
)
287 _ASM_EXTABLE(91b
,16b
)
290 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
291 : "1"(to
), "2"(from
), "0"(size
)
292 : "eax", "edx", "memory");
297 * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware.
298 * hyoshiok@miraclelinux.com
301 static unsigned long __copy_user_zeroing_intel_nocache(void *to
,
302 const void __user
*from
, unsigned long size
)
306 __asm__
__volatile__(
308 "0: movl 32(%4), %%eax\n"
311 "1: movl 64(%4), %%eax\n"
313 "2: movl 0(%4), %%eax\n"
314 "21: movl 4(%4), %%edx\n"
315 " movnti %%eax, 0(%3)\n"
316 " movnti %%edx, 4(%3)\n"
317 "3: movl 8(%4), %%eax\n"
318 "31: movl 12(%4),%%edx\n"
319 " movnti %%eax, 8(%3)\n"
320 " movnti %%edx, 12(%3)\n"
321 "4: movl 16(%4), %%eax\n"
322 "41: movl 20(%4), %%edx\n"
323 " movnti %%eax, 16(%3)\n"
324 " movnti %%edx, 20(%3)\n"
325 "10: movl 24(%4), %%eax\n"
326 "51: movl 28(%4), %%edx\n"
327 " movnti %%eax, 24(%3)\n"
328 " movnti %%edx, 28(%3)\n"
329 "11: movl 32(%4), %%eax\n"
330 "61: movl 36(%4), %%edx\n"
331 " movnti %%eax, 32(%3)\n"
332 " movnti %%edx, 36(%3)\n"
333 "12: movl 40(%4), %%eax\n"
334 "71: movl 44(%4), %%edx\n"
335 " movnti %%eax, 40(%3)\n"
336 " movnti %%edx, 44(%3)\n"
337 "13: movl 48(%4), %%eax\n"
338 "81: movl 52(%4), %%edx\n"
339 " movnti %%eax, 48(%3)\n"
340 " movnti %%edx, 52(%3)\n"
341 "14: movl 56(%4), %%eax\n"
342 "91: movl 60(%4), %%edx\n"
343 " movnti %%eax, 56(%3)\n"
344 " movnti %%edx, 60(%3)\n"
351 "5: movl %0, %%eax\n"
359 ".section .fixup,\"ax\"\n"
360 "9: lea 0(%%eax,%0,4),%0\n"
363 " xorl %%eax,%%eax\n"
372 _ASM_EXTABLE(21b
,16b
)
374 _ASM_EXTABLE(31b
,16b
)
376 _ASM_EXTABLE(41b
,16b
)
377 _ASM_EXTABLE(10b
,16b
)
378 _ASM_EXTABLE(51b
,16b
)
379 _ASM_EXTABLE(11b
,16b
)
380 _ASM_EXTABLE(61b
,16b
)
381 _ASM_EXTABLE(12b
,16b
)
382 _ASM_EXTABLE(71b
,16b
)
383 _ASM_EXTABLE(13b
,16b
)
384 _ASM_EXTABLE(81b
,16b
)
385 _ASM_EXTABLE(14b
,16b
)
386 _ASM_EXTABLE(91b
,16b
)
389 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
390 : "1"(to
), "2"(from
), "0"(size
)
391 : "eax", "edx", "memory");
395 static unsigned long __copy_user_intel_nocache(void *to
,
396 const void __user
*from
, unsigned long size
)
400 __asm__
__volatile__(
402 "0: movl 32(%4), %%eax\n"
405 "1: movl 64(%4), %%eax\n"
407 "2: movl 0(%4), %%eax\n"
408 "21: movl 4(%4), %%edx\n"
409 " movnti %%eax, 0(%3)\n"
410 " movnti %%edx, 4(%3)\n"
411 "3: movl 8(%4), %%eax\n"
412 "31: movl 12(%4),%%edx\n"
413 " movnti %%eax, 8(%3)\n"
414 " movnti %%edx, 12(%3)\n"
415 "4: movl 16(%4), %%eax\n"
416 "41: movl 20(%4), %%edx\n"
417 " movnti %%eax, 16(%3)\n"
418 " movnti %%edx, 20(%3)\n"
419 "10: movl 24(%4), %%eax\n"
420 "51: movl 28(%4), %%edx\n"
421 " movnti %%eax, 24(%3)\n"
422 " movnti %%edx, 28(%3)\n"
423 "11: movl 32(%4), %%eax\n"
424 "61: movl 36(%4), %%edx\n"
425 " movnti %%eax, 32(%3)\n"
426 " movnti %%edx, 36(%3)\n"
427 "12: movl 40(%4), %%eax\n"
428 "71: movl 44(%4), %%edx\n"
429 " movnti %%eax, 40(%3)\n"
430 " movnti %%edx, 44(%3)\n"
431 "13: movl 48(%4), %%eax\n"
432 "81: movl 52(%4), %%edx\n"
433 " movnti %%eax, 48(%3)\n"
434 " movnti %%edx, 52(%3)\n"
435 "14: movl 56(%4), %%eax\n"
436 "91: movl 60(%4), %%edx\n"
437 " movnti %%eax, 56(%3)\n"
438 " movnti %%edx, 60(%3)\n"
445 "5: movl %0, %%eax\n"
453 ".section .fixup,\"ax\"\n"
454 "9: lea 0(%%eax,%0,4),%0\n"
460 _ASM_EXTABLE(21b
,16b
)
462 _ASM_EXTABLE(31b
,16b
)
464 _ASM_EXTABLE(41b
,16b
)
465 _ASM_EXTABLE(10b
,16b
)
466 _ASM_EXTABLE(51b
,16b
)
467 _ASM_EXTABLE(11b
,16b
)
468 _ASM_EXTABLE(61b
,16b
)
469 _ASM_EXTABLE(12b
,16b
)
470 _ASM_EXTABLE(71b
,16b
)
471 _ASM_EXTABLE(13b
,16b
)
472 _ASM_EXTABLE(81b
,16b
)
473 _ASM_EXTABLE(14b
,16b
)
474 _ASM_EXTABLE(91b
,16b
)
477 : "=&c"(size
), "=&D" (d0
), "=&S" (d1
)
478 : "1"(to
), "2"(from
), "0"(size
)
479 : "eax", "edx", "memory");
486 * Leave these declared but undefined. They should not be any references to
489 unsigned long __copy_user_zeroing_intel(void *to
, const void __user
*from
,
491 unsigned long __copy_user_intel(void __user
*to
, const void *from
,
493 unsigned long __copy_user_zeroing_intel_nocache(void *to
,
494 const void __user
*from
, unsigned long size
);
495 #endif /* CONFIG_X86_INTEL_USERCOPY */
497 /* Generic arbitrary sized copy. */
498 #define __copy_user(to, from, size) \
500 int __d0, __d1, __d2; \
501 __asm__ __volatile__( \
517 ".section .fixup,\"ax\"\n" \
520 "3: lea 0(%3,%0,4),%0\n" \
523 _ASM_EXTABLE(4b,5b) \
524 _ASM_EXTABLE(0b,3b) \
525 _ASM_EXTABLE(1b,2b) \
526 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
527 : "3"(size), "0"(size), "1"(to), "2"(from) \
531 #define __copy_user_zeroing(to, from, size) \
533 int __d0, __d1, __d2; \
534 __asm__ __volatile__( \
550 ".section .fixup,\"ax\"\n" \
553 "3: lea 0(%3,%0,4),%0\n" \
556 " xorl %%eax,%%eax\n" \
562 _ASM_EXTABLE(4b,5b) \
563 _ASM_EXTABLE(0b,3b) \
564 _ASM_EXTABLE(1b,6b) \
565 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \
566 : "3"(size), "0"(size), "1"(to), "2"(from) \
570 unsigned long __copy_to_user_ll(void __user
*to
, const void *from
,
574 if (movsl_is_ok(to
, from
, n
))
575 __copy_user(to
, from
, n
);
577 n
= __copy_user_intel(to
, from
, n
);
581 EXPORT_SYMBOL(__copy_to_user_ll
);
583 unsigned long __copy_from_user_ll(void *to
, const void __user
*from
,
587 if (movsl_is_ok(to
, from
, n
))
588 __copy_user_zeroing(to
, from
, n
);
590 n
= __copy_user_zeroing_intel(to
, from
, n
);
594 EXPORT_SYMBOL(__copy_from_user_ll
);
596 unsigned long __copy_from_user_ll_nozero(void *to
, const void __user
*from
,
600 if (movsl_is_ok(to
, from
, n
))
601 __copy_user(to
, from
, n
);
603 n
= __copy_user_intel((void __user
*)to
,
604 (const void *)from
, n
);
608 EXPORT_SYMBOL(__copy_from_user_ll_nozero
);
610 unsigned long __copy_from_user_ll_nocache(void *to
, const void __user
*from
,
614 #ifdef CONFIG_X86_INTEL_USERCOPY
615 if (n
> 64 && cpu_has_xmm2
)
616 n
= __copy_user_zeroing_intel_nocache(to
, from
, n
);
618 __copy_user_zeroing(to
, from
, n
);
620 __copy_user_zeroing(to
, from
, n
);
625 EXPORT_SYMBOL(__copy_from_user_ll_nocache
);
627 unsigned long __copy_from_user_ll_nocache_nozero(void *to
, const void __user
*from
,
631 #ifdef CONFIG_X86_INTEL_USERCOPY
632 if (n
> 64 && cpu_has_xmm2
)
633 n
= __copy_user_intel_nocache(to
, from
, n
);
635 __copy_user(to
, from
, n
);
637 __copy_user(to
, from
, n
);
642 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero
);
645 * copy_to_user: - Copy a block of data into user space.
646 * @to: Destination address, in user space.
647 * @from: Source address, in kernel space.
648 * @n: Number of bytes to copy.
650 * Context: User context only. This function may sleep if pagefaults are
653 * Copy data from kernel space to user space.
655 * Returns number of bytes that could not be copied.
656 * On success, this will be zero.
658 unsigned long _copy_to_user(void __user
*to
, const void *from
, unsigned n
)
660 if (access_ok(VERIFY_WRITE
, to
, n
))
661 n
= __copy_to_user(to
, from
, n
);
664 EXPORT_SYMBOL(_copy_to_user
);
667 * copy_from_user: - Copy a block of data from user space.
668 * @to: Destination address, in kernel space.
669 * @from: Source address, in user space.
670 * @n: Number of bytes to copy.
672 * Context: User context only. This function may sleep if pagefaults are
675 * Copy data from user space to kernel space.
677 * Returns number of bytes that could not be copied.
678 * On success, this will be zero.
680 * If some data could not be copied, this function will pad the copied
681 * data to the requested size using zero bytes.
683 unsigned long _copy_from_user(void *to
, const void __user
*from
, unsigned n
)
685 if (access_ok(VERIFY_READ
, from
, n
))
686 n
= __copy_from_user(to
, from
, n
);
691 EXPORT_SYMBOL(_copy_from_user
);