2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 * -__clear_user( ) called multiple times during elf load was byte loop
10 * converted to do as much word clear as possible.
13 * -Hand crafted constant propagation for "constant" copy sizes
14 * -stock kernel shrunk by 33K at -O3
17 * -Added option to (UN)inline copy_(to|from)_user to reduce code sz
18 * -kernel shrunk by 200K even at -O3 (gcc 4.2.1)
19 * -Enabled when doing -Os
21 * Amit Bhor, Sameer Dhavale: Codito Technologies 2004
24 #ifndef _ASM_ARC_UACCESS_H
25 #define _ASM_ARC_UACCESS_H
27 #include <linux/string.h> /* for generic string functions */
30 #define __kernel_ok (uaccess_kernel())
33 * Algorithmically, for __user_ok() we want do:
34 * (start < TASK_SIZE) && (start+len < TASK_SIZE)
35 * where TASK_SIZE could either be retrieved from thread_info->addr_limit or
36 * emitted directly in code.
38 * This can however be rewritten as follows:
39 * (len <= TASK_SIZE) && (start+len < TASK_SIZE)
41 * Because it essentially checks if buffer end is within limit and @len is
42 * non-ngeative, which implies that buffer start will be within limit too.
44 * The reason for rewriting being, for majority of cases, @len is generally
45 * compile time constant, causing first sub-expression to be compile time
48 * The second part would generate weird large LIMMs e.g. (0x6000_0000 - 0x10),
49 * so we check for TASK_SIZE using get_fs() since the addr_limit load from mem
50 * would already have been done at this call site for __kernel_ok()
53 #define __user_ok(addr, sz) (((sz) <= TASK_SIZE) && \
54 ((addr) <= (get_fs() - (sz))))
55 #define __access_ok(addr, sz) (unlikely(__kernel_ok) || \
56 likely(__user_ok((addr), (sz))))
58 /*********** Single byte/hword/word copies ******************/
60 #define __get_user_fn(sz, u, k) \
62 long __ret = 0; /* success by default */ \
64 case 1: __arc_get_user_one(*(k), u, "ldb", __ret); break; \
65 case 2: __arc_get_user_one(*(k), u, "ldw", __ret); break; \
66 case 4: __arc_get_user_one(*(k), u, "ld", __ret); break; \
67 case 8: __arc_get_user_one_64(*(k), u, __ret); break; \
73 * Returns 0 on success, -EFAULT if not.
74 * @ret already contains 0 - given that errors will be less likely
75 * (hence +r asm constraint below).
76 * In case of error, fixup code will make it -EFAULT
78 #define __arc_get_user_one(dst, src, op, ret) \
79 __asm__ __volatile__( \
82 " .section .fixup, \"ax\"\n" \
84 "3: # return -EFAULT\n" \
86 " # zero out dst ptr\n" \
90 " .section __ex_table, \"a\"\n" \
95 : "+r" (ret), "=r" (dst) \
96 : "r" (src), "ir" (-EFAULT))
98 #define __arc_get_user_one_64(dst, src, ret) \
99 __asm__ __volatile__( \
101 "4: ld %R1,[%2, 4]\n" \
103 " .section .fixup, \"ax\"\n" \
105 "3: # return -EFAULT\n" \
107 " # zero out dst ptr\n" \
112 " .section __ex_table, \"a\"\n" \
118 : "+r" (ret), "=r" (dst) \
119 : "r" (src), "ir" (-EFAULT))
121 #define __put_user_fn(sz, u, k) \
123 long __ret = 0; /* success by default */ \
125 case 1: __arc_put_user_one(*(k), u, "stb", __ret); break; \
126 case 2: __arc_put_user_one(*(k), u, "stw", __ret); break; \
127 case 4: __arc_put_user_one(*(k), u, "st", __ret); break; \
128 case 8: __arc_put_user_one_64(*(k), u, __ret); break; \
133 #define __arc_put_user_one(src, dst, op, ret) \
134 __asm__ __volatile__( \
135 "1: "op" %1,[%2]\n" \
137 " .section .fixup, \"ax\"\n" \
142 " .section __ex_table, \"a\"\n" \
148 : "r" (src), "r" (dst), "ir" (-EFAULT))
150 #define __arc_put_user_one_64(src, dst, ret) \
151 __asm__ __volatile__( \
153 "4: st %R1,[%2, 4]\n" \
155 " .section .fixup, \"ax\"\n" \
160 " .section __ex_table, \"a\"\n" \
167 : "r" (src), "r" (dst), "ir" (-EFAULT))
170 static inline unsigned long
171 raw_copy_from_user(void *to
, const void __user
*from
, unsigned long n
)
175 unsigned long tmp1
, tmp2
, tmp3
, tmp4
;
176 unsigned long orig_n
= n
;
182 if (((unsigned long)to
& 0x3) || ((unsigned long)from
& 0x3)) {
186 __asm__
__volatile__ (
187 " mov.f lp_count, %0 \n"
189 "1: ldb.ab %1, [%3, 1] \n"
190 " stb.ab %1, [%2, 1] \n"
193 " .section .fixup, \"ax\" \n"
197 " .section __ex_table, \"a\" \n"
204 * Note as an '&' earlyclobber operand to make sure the
205 * temporary register inside the loop is not the same as
208 "=&r" (tmp
), "+r" (to
), "+r" (from
)
210 : "lp_count", "lp_start", "lp_end", "memory");
216 * Hand-crafted constant propagation to reduce code sz of the
217 * laddered copy 16x,8,4,2,1
219 if (__builtin_constant_p(orig_n
)) {
223 orig_n
= orig_n
% 16;
225 __asm__
__volatile__(
226 " lsr lp_count, %7,4 \n"
228 "1: ld.ab %3, [%2, 4] \n"
229 "11: ld.ab %4, [%2, 4] \n"
230 "12: ld.ab %5, [%2, 4] \n"
231 "13: ld.ab %6, [%2, 4] \n"
232 " st.ab %3, [%1, 4] \n"
233 " st.ab %4, [%1, 4] \n"
234 " st.ab %5, [%1, 4] \n"
235 " st.ab %6, [%1, 4] \n"
238 " .section .fixup, \"ax\" \n"
242 " .section __ex_table, \"a\" \n"
249 : "+r" (res
), "+r"(to
), "+r"(from
),
250 "=r"(tmp1
), "=r"(tmp2
), "=r"(tmp3
), "=r"(tmp4
)
252 : "lp_count", "memory");
257 __asm__
__volatile__(
258 "14: ld.ab %3, [%2,4] \n"
259 "15: ld.ab %4, [%2,4] \n"
260 " st.ab %3, [%1,4] \n"
261 " st.ab %4, [%1,4] \n"
264 " .section .fixup, \"ax\" \n"
268 " .section __ex_table, \"a\" \n"
273 : "+r" (res
), "+r"(to
), "+r"(from
),
274 "=r"(tmp1
), "=r"(tmp2
)
281 __asm__
__volatile__(
282 "16: ld.ab %3, [%2,4] \n"
283 " st.ab %3, [%1,4] \n"
286 " .section .fixup, \"ax\" \n"
290 " .section __ex_table, \"a\" \n"
294 : "+r" (res
), "+r"(to
), "+r"(from
), "=r"(tmp1
)
301 __asm__
__volatile__(
302 "17: ldw.ab %3, [%2,2] \n"
303 " stw.ab %3, [%1,2] \n"
306 " .section .fixup, \"ax\" \n"
310 " .section __ex_table, \"a\" \n"
314 : "+r" (res
), "+r"(to
), "+r"(from
), "=r"(tmp1
)
319 __asm__
__volatile__(
320 "18: ldb.ab %3, [%2,2] \n"
321 " stb.ab %3, [%1,2] \n"
324 " .section .fixup, \"ax\" \n"
328 " .section __ex_table, \"a\" \n"
332 : "+r" (res
), "+r"(to
), "+r"(from
), "=r"(tmp1
)
336 } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
338 __asm__
__volatile__(
340 " lsr.f lp_count, %3,4 \n" /* 16x bytes */
342 "1: ld.ab %5, [%2, 4] \n"
343 "11: ld.ab %6, [%2, 4] \n"
344 "12: ld.ab %7, [%2, 4] \n"
345 "13: ld.ab %8, [%2, 4] \n"
346 " st.ab %5, [%1, 4] \n"
347 " st.ab %6, [%1, 4] \n"
348 " st.ab %7, [%1, 4] \n"
349 " st.ab %8, [%1, 4] \n"
351 "3: and.f %3,%3,0xf \n" /* stragglers */
353 " bbit0 %3,3,31f \n" /* 8 bytes left */
354 "14: ld.ab %5, [%2,4] \n"
355 "15: ld.ab %6, [%2,4] \n"
356 " st.ab %5, [%1,4] \n"
357 " st.ab %6, [%1,4] \n"
359 "31: bbit0 %3,2,32f \n" /* 4 bytes left */
360 "16: ld.ab %5, [%2,4] \n"
361 " st.ab %5, [%1,4] \n"
363 "32: bbit0 %3,1,33f \n" /* 2 bytes left */
364 "17: ldw.ab %5, [%2,2] \n"
365 " stw.ab %5, [%1,2] \n"
367 "33: bbit0 %3,0,34f \n"
368 "18: ldb.ab %5, [%2,1] \n" /* 1 byte left */
369 " stb.ab %5, [%1,1] \n"
372 " .section .fixup, \"ax\" \n"
376 " .section __ex_table, \"a\" \n"
388 : "=r" (res
), "+r"(to
), "+r"(from
), "+r"(n
), "=r"(val
),
389 "=r"(tmp1
), "=r"(tmp2
), "=r"(tmp3
), "=r"(tmp4
)
391 : "lp_count", "memory");
397 static inline unsigned long
398 raw_copy_to_user(void __user
*to
, const void *from
, unsigned long n
)
402 unsigned long tmp1
, tmp2
, tmp3
, tmp4
;
403 unsigned long orig_n
= n
;
409 if (((unsigned long)to
& 0x3) || ((unsigned long)from
& 0x3)) {
413 __asm__
__volatile__(
414 " mov.f lp_count, %0 \n"
416 " ldb.ab %1, [%3, 1] \n"
417 "1: stb.ab %1, [%2, 1] \n"
420 " .section .fixup, \"ax\" \n"
424 " .section __ex_table, \"a\" \n"
430 /* Note as an '&' earlyclobber operand to make sure the
431 * temporary register inside the loop is not the same as
434 "=&r" (tmp
), "+r" (to
), "+r" (from
)
436 : "lp_count", "lp_start", "lp_end", "memory");
441 if (__builtin_constant_p(orig_n
)) {
445 orig_n
= orig_n
% 16;
447 __asm__
__volatile__(
448 " lsr lp_count, %7,4 \n"
450 " ld.ab %3, [%2, 4] \n"
451 " ld.ab %4, [%2, 4] \n"
452 " ld.ab %5, [%2, 4] \n"
453 " ld.ab %6, [%2, 4] \n"
454 "1: st.ab %3, [%1, 4] \n"
455 "11: st.ab %4, [%1, 4] \n"
456 "12: st.ab %5, [%1, 4] \n"
457 "13: st.ab %6, [%1, 4] \n"
460 " .section .fixup, \"ax\" \n"
464 " .section __ex_table, \"a\" \n"
471 : "+r" (res
), "+r"(to
), "+r"(from
),
472 "=r"(tmp1
), "=r"(tmp2
), "=r"(tmp3
), "=r"(tmp4
)
474 : "lp_count", "memory");
479 __asm__
__volatile__(
480 " ld.ab %3, [%2,4] \n"
481 " ld.ab %4, [%2,4] \n"
482 "14: st.ab %3, [%1,4] \n"
483 "15: st.ab %4, [%1,4] \n"
486 " .section .fixup, \"ax\" \n"
490 " .section __ex_table, \"a\" \n"
495 : "+r" (res
), "+r"(to
), "+r"(from
),
496 "=r"(tmp1
), "=r"(tmp2
)
503 __asm__
__volatile__(
504 " ld.ab %3, [%2,4] \n"
505 "16: st.ab %3, [%1,4] \n"
508 " .section .fixup, \"ax\" \n"
512 " .section __ex_table, \"a\" \n"
516 : "+r" (res
), "+r"(to
), "+r"(from
), "=r"(tmp1
)
523 __asm__
__volatile__(
524 " ldw.ab %3, [%2,2] \n"
525 "17: stw.ab %3, [%1,2] \n"
528 " .section .fixup, \"ax\" \n"
532 " .section __ex_table, \"a\" \n"
536 : "+r" (res
), "+r"(to
), "+r"(from
), "=r"(tmp1
)
541 __asm__
__volatile__(
542 " ldb.ab %3, [%2,1] \n"
543 "18: stb.ab %3, [%1,1] \n"
546 " .section .fixup, \"ax\" \n"
550 " .section __ex_table, \"a\" \n"
554 : "+r" (res
), "+r"(to
), "+r"(from
), "=r"(tmp1
)
558 } else { /* n is NOT constant, so laddered copy of 16x,8,4,2,1 */
560 __asm__
__volatile__(
562 " lsr.f lp_count, %3,4 \n" /* 16x bytes */
564 " ld.ab %5, [%2, 4] \n"
565 " ld.ab %6, [%2, 4] \n"
566 " ld.ab %7, [%2, 4] \n"
567 " ld.ab %8, [%2, 4] \n"
568 "1: st.ab %5, [%1, 4] \n"
569 "11: st.ab %6, [%1, 4] \n"
570 "12: st.ab %7, [%1, 4] \n"
571 "13: st.ab %8, [%1, 4] \n"
573 "3: and.f %3,%3,0xf \n" /* stragglers */
575 " bbit0 %3,3,31f \n" /* 8 bytes left */
576 " ld.ab %5, [%2,4] \n"
577 " ld.ab %6, [%2,4] \n"
578 "14: st.ab %5, [%1,4] \n"
579 "15: st.ab %6, [%1,4] \n"
580 " sub.f %0, %0, 8 \n"
581 "31: bbit0 %3,2,32f \n" /* 4 bytes left */
582 " ld.ab %5, [%2,4] \n"
583 "16: st.ab %5, [%1,4] \n"
584 " sub.f %0, %0, 4 \n"
585 "32: bbit0 %3,1,33f \n" /* 2 bytes left */
586 " ldw.ab %5, [%2,2] \n"
587 "17: stw.ab %5, [%1,2] \n"
588 " sub.f %0, %0, 2 \n"
589 "33: bbit0 %3,0,34f \n"
590 " ldb.ab %5, [%2,1] \n" /* 1 byte left */
591 "18: stb.ab %5, [%1,1] \n"
592 " sub.f %0, %0, 1 \n"
594 " .section .fixup, \"ax\" \n"
598 " .section __ex_table, \"a\" \n"
610 : "=r" (res
), "+r"(to
), "+r"(from
), "+r"(n
), "=r"(val
),
611 "=r"(tmp1
), "=r"(tmp2
), "=r"(tmp3
), "=r"(tmp4
)
613 : "lp_count", "memory");
619 static inline unsigned long __arc_clear_user(void __user
*to
, unsigned long n
)
622 unsigned char *d_char
= to
;
624 __asm__
__volatile__(
625 " bbit0 %0, 0, 1f \n"
626 "75: stb.ab %2, [%0,1] \n"
628 "1: bbit0 %0, 1, 2f \n"
629 "76: stw.ab %2, [%0,2] \n"
631 "2: asr.f lp_count, %1, 2 \n"
633 "77: st.ab %2, [%0,4] \n"
635 "3: bbit0 %1, 1, 4f \n"
636 "78: stw.ab %2, [%0,2] \n"
638 "4: bbit0 %1, 0, 5f \n"
639 "79: stb.ab %2, [%0,1] \n"
642 " .section .fixup, \"ax\" \n"
646 " .section __ex_table, \"a\" \n"
654 : "+r"(d_char
), "+r"(res
)
656 : "lp_count", "lp_start", "lp_end", "memory");
662 __arc_strncpy_from_user(char *dst
, const char __user
*src
, long count
)
670 __asm__
__volatile__(
671 " mov lp_count, %5 \n"
673 "1: ldb.ab %3, [%2, 1] \n"
674 " breq.d %3, 0, 3f \n"
675 " stb.ab %3, [%1, 1] \n"
676 " add %0, %0, 1 # Num of NON NULL bytes copied \n"
678 " .section .fixup, \"ax\" \n"
680 "4: mov %0, %4 # sets @res as -EFAULT \n"
683 " .section __ex_table, \"a\" \n"
687 : "+r"(res
), "+r"(dst
), "+r"(src
), "=r"(val
)
688 : "g"(-EFAULT
), "r"(count
)
689 : "lp_count", "lp_start", "lp_end", "memory");
694 static inline long __arc_strnlen_user(const char __user
*s
, long n
)
699 __asm__
__volatile__(
701 "1: ldb.ab %3, [%0, 1] \n"
702 " breq.d %3, 0, 2f \n"
703 " sub.f %2, %2, 1 \n"
706 "2: sub %0, %1, %2 \n"
708 " .section .fixup, \"ax\" \n"
713 " .section __ex_table, \"a\" \n"
717 : "=r"(res
), "=r"(tmp1
), "=r"(cnt
), "=r"(val
)
724 #ifndef CONFIG_CC_OPTIMIZE_FOR_SIZE
726 #define INLINE_COPY_TO_USER
727 #define INLINE_COPY_FROM_USER
729 #define __clear_user(d, n) __arc_clear_user(d, n)
730 #define __strncpy_from_user(d, s, n) __arc_strncpy_from_user(d, s, n)
731 #define __strnlen_user(s, n) __arc_strnlen_user(s, n)
733 extern unsigned long arc_clear_user_noinline(void __user
*to
,
735 extern long arc_strncpy_from_user_noinline (char *dst
, const char __user
*src
,
737 extern long arc_strnlen_user_noinline(const char __user
*src
, long n
);
739 #define __clear_user(d, n) arc_clear_user_noinline(d, n)
740 #define __strncpy_from_user(d, s, n) arc_strncpy_from_user_noinline(d, s, n)
741 #define __strnlen_user(s, n) arc_strnlen_user_noinline(s, n)
745 #include <asm-generic/uaccess.h>