arch/x86/lib/usercopy_32.c

   1 /*
   2  * User address space access functions.
   3  * The non inlined parts of asm-i386/uaccess.h are here.
   4  *
   5  * Copyright 1997 Andi Kleen <ak@muc.de>
   6  * Copyright 1997 Linus Torvalds
   7  */
   8 #include <linux/export.h>
   9 #include <linux/uaccess.h>
  10 #include <asm/mmx.h>
  11 #include <asm/asm.h>
  12
  13 #ifdef CONFIG_X86_INTEL_USERCOPY
  14 /*
  15  * Alignment at which movsl is preferred for bulk memory copies.
  16  */
  17 struct movsl_mask movsl_mask __read_mostly;
  18 #endif
  19
  20 static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
  21 {
  22 #ifdef CONFIG_X86_INTEL_USERCOPY
  23         if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask))
  24                 return 0;
  25 #endif
  26         return 1;
  27 }
  28 #define movsl_is_ok(a1, a2, n) \
  29         __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n))
  30
  31 /*
  32  * Zero Userspace
  33  */
  34
  35 #define __do_clear_user(addr,size)                                      \
  36 do {                                                                    \
  37         int __d0;                                                       \
  38         might_fault();                                                  \
  39         __asm__ __volatile__(                                           \
  40                 ASM_STAC "\n"                                           \
  41                 "0:     rep; stosl\n"                                   \
  42                 "       movl %2,%0\n"                                   \
  43                 "1:     rep; stosb\n"                                   \
  44                 "2: " ASM_CLAC "\n"                                     \
  45                 ".section .fixup,\"ax\"\n"                              \
  46                 "3:     lea 0(%2,%0,4),%0\n"                            \
  47                 "       jmp 2b\n"                                       \
  48                 ".previous\n"                                           \
  49                 _ASM_EXTABLE(0b,3b)                                     \
  50                 _ASM_EXTABLE(1b,2b)                                     \
  51                 : "=&c"(size), "=&D" (__d0)                             \
  52                 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0));     \
  53 } while (0)
  54
  55 /**
  56  * clear_user: - Zero a block of memory in user space.
  57  * @to:   Destination address, in user space.
  58  * @n:    Number of bytes to zero.
  59  *
  60  * Zero a block of memory in user space.
  61  *
  62  * Returns number of bytes that could not be cleared.
  63  * On success, this will be zero.
  64  */
  65 unsigned long
  66 clear_user(void __user *to, unsigned long n)
  67 {
  68         might_fault();
  69         if (access_ok(VERIFY_WRITE, to, n))
  70                 __do_clear_user(to, n);
  71         return n;
  72 }
  73 EXPORT_SYMBOL(clear_user);
  74
  75 /**
  76  * __clear_user: - Zero a block of memory in user space, with less checking.
  77  * @to:   Destination address, in user space.
  78  * @n:    Number of bytes to zero.
  79  *
  80  * Zero a block of memory in user space.  Caller must check
  81  * the specified block with access_ok() before calling this function.
  82  *
  83  * Returns number of bytes that could not be cleared.
  84  * On success, this will be zero.
  85  */
  86 unsigned long
  87 __clear_user(void __user *to, unsigned long n)
  88 {
  89         __do_clear_user(to, n);
  90         return n;
  91 }
  92 EXPORT_SYMBOL(__clear_user);
  93
  94 #ifdef CONFIG_X86_INTEL_USERCOPY
  95 static unsigned long
  96 __copy_user_intel(void __user *to, const void *from, unsigned long size)
  97 {
  98         int d0, d1;
  99         __asm__ __volatile__(
 100                        "       .align 2,0x90\n"
 101                        "1:     movl 32(%4), %%eax\n"
 102                        "       cmpl $67, %0\n"
 103                        "       jbe 3f\n"
 104                        "2:     movl 64(%4), %%eax\n"
 105                        "       .align 2,0x90\n"
 106                        "3:     movl 0(%4), %%eax\n"
 107                        "4:     movl 4(%4), %%edx\n"
 108                        "5:     movl %%eax, 0(%3)\n"
 109                        "6:     movl %%edx, 4(%3)\n"
 110                        "7:     movl 8(%4), %%eax\n"
 111                        "8:     movl 12(%4),%%edx\n"
 112                        "9:     movl %%eax, 8(%3)\n"
 113                        "10:    movl %%edx, 12(%3)\n"
 114                        "11:    movl 16(%4), %%eax\n"
 115                        "12:    movl 20(%4), %%edx\n"
 116                        "13:    movl %%eax, 16(%3)\n"
 117                        "14:    movl %%edx, 20(%3)\n"
 118                        "15:    movl 24(%4), %%eax\n"
 119                        "16:    movl 28(%4), %%edx\n"
 120                        "17:    movl %%eax, 24(%3)\n"
 121                        "18:    movl %%edx, 28(%3)\n"
 122                        "19:    movl 32(%4), %%eax\n"
 123                        "20:    movl 36(%4), %%edx\n"
 124                        "21:    movl %%eax, 32(%3)\n"
 125                        "22:    movl %%edx, 36(%3)\n"
 126                        "23:    movl 40(%4), %%eax\n"
 127                        "24:    movl 44(%4), %%edx\n"
 128                        "25:    movl %%eax, 40(%3)\n"
 129                        "26:    movl %%edx, 44(%3)\n"
 130                        "27:    movl 48(%4), %%eax\n"
 131                        "28:    movl 52(%4), %%edx\n"
 132                        "29:    movl %%eax, 48(%3)\n"
 133                        "30:    movl %%edx, 52(%3)\n"
 134                        "31:    movl 56(%4), %%eax\n"
 135                        "32:    movl 60(%4), %%edx\n"
 136                        "33:    movl %%eax, 56(%3)\n"
 137                        "34:    movl %%edx, 60(%3)\n"
 138                        "       addl $-64, %0\n"
 139                        "       addl $64, %4\n"
 140                        "       addl $64, %3\n"
 141                        "       cmpl $63, %0\n"
 142                        "       ja  1b\n"
 143                        "35:    movl  %0, %%eax\n"
 144                        "       shrl  $2, %0\n"
 145                        "       andl  $3, %%eax\n"
 146                        "       cld\n"
 147                        "99:    rep; movsl\n"
 148                        "36:    movl %%eax, %0\n"
 149                        "37:    rep; movsb\n"
 150                        "100:\n"
 151                        ".section .fixup,\"ax\"\n"
 152                        "101:   lea 0(%%eax,%0,4),%0\n"
 153                        "       jmp 100b\n"
 154                        ".previous\n"
 155                        _ASM_EXTABLE(1b,100b)
 156                        _ASM_EXTABLE(2b,100b)
 157                        _ASM_EXTABLE(3b,100b)
 158                        _ASM_EXTABLE(4b,100b)
 159                        _ASM_EXTABLE(5b,100b)
 160                        _ASM_EXTABLE(6b,100b)
 161                        _ASM_EXTABLE(7b,100b)
 162                        _ASM_EXTABLE(8b,100b)
 163                        _ASM_EXTABLE(9b,100b)
 164                        _ASM_EXTABLE(10b,100b)
 165                        _ASM_EXTABLE(11b,100b)
 166                        _ASM_EXTABLE(12b,100b)
 167                        _ASM_EXTABLE(13b,100b)
 168                        _ASM_EXTABLE(14b,100b)
 169                        _ASM_EXTABLE(15b,100b)
 170                        _ASM_EXTABLE(16b,100b)
 171                        _ASM_EXTABLE(17b,100b)
 172                        _ASM_EXTABLE(18b,100b)
 173                        _ASM_EXTABLE(19b,100b)
 174                        _ASM_EXTABLE(20b,100b)
 175                        _ASM_EXTABLE(21b,100b)
 176                        _ASM_EXTABLE(22b,100b)
 177                        _ASM_EXTABLE(23b,100b)
 178                        _ASM_EXTABLE(24b,100b)
 179                        _ASM_EXTABLE(25b,100b)
 180                        _ASM_EXTABLE(26b,100b)
 181                        _ASM_EXTABLE(27b,100b)
 182                        _ASM_EXTABLE(28b,100b)
 183                        _ASM_EXTABLE(29b,100b)
 184                        _ASM_EXTABLE(30b,100b)
 185                        _ASM_EXTABLE(31b,100b)
 186                        _ASM_EXTABLE(32b,100b)
 187                        _ASM_EXTABLE(33b,100b)
 188                        _ASM_EXTABLE(34b,100b)
 189                        _ASM_EXTABLE(35b,100b)
 190                        _ASM_EXTABLE(36b,100b)
 191                        _ASM_EXTABLE(37b,100b)
 192                        _ASM_EXTABLE(99b,101b)
 193                        : "=&c"(size), "=&D" (d0), "=&S" (d1)
 194                        :  "1"(to), "2"(from), "0"(size)
 195                        : "eax", "edx", "memory");
 196         return size;
 197 }
 198
 199 static unsigned long __copy_user_intel_nocache(void *to,
 200                                 const void __user *from, unsigned long size)
 201 {
 202         int d0, d1;
 203
 204         __asm__ __volatile__(
 205                "        .align 2,0x90\n"
 206                "0:      movl 32(%4), %%eax\n"
 207                "        cmpl $67, %0\n"
 208                "        jbe 2f\n"
 209                "1:      movl 64(%4), %%eax\n"
 210                "        .align 2,0x90\n"
 211                "2:      movl 0(%4), %%eax\n"
 212                "21:     movl 4(%4), %%edx\n"
 213                "        movnti %%eax, 0(%3)\n"
 214                "        movnti %%edx, 4(%3)\n"
 215                "3:      movl 8(%4), %%eax\n"
 216                "31:     movl 12(%4),%%edx\n"
 217                "        movnti %%eax, 8(%3)\n"
 218                "        movnti %%edx, 12(%3)\n"
 219                "4:      movl 16(%4), %%eax\n"
 220                "41:     movl 20(%4), %%edx\n"
 221                "        movnti %%eax, 16(%3)\n"
 222                "        movnti %%edx, 20(%3)\n"
 223                "10:     movl 24(%4), %%eax\n"
 224                "51:     movl 28(%4), %%edx\n"
 225                "        movnti %%eax, 24(%3)\n"
 226                "        movnti %%edx, 28(%3)\n"
 227                "11:     movl 32(%4), %%eax\n"
 228                "61:     movl 36(%4), %%edx\n"
 229                "        movnti %%eax, 32(%3)\n"
 230                "        movnti %%edx, 36(%3)\n"
 231                "12:     movl 40(%4), %%eax\n"
 232                "71:     movl 44(%4), %%edx\n"
 233                "        movnti %%eax, 40(%3)\n"
 234                "        movnti %%edx, 44(%3)\n"
 235                "13:     movl 48(%4), %%eax\n"
 236                "81:     movl 52(%4), %%edx\n"
 237                "        movnti %%eax, 48(%3)\n"
 238                "        movnti %%edx, 52(%3)\n"
 239                "14:     movl 56(%4), %%eax\n"
 240                "91:     movl 60(%4), %%edx\n"
 241                "        movnti %%eax, 56(%3)\n"
 242                "        movnti %%edx, 60(%3)\n"
 243                "        addl $-64, %0\n"
 244                "        addl $64, %4\n"
 245                "        addl $64, %3\n"
 246                "        cmpl $63, %0\n"
 247                "        ja  0b\n"
 248                "        sfence \n"
 249                "5:      movl  %0, %%eax\n"
 250                "        shrl  $2, %0\n"
 251                "        andl $3, %%eax\n"
 252                "        cld\n"
 253                "6:      rep; movsl\n"
 254                "        movl %%eax,%0\n"
 255                "7:      rep; movsb\n"
 256                "8:\n"
 257                ".section .fixup,\"ax\"\n"
 258                "9:      lea 0(%%eax,%0,4),%0\n"
 259                "16:     jmp 8b\n"
 260                ".previous\n"
 261                _ASM_EXTABLE(0b,16b)
 262                _ASM_EXTABLE(1b,16b)
 263                _ASM_EXTABLE(2b,16b)
 264                _ASM_EXTABLE(21b,16b)
 265                _ASM_EXTABLE(3b,16b)
 266                _ASM_EXTABLE(31b,16b)
 267                _ASM_EXTABLE(4b,16b)
 268                _ASM_EXTABLE(41b,16b)
 269                _ASM_EXTABLE(10b,16b)
 270                _ASM_EXTABLE(51b,16b)
 271                _ASM_EXTABLE(11b,16b)
 272                _ASM_EXTABLE(61b,16b)
 273                _ASM_EXTABLE(12b,16b)
 274                _ASM_EXTABLE(71b,16b)
 275                _ASM_EXTABLE(13b,16b)
 276                _ASM_EXTABLE(81b,16b)
 277                _ASM_EXTABLE(14b,16b)
 278                _ASM_EXTABLE(91b,16b)
 279                _ASM_EXTABLE(6b,9b)
 280                _ASM_EXTABLE(7b,16b)
 281                : "=&c"(size), "=&D" (d0), "=&S" (d1)
 282                :  "1"(to), "2"(from), "0"(size)
 283                : "eax", "edx", "memory");
 284         return size;
 285 }
 286
 287 #else
 288
 289 /*
 290  * Leave these declared but undefined.  They should not be any references to
 291  * them
 292  */
 293 unsigned long __copy_user_intel(void __user *to, const void *from,
 294                                         unsigned long size);
 295 #endif /* CONFIG_X86_INTEL_USERCOPY */
 296
 297 /* Generic arbitrary sized copy.  */
 298 #define __copy_user(to, from, size)                                     \
 299 do {                                                                    \
 300         int __d0, __d1, __d2;                                           \
 301         __asm__ __volatile__(                                           \
 302                 "       cmp  $7,%0\n"                                   \
 303                 "       jbe  1f\n"                                      \
 304                 "       movl %1,%0\n"                                   \
 305                 "       negl %0\n"                                      \
 306                 "       andl $7,%0\n"                                   \
 307                 "       subl %0,%3\n"                                   \
 308                 "4:     rep; movsb\n"                                   \
 309                 "       movl %3,%0\n"                                   \
 310                 "       shrl $2,%0\n"                                   \
 311                 "       andl $3,%3\n"                                   \
 312                 "       .align 2,0x90\n"                                \
 313                 "0:     rep; movsl\n"                                   \
 314                 "       movl %3,%0\n"                                   \
 315                 "1:     rep; movsb\n"                                   \
 316                 "2:\n"                                                  \
 317                 ".section .fixup,\"ax\"\n"                              \
 318                 "5:     addl %3,%0\n"                                   \
 319                 "       jmp 2b\n"                                       \
 320                 "3:     lea 0(%3,%0,4),%0\n"                            \
 321                 "       jmp 2b\n"                                       \
 322                 ".previous\n"                                           \
 323                 _ASM_EXTABLE(4b,5b)                                     \
 324                 _ASM_EXTABLE(0b,3b)                                     \
 325                 _ASM_EXTABLE(1b,2b)                                     \
 326                 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)   \
 327                 : "3"(size), "0"(size), "1"(to), "2"(from)              \
 328                 : "memory");                                            \
 329 } while (0)
 330
 331 unsigned long __copy_user_ll(void *to, const void *from, unsigned long n)
 332 {
 333         stac();
 334         if (movsl_is_ok(to, from, n))
 335                 __copy_user(to, from, n);
 336         else
 337                 n = __copy_user_intel(to, from, n);
 338         clac();
 339         return n;
 340 }
 341 EXPORT_SYMBOL(__copy_user_ll);
 342
 343 unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
 344                                         unsigned long n)
 345 {
 346         stac();
 347 #ifdef CONFIG_X86_INTEL_USERCOPY
 348         if (n > 64 && static_cpu_has(X86_FEATURE_XMM2))
 349                 n = __copy_user_intel_nocache(to, from, n);
 350         else
 351                 __copy_user(to, from, n);
 352 #else
 353         __copy_user(to, from, n);
 354 #endif
 355         clac();
 356         return n;
 357 }
 358 EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);