arch/x86/lib/memmove_64.c

   1 /* Normally compiler builtins are used, but sometimes the compiler calls out
   2    of line code. Based on asm-i386/string.h.
   3  */
   4 #define _STRING_C
   5 #include <linux/string.h>
   6 #include <linux/module.h>
   7
   8 #undef memmove
   9 void *memmove(void *dest, const void *src, size_t count)
  10 {
  11         unsigned long d0,d1,d2,d3,d4,d5,d6,d7;
  12         char *ret;
  13
  14         __asm__ __volatile__(
  15                 /* Handle more 32bytes in loop */
  16                 "mov %2, %3\n\t"
  17                 "cmp $0x20, %0\n\t"
  18                 "jb     1f\n\t"
  19
  20                 /* Decide forward/backward copy mode */
  21                 "cmp %2, %1\n\t"
  22                 "jb     2f\n\t"
  23
  24                 /*
  25                  * movsq instruction have many startup latency
  26                  * so we handle small size by general register.
  27                  */
  28                 "cmp  $680, %0\n\t"
  29                 "jb 3f\n\t"
  30                 /*
  31                  * movsq instruction is only good for aligned case.
  32                  */
  33                 "cmpb %%dil, %%sil\n\t"
  34                 "je 4f\n\t"
  35                 "3:\n\t"
  36                 "sub $0x20, %0\n\t"
  37                 /*
  38                  * We gobble 32byts forward in each loop.
  39                  */
  40                 "5:\n\t"
  41                 "sub $0x20, %0\n\t"
  42                 "movq 0*8(%1), %4\n\t"
  43                 "movq 1*8(%1), %5\n\t"
  44                 "movq 2*8(%1), %6\n\t"
  45                 "movq 3*8(%1), %7\n\t"
  46                 "leaq 4*8(%1), %1\n\t"
  47
  48                 "movq %4, 0*8(%2)\n\t"
  49                 "movq %5, 1*8(%2)\n\t"
  50                 "movq %6, 2*8(%2)\n\t"
  51                 "movq %7, 3*8(%2)\n\t"
  52                 "leaq 4*8(%2), %2\n\t"
  53                 "jae 5b\n\t"
  54                 "addq $0x20, %0\n\t"
  55                 "jmp 1f\n\t"
  56                 /*
  57                  * Handle data forward by movsq.
  58                  */
  59                 ".p2align 4\n\t"
  60                 "4:\n\t"
  61                 "movq %0, %8\n\t"
  62                 "movq -8(%1, %0), %4\n\t"
  63                 "lea -8(%2, %0), %5\n\t"
  64                 "shrq $3, %8\n\t"
  65                 "rep movsq\n\t"
  66                 "movq %4, (%5)\n\t"
  67                 "jmp 13f\n\t"
  68                 /*
  69                  * Handle data backward by movsq.
  70                  */
  71                 ".p2align 4\n\t"
  72                 "7:\n\t"
  73                 "movq %0, %8\n\t"
  74                 "movq (%1), %4\n\t"
  75                 "movq %2, %5\n\t"
  76                 "leaq -8(%1, %0), %1\n\t"
  77                 "leaq -8(%2, %0), %2\n\t"
  78                 "shrq $3, %8\n\t"
  79                 "std\n\t"
  80                 "rep movsq\n\t"
  81                 "cld\n\t"
  82                 "movq %4, (%5)\n\t"
  83                 "jmp 13f\n\t"
  84
  85                 /*
  86                  * Start to prepare for backward copy.
  87                  */
  88                 ".p2align 4\n\t"
  89                 "2:\n\t"
  90                 "cmp $680, %0\n\t"
  91                 "jb 6f \n\t"
  92                 "cmp %%dil, %%sil\n\t"
  93                 "je 7b \n\t"
  94                 "6:\n\t"
  95                 /*
  96                  * Calculate copy position to tail.
  97                  */
  98                 "addq %0, %1\n\t"
  99                 "addq %0, %2\n\t"
 100                 "subq $0x20, %0\n\t"
 101                 /*
 102                  * We gobble 32byts backward in each loop.
 103                  */
 104                 "8:\n\t"
 105                 "subq $0x20, %0\n\t"
 106                 "movq -1*8(%1), %4\n\t"
 107                 "movq -2*8(%1), %5\n\t"
 108                 "movq -3*8(%1), %6\n\t"
 109                 "movq -4*8(%1), %7\n\t"
 110                 "leaq -4*8(%1), %1\n\t"
 111
 112                 "movq %4, -1*8(%2)\n\t"
 113                 "movq %5, -2*8(%2)\n\t"
 114                 "movq %6, -3*8(%2)\n\t"
 115                 "movq %7, -4*8(%2)\n\t"
 116                 "leaq -4*8(%2), %2\n\t"
 117                 "jae 8b\n\t"
 118                 /*
 119                  * Calculate copy position to head.
 120                  */
 121                 "addq $0x20, %0\n\t"
 122                 "subq %0, %1\n\t"
 123                 "subq %0, %2\n\t"
 124                 "1:\n\t"
 125                 "cmpq $16, %0\n\t"
 126                 "jb 9f\n\t"
 127                 /*
 128                  * Move data from 16 bytes to 31 bytes.
 129                  */
 130                 "movq 0*8(%1), %4\n\t"
 131                 "movq 1*8(%1), %5\n\t"
 132                 "movq -2*8(%1, %0), %6\n\t"
 133                 "movq -1*8(%1, %0), %7\n\t"
 134                 "movq %4, 0*8(%2)\n\t"
 135                 "movq %5, 1*8(%2)\n\t"
 136                 "movq %6, -2*8(%2, %0)\n\t"
 137                 "movq %7, -1*8(%2, %0)\n\t"
 138                 "jmp 13f\n\t"
 139                 ".p2align 4\n\t"
 140                 "9:\n\t"
 141                 "cmpq $8, %0\n\t"
 142                 "jb 10f\n\t"
 143                 /*
 144                  * Move data from 8 bytes to 15 bytes.
 145                  */
 146                 "movq 0*8(%1), %4\n\t"
 147                 "movq -1*8(%1, %0), %5\n\t"
 148                 "movq %4, 0*8(%2)\n\t"
 149                 "movq %5, -1*8(%2, %0)\n\t"
 150                 "jmp 13f\n\t"
 151                 "10:\n\t"
 152                 "cmpq $4, %0\n\t"
 153                 "jb 11f\n\t"
 154                 /*
 155                  * Move data from 4 bytes to 7 bytes.
 156                  */
 157                 "movl (%1), %4d\n\t"
 158                 "movl -4(%1, %0), %5d\n\t"
 159                 "movl %4d, (%2)\n\t"
 160                 "movl %5d, -4(%2, %0)\n\t"
 161                 "jmp 13f\n\t"
 162                 "11:\n\t"
 163                 "cmp $2, %0\n\t"
 164                 "jb 12f\n\t"
 165                 /*
 166                  * Move data from 2 bytes to 3 bytes.
 167                  */
 168                 "movw (%1), %4w\n\t"
 169                 "movw -2(%1, %0), %5w\n\t"
 170                 "movw %4w, (%2)\n\t"
 171                 "movw %5w, -2(%2, %0)\n\t"
 172                 "jmp 13f\n\t"
 173                 "12:\n\t"
 174                 "cmp $1, %0\n\t"
 175                 "jb 13f\n\t"
 176                 /*
 177                  * Move data for 1 byte.
 178                  */
 179                 "movb (%1), %4b\n\t"
 180                 "movb %4b, (%2)\n\t"
 181                 "13:\n\t"
 182                 : "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) ,
 183                   "=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7)
 184                 :"0" (count),
 185                  "1" (src),
 186                  "2" (dest)
 187                 :"memory");
 188
 189                 return ret;
 190
 191 }
 192 EXPORT_SYMBOL(memmove);