include/linux/memcopy.h

   1 /*
   2  * memcopy.h -- definitions for memory copy functions.  Generic C version.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License as published by the Free
   6  * Software Foundation; either version 2 of the License, or (at your option)
   7  * any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
  12  * Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The code is derived from the GNU C Library.
  19  * Copyright (C) 1991, 1992, 1993, 1997, 2004 Free Software Foundation, Inc.
  20  */
  21 #ifndef _LINUX_MEMCOPY_H_
  22 #define _LINUX_MEMCOPY_H_
  23
  24 /*
  25  * The strategy of the memory functions is:
  26  *
  27  *   1. Copy bytes until the destination pointer is aligned.
  28  *
  29  *   2. Copy words in unrolled loops.  If the source and destination
  30  *      are not aligned in the same way, use word memory operations,
  31  *      but shift and merge two read words before writing.
  32  *
  33  *   3. Copy the few remaining bytes.
  34  *
  35  * This is fast on processors that have at least 10 registers for
  36  * allocation by GCC, and that can access memory at reg+const in one
  37  * instruction.
  38  */
  39
  40 #include <linux/types.h>
  41 #include <linux/compiler.h>
  42 #include <asm/byteorder.h>
  43
  44 /*
  45  * The macros defined in this file are:
  46  *
  47  * BYTE_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_to_copy)
  48  *
  49  * BYTE_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_to_copy)
  50  *
  51  * WORD_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_remaining, nbytes_to_copy)
  52  *
  53  * WORD_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_remaining, nbytes_to_copy)
  54  *
  55  * MERGE(old_word, sh_1, new_word, sh_2)
  56  *
  57  * MEM_COPY_FWD(dst_beg_ptr, src_beg_ptr, nbytes_to_copy)
  58  *
  59  * MEM_COPY_BWD(dst_end_ptr, src_end_ptr, nbytes_to_copy)
  60  */
  61
  62 #define OP_T_THRESHOLD  16
  63
  64 /*
  65  * Type to use for aligned memory operations.
  66  * This should normally be the biggest type supported by a single load
  67  * and store.
  68  */
  69 #define op_t    unsigned long int
  70 #define OPSIZ   (sizeof(op_t))
  71
  72 /* Type to use for unaligned operations.  */
  73 typedef unsigned char byte;
  74
  75 #ifndef MERGE
  76 # ifdef __LITTLE_ENDIAN
  77 #  define MERGE(w0, sh_1, w1, sh_2) (((w0) >> (sh_1)) | ((w1) << (sh_2)))
  78 # elif defined(__BIG_ENDIAN)
  79 #  define MERGE(w0, sh_1, w1, sh_2) (((w0) << (sh_1)) | ((w1) >> (sh_2)))
  80 # else
  81 #  error "Macro MERGE() hasn't defined!"
  82 # endif
  83 #endif
  84
  85 /*
  86  * Copy exactly NBYTES bytes from SRC_BP to DST_BP,
  87  * without any assumptions about alignment of the pointers.
  88  */
  89 #ifndef BYTE_COPY_FWD
  90 #define BYTE_COPY_FWD(dst_bp, src_bp, nbytes)                                 \
  91 do {                                                                          \
  92         size_t __nbytes = (nbytes);                                           \
  93         while (__nbytes > 0) {                                                \
  94                 byte __x = ((byte *) src_bp)[0];                              \
  95                 src_bp += 1;                                                  \
  96                 __nbytes -= 1;                                                \
  97                 ((byte *) dst_bp)[0] = __x;                                   \
  98                 dst_bp += 1;                                                  \
  99         }                                                                     \
 100 } while (0)
 101 #endif
 102
 103 /*
 104  * Copy exactly NBYTES_TO_COPY bytes from SRC_END_PTR to DST_END_PTR,
 105  * beginning at the bytes right before the pointers and continuing towards
 106  * smaller addresses.  Don't assume anything about alignment of the
 107  * pointers.
 108  */
 109 #ifndef BYTE_COPY_BWD
 110 #define BYTE_COPY_BWD(dst_ep, src_ep, nbytes)                                 \
 111 do {                                                                          \
 112         size_t __nbytes = (nbytes);                                           \
 113         while (__nbytes > 0) {                                                \
 114                 byte __x;                                                     \
 115                 src_ep -= 1;                                                  \
 116                 __x = ((byte *) src_ep)[0];                                   \
 117                 dst_ep -= 1;                                                  \
 118                 __nbytes -= 1;                                                \
 119                 ((byte *) dst_ep)[0] = __x;                                   \
 120         }                                                                     \
 121 } while (0)
 122 #endif
 123 /*
 124  * Copy *up to* NBYTES bytes from SRC_BP to DST_BP, with
 125  * the assumption that DST_BP is aligned on an OPSIZ multiple.  If
 126  * not all bytes could be easily copied, store remaining number of bytes
 127  * in NBYTES_LEFT, otherwise store 0.
 128  */
 129 extern void _wordcopy_fwd_aligned(long int, long int, size_t);
 130 extern void _wordcopy_fwd_dest_aligned(long int, long int, size_t);
 131 #ifndef WORD_COPY_FWD
 132 #define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes)                    \
 133 do {                                                                          \
 134         if (src_bp % OPSIZ == 0)                                              \
 135                 _wordcopy_fwd_aligned (dst_bp, src_bp, (nbytes) / OPSIZ);     \
 136         else                                                                  \
 137                 _wordcopy_fwd_dest_aligned (dst_bp, src_bp, (nbytes) / OPSIZ);\
 138                                                                               \
 139         src_bp += (nbytes) & -OPSIZ;                                          \
 140         dst_bp += (nbytes) & -OPSIZ;                                          \
 141         (nbytes_left) = (nbytes) % OPSIZ;                                     \
 142 } while (0)
 143 #endif
 144
 145 /*
 146  * Copy *up to* NBYTES_TO_COPY bytes from SRC_END_PTR to DST_END_PTR,
 147  * beginning at the words (of type op_t) right before the pointers and
 148  * continuing towards smaller addresses.  May take advantage of that
 149  * DST_END_PTR is aligned on an OPSIZ multiple.  If not all bytes could be
 150  * easily copied, store remaining number of bytes in NBYTES_REMAINING,
 151  * otherwise store 0.
 152  */
 153 extern void _wordcopy_bwd_aligned(long int, long int, size_t);
 154 extern void _wordcopy_bwd_dest_aligned(long int, long int, size_t);
 155 #ifndef WORD_COPY_BWD
 156 #define WORD_COPY_BWD(dst_ep, src_ep, nbytes_left, nbytes)                    \
 157 do {                                                                          \
 158         if (src_ep % OPSIZ == 0)                                              \
 159                 _wordcopy_bwd_aligned (dst_ep, src_ep, (nbytes) / OPSIZ);     \
 160         else                                                                  \
 161                 _wordcopy_bwd_dest_aligned (dst_ep, src_ep, (nbytes) / OPSIZ);\
 162                                                                               \
 163         src_ep -= (nbytes) & -OPSIZ;                                          \
 164         dst_ep -= (nbytes) & -OPSIZ;                                          \
 165         (nbytes_left) = (nbytes) % OPSIZ;                                     \
 166 } while (0)
 167 #endif
 168
 169 /* Copy memory from the beginning to the end */
 170 #ifndef MEM_COPY_FWD
 171 static __always_inline void mem_copy_fwd(unsigned long dstp,
 172                                         unsigned long srcp,
 173                                         size_t count)
 174 {
 175         /* If there not too few bytes to copy, use word copy. */
 176         if (count >= OP_T_THRESHOLD) {
 177                 /* Copy just a few bytes to make dstp aligned. */
 178                 count -= (-dstp) % OPSIZ;
 179                 BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ);
 180
 181                 /*
 182                  * Copy from srcp to dstp taking advantage of the known
 183                  * alignment of dstp. Number if bytes remaining is put in
 184                  * the third argument.
 185                  */
 186                 WORD_COPY_FWD(dstp, srcp, count, count);
 187
 188                 /* Fall out and copy the tail. */
 189         }
 190
 191         /* There are just a few bytes to copy. Use byte memory operations. */
 192         BYTE_COPY_FWD(dstp, srcp, count);
 193 }
 194 #endif
 195
 196 /* Copy memory from the end to the beginning.  */
 197 #ifndef MEM_COPY_BWD
 198 static __always_inline void mem_copy_bwd(unsigned long dstp,
 199                                         unsigned long srcp,
 200                                         size_t count)
 201 {
 202         srcp += count;
 203         dstp += count;
 204
 205         /* If there not too few bytes to copy, use word copy. */
 206         if (count >= OP_T_THRESHOLD) {
 207                 /* Copy just a few bytes to make dstp aligned. */
 208                 count -= dstp % OPSIZ;
 209                 BYTE_COPY_BWD(dstp, srcp, dstp % OPSIZ);
 210
 211                 /*
 212                  * Copy from srcp to dstp taking advantage of the known
 213                  * alignment of dstp. Number if bytes remaining is put in
 214                  * the third argument.
 215                  */
 216                 WORD_COPY_BWD(dstp, srcp, count, count);
 217
 218                 /* Fall out and copy the tail. */
 219         }
 220
 221         /* There are just a few bytes to copy. Use byte memory operations. */
 222         BYTE_COPY_BWD (dstp, srcp, count);
 223 }
 224 #endif
 225
 226 #endif
 227