lib/memcopy.c

   1 /*
   2  * memcopy.c -- subroutines for memory copy functions.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License as published by the Free
   6  * Software Foundation; either version 2 of the License, or (at your option)
   7  * any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
  12  * Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The code is derived from the GNU C Library.
  19  * Copyright (C) 1991, 1992, 1993, 1997, 2004 Free Software Foundation, Inc.
  20  */
  21
  22 /* BE VERY CAREFUL IF YOU CHANGE THIS CODE...!  */
  23
  24 #include <linux/memcopy.h>
  25
  26 /*
  27  * _wordcopy_fwd_aligned -- Copy block beginning at SRCP to block beginning
  28  * at DSTP with LEN `op_t' words (not LEN bytes!).
  29  * Both SRCP and DSTP should be aligned for memory operations on `op_t's.
  30  */
  31 void _wordcopy_fwd_aligned (long int dstp, long int srcp, size_t len)
  32 {
  33         op_t a0, a1;
  34
  35         switch (len % 8) {
  36         case 2:
  37                 a0 = ((op_t *) srcp)[0];
  38                 srcp -= 6 * OPSIZ;
  39                 dstp -= 7 * OPSIZ;
  40                 len += 6;
  41                 goto do1;
  42         case 3:
  43                 a1 = ((op_t *) srcp)[0];
  44                 srcp -= 5 * OPSIZ;
  45                 dstp -= 6 * OPSIZ;
  46                 len += 5;
  47                 goto do2;
  48         case 4:
  49                 a0 = ((op_t *) srcp)[0];
  50                 srcp -= 4 * OPSIZ;
  51                 dstp -= 5 * OPSIZ;
  52                 len += 4;
  53                 goto do3;
  54         case 5:
  55                 a1 = ((op_t *) srcp)[0];
  56                 srcp -= 3 * OPSIZ;
  57                 dstp -= 4 * OPSIZ;
  58                 len += 3;
  59                 goto do4;
  60         case 6:
  61                 a0 = ((op_t *) srcp)[0];
  62                 srcp -= 2 * OPSIZ;
  63                 dstp -= 3 * OPSIZ;
  64                 len += 2;
  65                 goto do5;
  66         case 7:
  67                 a1 = ((op_t *) srcp)[0];
  68                 srcp -= 1 * OPSIZ;
  69                 dstp -= 2 * OPSIZ;
  70                 len += 1;
  71                 goto do6;
  72         case 0:
  73                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
  74                         return;
  75                 a0 = ((op_t *) srcp)[0];
  76                 srcp -= 0 * OPSIZ;
  77                 dstp -= 1 * OPSIZ;
  78                 goto do7;
  79         case 1:
  80                 a1 = ((op_t *) srcp)[0];
  81                 srcp -=-1 * OPSIZ;
  82                 dstp -= 0 * OPSIZ;
  83                 len -= 1;
  84                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
  85                         goto do0;
  86                 goto do8;                       /* No-op.  */
  87         }
  88
  89         do {
  90 do8:
  91                 a0 = ((op_t *) srcp)[0];
  92                 ((op_t *) dstp)[0] = a1;
  93 do7:
  94                 a1 = ((op_t *) srcp)[1];
  95                 ((op_t *) dstp)[1] = a0;
  96 do6:
  97                 a0 = ((op_t *) srcp)[2];
  98                 ((op_t *) dstp)[2] = a1;
  99 do5:
 100                 a1 = ((op_t *) srcp)[3];
 101                 ((op_t *) dstp)[3] = a0;
 102 do4:
 103                 a0 = ((op_t *) srcp)[4];
 104                 ((op_t *) dstp)[4] = a1;
 105 do3:
 106                 a1 = ((op_t *) srcp)[5];
 107                 ((op_t *) dstp)[5] = a0;
 108 do2:
 109                 a0 = ((op_t *) srcp)[6];
 110                 ((op_t *) dstp)[6] = a1;
 111 do1:
 112                 a1 = ((op_t *) srcp)[7];
 113                 ((op_t *) dstp)[7] = a0;
 114
 115                 srcp += 8 * OPSIZ;
 116                 dstp += 8 * OPSIZ;
 117                 len -= 8;
 118         } while (len != 0);
 119
 120         /*
 121          * This is the right position for do0.  Please don't move it into
 122          * the loop.
 123          */
 124 do0:
 125         ((op_t *) dstp)[0] = a1;
 126 }
 127
 128 /*
 129  * _wordcopy_fwd_dest_aligned -- Copy block beginning at SRCP to block
 130  * beginning at DSTP with LEN `op_t' words (not LEN bytes!). DSTP should
 131  * be aligned for memory operations on `op_t's, but SRCP must *not* be aligned.
 132  */
 133
 134 void _wordcopy_fwd_dest_aligned (long int dstp, long int srcp, size_t len)
 135 {
 136         op_t a0, a1, a2, a3;
 137         int sh_1, sh_2;
 138
 139         /*
 140          * Calculate how to shift a word read at the memory operation aligned
 141          * srcp to make it aligned for copy.
 142          */
 143         sh_1 = 8 * (srcp % OPSIZ);
 144         sh_2 = 8 * OPSIZ - sh_1;
 145
 146         /*
 147          * Make SRCP aligned by rounding it down to the beginning of the `op_t'
 148          * it points in the middle of.
 149          */
 150         srcp &= -OPSIZ;
 151
 152         switch (len % 4) {
 153         case 2:
 154                 a1 = ((op_t *) srcp)[0];
 155                 a2 = ((op_t *) srcp)[1];
 156                 srcp -= 1 * OPSIZ;
 157                 dstp -= 3 * OPSIZ;
 158                 len += 2;
 159                 goto do1;
 160         case 3:
 161                 a0 = ((op_t *) srcp)[0];
 162                 a1 = ((op_t *) srcp)[1];
 163                 srcp -= 0 * OPSIZ;
 164                 dstp -= 2 * OPSIZ;
 165                 len += 1;
 166                 goto do2;
 167         case 0:
 168                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
 169                         return;
 170                 a3 = ((op_t *) srcp)[0];
 171                 a0 = ((op_t *) srcp)[1];
 172                 srcp -=-1 * OPSIZ;
 173                 dstp -= 1 * OPSIZ;
 174                 len += 0;
 175                 goto do3;
 176         case 1:
 177                 a2 = ((op_t *) srcp)[0];
 178                 a3 = ((op_t *) srcp)[1];
 179                 srcp -=-2 * OPSIZ;
 180                 dstp -= 0 * OPSIZ;
 181                 len -= 1;
 182                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
 183                         goto do0;
 184                 goto do4;                       /* No-op. */
 185         }
 186
 187         do {
 188 do4:
 189                 a0 = ((op_t *) srcp)[0];
 190                 ((op_t *) dstp)[0] = MERGE (a2, sh_1, a3, sh_2);
 191 do3:
 192                 a1 = ((op_t *) srcp)[1];
 193                 ((op_t *) dstp)[1] = MERGE (a3, sh_1, a0, sh_2);
 194 do2:
 195                 a2 = ((op_t *) srcp)[2];
 196                 ((op_t *) dstp)[2] = MERGE (a0, sh_1, a1, sh_2);
 197 do1:
 198                 a3 = ((op_t *) srcp)[3];
 199                 ((op_t *) dstp)[3] = MERGE (a1, sh_1, a2, sh_2);
 200
 201                 srcp += 4 * OPSIZ;
 202                 dstp += 4 * OPSIZ;
 203                 len -= 4;
 204         } while (len != 0);
 205
 206         /*
 207          * This is the right position for do0.  Please don't move it into
 208          * the loop.
 209          */
 210 do0:
 211         ((op_t *) dstp)[0] = MERGE (a2, sh_1, a3, sh_2);
 212 }
 213
 214 /*
 215  * _wordcopy_bwd_aligned -- Copy block finishing right before
 216  * SRCP to block finishing right before DSTP with LEN `op_t' words (not LEN
 217  * bytes!).  Both SRCP and DSTP should be aligned for memory operations
 218  * on `op_t's.
 219  */
 220 void _wordcopy_bwd_aligned (long int dstp, long int srcp, size_t len)
 221 {
 222         op_t a0, a1;
 223
 224         switch (len % 8) {
 225         case 2:
 226                 srcp -= 2 * OPSIZ;
 227                 dstp -= 1 * OPSIZ;
 228                 a0 = ((op_t *) srcp)[1];
 229                 len += 6;
 230                 goto do1;
 231         case 3:
 232                 srcp -= 3 * OPSIZ;
 233                 dstp -= 2 * OPSIZ;
 234                 a1 = ((op_t *) srcp)[2];
 235                 len += 5;
 236                 goto do2;
 237         case 4:
 238                 srcp -= 4 * OPSIZ;
 239                 dstp -= 3 * OPSIZ;
 240                 a0 = ((op_t *) srcp)[3];
 241                 len += 4;
 242                 goto do3;
 243         case 5:
 244                 srcp -= 5 * OPSIZ;
 245                 dstp -= 4 * OPSIZ;
 246                 a1 = ((op_t *) srcp)[4];
 247                 len += 3;
 248                 goto do4;
 249         case 6:
 250                 srcp -= 6 * OPSIZ;
 251                 dstp -= 5 * OPSIZ;
 252                 a0 = ((op_t *) srcp)[5];
 253                 len += 2;
 254                 goto do5;
 255         case 7:
 256                 srcp -= 7 * OPSIZ;
 257                 dstp -= 6 * OPSIZ;
 258                 a1 = ((op_t *) srcp)[6];
 259                 len += 1;
 260                 goto do6;
 261         case 0:
 262                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
 263                         return;
 264                 srcp -= 8 * OPSIZ;
 265                 dstp -= 7 * OPSIZ;
 266                 a0 = ((op_t *) srcp)[7];
 267                 goto do7;
 268         case 1:
 269                 srcp -= 9 * OPSIZ;
 270                 dstp -= 8 * OPSIZ;
 271                 a1 = ((op_t *) srcp)[8];
 272                 len -= 1;
 273                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
 274                         goto do0;
 275                 goto do8;                       /* No-op.  */
 276         }
 277
 278         do {
 279 do8:
 280                 a0 = ((op_t *) srcp)[7];
 281                 ((op_t *) dstp)[7] = a1;
 282 do7:
 283                 a1 = ((op_t *) srcp)[6];
 284                 ((op_t *) dstp)[6] = a0;
 285 do6:
 286                 a0 = ((op_t *) srcp)[5];
 287                 ((op_t *) dstp)[5] = a1;
 288 do5:
 289                 a1 = ((op_t *) srcp)[4];
 290                 ((op_t *) dstp)[4] = a0;
 291 do4:
 292                 a0 = ((op_t *) srcp)[3];
 293                 ((op_t *) dstp)[3] = a1;
 294 do3:
 295                 a1 = ((op_t *) srcp)[2];
 296                 ((op_t *) dstp)[2] = a0;
 297 do2:
 298                 a0 = ((op_t *) srcp)[1];
 299                 ((op_t *) dstp)[1] = a1;
 300 do1:
 301                 a1 = ((op_t *) srcp)[0];
 302                 ((op_t *) dstp)[0] = a0;
 303
 304                 srcp -= 8 * OPSIZ;
 305                 dstp -= 8 * OPSIZ;
 306                 len -= 8;
 307         } while (len != 0);
 308
 309         /*
 310          * This is the right position for do0.  Please don't move it into
 311          * the loop.
 312          */
 313 do0:
 314         ((op_t *) dstp)[7] = a1;
 315 }
 316
 317 /*
 318  * _wordcopy_bwd_dest_aligned -- Copy block finishing right before SRCP to
 319  * block finishing right before DSTP with LEN `op_t' words (not LEN bytes!).
 320  * DSTP should be aligned for memory operations on `op_t', but SRCP must *not*
 321  * be aligned.
 322  */
 323 void _wordcopy_bwd_dest_aligned (long int dstp, long int srcp, size_t len)
 324 {
 325         op_t a0, a1, a2, a3;
 326         int sh_1, sh_2;
 327
 328         /*
 329          * Calculate how to shift a word read at the memory operation aligned
 330          * srcp to make it aligned for copy.
 331          */
 332
 333         sh_1 = 8 * (srcp % OPSIZ);
 334         sh_2 = 8 * OPSIZ - sh_1;
 335
 336         /*
 337          * Make srcp aligned by rounding it down to the beginning of the op_t
 338          * it points in the middle of.
 339          */
 340         srcp &= -OPSIZ;
 341         srcp += OPSIZ;
 342
 343         switch (len % 4) {
 344         case 2:
 345                 srcp -= 3 * OPSIZ;
 346                 dstp -= 1 * OPSIZ;
 347                 a2 = ((op_t *) srcp)[2];
 348                 a1 = ((op_t *) srcp)[1];
 349                 len += 2;
 350                 goto do1;
 351         case 3:
 352                 srcp -= 4 * OPSIZ;
 353                 dstp -= 2 * OPSIZ;
 354                 a3 = ((op_t *) srcp)[3];
 355                 a2 = ((op_t *) srcp)[2];
 356                 len += 1;
 357                 goto do2;
 358         case 0:
 359                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
 360                         return;
 361                 srcp -= 5 * OPSIZ;
 362                 dstp -= 3 * OPSIZ;
 363                 a0 = ((op_t *) srcp)[4];
 364                 a3 = ((op_t *) srcp)[3];
 365                 goto do3;
 366         case 1:
 367                 srcp -= 6 * OPSIZ;
 368                 dstp -= 4 * OPSIZ;
 369                 a1 = ((op_t *) srcp)[5];
 370                 a0 = ((op_t *) srcp)[4];
 371                 len -= 1;
 372                 if (OP_T_THRESHOLD <= 3 * OPSIZ && len == 0)
 373                         goto do0;
 374                 goto do4;                       /* No-op.  */
 375         }
 376
 377         do {
 378 do4:
 379                 a3 = ((op_t *) srcp)[3];
 380                 ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2);
 381 do3:
 382                 a2 = ((op_t *) srcp)[2];
 383                 ((op_t *) dstp)[2] = MERGE (a3, sh_1, a0, sh_2);
 384 do2:
 385                 a1 = ((op_t *) srcp)[1];
 386                 ((op_t *) dstp)[1] = MERGE (a2, sh_1, a3, sh_2);
 387 do1:
 388                 a0 = ((op_t *) srcp)[0];
 389                 ((op_t *) dstp)[0] = MERGE (a1, sh_1, a2, sh_2);
 390
 391                 srcp -= 4 * OPSIZ;
 392                 dstp -= 4 * OPSIZ;
 393                 len -= 4;
 394         } while (len != 0);
 395
 396         /*
 397          * This is the right position for do0.  Please don't move it into
 398          * the loop.
 399          */
 400 do0:
 401         ((op_t *) dstp)[3] = MERGE (a0, sh_1, a1, sh_2);
 402 }
 403
 404