arch/powerpc/lib/copyuser_power7.S

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  *
   4  * Copyright (C) IBM Corporation, 2011
   5  *
   6  * Author: Anton Blanchard <anton@au.ibm.com>
   7  */
   8 #include <asm/ppc_asm.h>
   9
  10 #ifndef SELFTEST_CASE
  11 /* 0 == don't use VMX, 1 == use VMX */
  12 #define SELFTEST_CASE   0
  13 #endif
  14
  15 #ifdef __BIG_ENDIAN__
  16 #define LVS(VRT,RA,RB)          lvsl    VRT,RA,RB
  17 #define VPERM(VRT,VRA,VRB,VRC)  vperm   VRT,VRA,VRB,VRC
  18 #else
  19 #define LVS(VRT,RA,RB)          lvsr    VRT,RA,RB
  20 #define VPERM(VRT,VRA,VRB,VRC)  vperm   VRT,VRB,VRA,VRC
  21 #endif
  22
  23         .macro err1
  24 100:
  25         EX_TABLE(100b,.Ldo_err1)
  26         .endm
  27
  28         .macro err2
  29 200:
  30         EX_TABLE(200b,.Ldo_err2)
  31         .endm
  32
  33 #ifdef CONFIG_ALTIVEC
  34         .macro err3
  35 300:
  36         EX_TABLE(300b,.Ldo_err3)
  37         .endm
  38
  39         .macro err4
  40 400:
  41         EX_TABLE(400b,.Ldo_err4)
  42         .endm
  43
  44
  45 .Ldo_err4:
  46         ld      r16,STK_REG(R16)(r1)
  47         ld      r15,STK_REG(R15)(r1)
  48         ld      r14,STK_REG(R14)(r1)
  49 .Ldo_err3:
  50         bl      CFUNC(exit_vmx_usercopy)
  51         ld      r0,STACKFRAMESIZE+16(r1)
  52         mtlr    r0
  53         b       .Lexit
  54 #endif /* CONFIG_ALTIVEC */
  55
  56 .Ldo_err2:
  57         ld      r22,STK_REG(R22)(r1)
  58         ld      r21,STK_REG(R21)(r1)
  59         ld      r20,STK_REG(R20)(r1)
  60         ld      r19,STK_REG(R19)(r1)
  61         ld      r18,STK_REG(R18)(r1)
  62         ld      r17,STK_REG(R17)(r1)
  63         ld      r16,STK_REG(R16)(r1)
  64         ld      r15,STK_REG(R15)(r1)
  65         ld      r14,STK_REG(R14)(r1)
  66 .Lexit:
  67         addi    r1,r1,STACKFRAMESIZE
  68 .Ldo_err1:
  69         ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
  70         ld      r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
  71         ld      r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
  72         b       __copy_tofrom_user_base
  73
  74
  75 _GLOBAL(__copy_tofrom_user_power7)
  76         cmpldi  r5,16
  77         cmpldi  cr1,r5,3328
  78
  79         std     r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
  80         std     r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
  81         std     r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
  82
  83         blt     .Lshort_copy
  84
  85 #ifdef CONFIG_ALTIVEC
  86 test_feature = SELFTEST_CASE
  87 BEGIN_FTR_SECTION
  88         bgt     cr1,.Lvmx_copy
  89 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  90 #endif
  91
  92 .Lnonvmx_copy:
  93         /* Get the source 8B aligned */
  94         neg     r6,r4
  95         mtocrf  0x01,r6
  96         clrldi  r6,r6,(64-3)
  97
  98         bf      cr7*4+3,1f
  99 err1;   lbz     r0,0(r4)
 100         addi    r4,r4,1
 101 err1;   stb     r0,0(r3)
 102         addi    r3,r3,1
 103
 104 1:      bf      cr7*4+2,2f
 105 err1;   lhz     r0,0(r4)
 106         addi    r4,r4,2
 107 err1;   sth     r0,0(r3)
 108         addi    r3,r3,2
 109
 110 2:      bf      cr7*4+1,3f
 111 err1;   lwz     r0,0(r4)
 112         addi    r4,r4,4
 113 err1;   stw     r0,0(r3)
 114         addi    r3,r3,4
 115
 116 3:      sub     r5,r5,r6
 117         cmpldi  r5,128
 118         blt     5f
 119
 120         mflr    r0
 121         stdu    r1,-STACKFRAMESIZE(r1)
 122         std     r14,STK_REG(R14)(r1)
 123         std     r15,STK_REG(R15)(r1)
 124         std     r16,STK_REG(R16)(r1)
 125         std     r17,STK_REG(R17)(r1)
 126         std     r18,STK_REG(R18)(r1)
 127         std     r19,STK_REG(R19)(r1)
 128         std     r20,STK_REG(R20)(r1)
 129         std     r21,STK_REG(R21)(r1)
 130         std     r22,STK_REG(R22)(r1)
 131         std     r0,STACKFRAMESIZE+16(r1)
 132
 133         srdi    r6,r5,7
 134         mtctr   r6
 135
 136         /* Now do cacheline (128B) sized loads and stores. */
 137         .align  5
 138 4:
 139 err2;   ld      r0,0(r4)
 140 err2;   ld      r6,8(r4)
 141 err2;   ld      r7,16(r4)
 142 err2;   ld      r8,24(r4)
 143 err2;   ld      r9,32(r4)
 144 err2;   ld      r10,40(r4)
 145 err2;   ld      r11,48(r4)
 146 err2;   ld      r12,56(r4)
 147 err2;   ld      r14,64(r4)
 148 err2;   ld      r15,72(r4)
 149 err2;   ld      r16,80(r4)
 150 err2;   ld      r17,88(r4)
 151 err2;   ld      r18,96(r4)
 152 err2;   ld      r19,104(r4)
 153 err2;   ld      r20,112(r4)
 154 err2;   ld      r21,120(r4)
 155         addi    r4,r4,128
 156 err2;   std     r0,0(r3)
 157 err2;   std     r6,8(r3)
 158 err2;   std     r7,16(r3)
 159 err2;   std     r8,24(r3)
 160 err2;   std     r9,32(r3)
 161 err2;   std     r10,40(r3)
 162 err2;   std     r11,48(r3)
 163 err2;   std     r12,56(r3)
 164 err2;   std     r14,64(r3)
 165 err2;   std     r15,72(r3)
 166 err2;   std     r16,80(r3)
 167 err2;   std     r17,88(r3)
 168 err2;   std     r18,96(r3)
 169 err2;   std     r19,104(r3)
 170 err2;   std     r20,112(r3)
 171 err2;   std     r21,120(r3)
 172         addi    r3,r3,128
 173         bdnz    4b
 174
 175         clrldi  r5,r5,(64-7)
 176
 177         ld      r14,STK_REG(R14)(r1)
 178         ld      r15,STK_REG(R15)(r1)
 179         ld      r16,STK_REG(R16)(r1)
 180         ld      r17,STK_REG(R17)(r1)
 181         ld      r18,STK_REG(R18)(r1)
 182         ld      r19,STK_REG(R19)(r1)
 183         ld      r20,STK_REG(R20)(r1)
 184         ld      r21,STK_REG(R21)(r1)
 185         ld      r22,STK_REG(R22)(r1)
 186         addi    r1,r1,STACKFRAMESIZE
 187
 188         /* Up to 127B to go */
 189 5:      srdi    r6,r5,4
 190         mtocrf  0x01,r6
 191
 192 6:      bf      cr7*4+1,7f
 193 err1;   ld      r0,0(r4)
 194 err1;   ld      r6,8(r4)
 195 err1;   ld      r7,16(r4)
 196 err1;   ld      r8,24(r4)
 197 err1;   ld      r9,32(r4)
 198 err1;   ld      r10,40(r4)
 199 err1;   ld      r11,48(r4)
 200 err1;   ld      r12,56(r4)
 201         addi    r4,r4,64
 202 err1;   std     r0,0(r3)
 203 err1;   std     r6,8(r3)
 204 err1;   std     r7,16(r3)
 205 err1;   std     r8,24(r3)
 206 err1;   std     r9,32(r3)
 207 err1;   std     r10,40(r3)
 208 err1;   std     r11,48(r3)
 209 err1;   std     r12,56(r3)
 210         addi    r3,r3,64
 211
 212         /* Up to 63B to go */
 213 7:      bf      cr7*4+2,8f
 214 err1;   ld      r0,0(r4)
 215 err1;   ld      r6,8(r4)
 216 err1;   ld      r7,16(r4)
 217 err1;   ld      r8,24(r4)
 218         addi    r4,r4,32
 219 err1;   std     r0,0(r3)
 220 err1;   std     r6,8(r3)
 221 err1;   std     r7,16(r3)
 222 err1;   std     r8,24(r3)
 223         addi    r3,r3,32
 224
 225         /* Up to 31B to go */
 226 8:      bf      cr7*4+3,9f
 227 err1;   ld      r0,0(r4)
 228 err1;   ld      r6,8(r4)
 229         addi    r4,r4,16
 230 err1;   std     r0,0(r3)
 231 err1;   std     r6,8(r3)
 232         addi    r3,r3,16
 233
 234 9:      clrldi  r5,r5,(64-4)
 235
 236         /* Up to 15B to go */
 237 .Lshort_copy:
 238         mtocrf  0x01,r5
 239         bf      cr7*4+0,12f
 240 err1;   lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 241 err1;   lwz     r6,4(r4)
 242         addi    r4,r4,8
 243 err1;   stw     r0,0(r3)
 244 err1;   stw     r6,4(r3)
 245         addi    r3,r3,8
 246
 247 12:     bf      cr7*4+1,13f
 248 err1;   lwz     r0,0(r4)
 249         addi    r4,r4,4
 250 err1;   stw     r0,0(r3)
 251         addi    r3,r3,4
 252
 253 13:     bf      cr7*4+2,14f
 254 err1;   lhz     r0,0(r4)
 255         addi    r4,r4,2
 256 err1;   sth     r0,0(r3)
 257         addi    r3,r3,2
 258
 259 14:     bf      cr7*4+3,15f
 260 err1;   lbz     r0,0(r4)
 261 err1;   stb     r0,0(r3)
 262
 263 15:     li      r3,0
 264         blr
 265
 266 .Lunwind_stack_nonvmx_copy:
 267         addi    r1,r1,STACKFRAMESIZE
 268         b       .Lnonvmx_copy
 269
 270 .Lvmx_copy:
 271 #ifdef CONFIG_ALTIVEC
 272         mflr    r0
 273         std     r0,16(r1)
 274         stdu    r1,-STACKFRAMESIZE(r1)
 275         bl      CFUNC(enter_vmx_usercopy)
 276         cmpwi   cr1,r3,0
 277         ld      r0,STACKFRAMESIZE+16(r1)
 278         ld      r3,STK_REG(R31)(r1)
 279         ld      r4,STK_REG(R30)(r1)
 280         ld      r5,STK_REG(R29)(r1)
 281         mtlr    r0
 282
 283         /*
 284          * We prefetch both the source and destination using enhanced touch
 285          * instructions. We use a stream ID of 0 for the load side and
 286          * 1 for the store side.
 287          */
 288         clrrdi  r6,r4,7
 289         clrrdi  r9,r3,7
 290         ori     r9,r9,1         /* stream=1 */
 291
 292         srdi    r7,r5,7         /* length in cachelines, capped at 0x3FF */
 293         cmpldi  r7,0x3FF
 294         ble     1f
 295         li      r7,0x3FF
 296 1:      lis     r0,0x0E00       /* depth=7 */
 297         sldi    r7,r7,7
 298         or      r7,r7,r0
 299         ori     r10,r7,1        /* stream=1 */
 300
 301         DCBT_SETUP_STREAMS(r6, r7, r9, r10, r8)
 302
 303         beq     cr1,.Lunwind_stack_nonvmx_copy
 304
 305         /*
 306          * If source and destination are not relatively aligned we use a
 307          * slower permute loop.
 308          */
 309         xor     r6,r4,r3
 310         rldicl. r6,r6,0,(64-4)
 311         bne     .Lvmx_unaligned_copy
 312
 313         /* Get the destination 16B aligned */
 314         neg     r6,r3
 315         mtocrf  0x01,r6
 316         clrldi  r6,r6,(64-4)
 317
 318         bf      cr7*4+3,1f
 319 err3;   lbz     r0,0(r4)
 320         addi    r4,r4,1
 321 err3;   stb     r0,0(r3)
 322         addi    r3,r3,1
 323
 324 1:      bf      cr7*4+2,2f
 325 err3;   lhz     r0,0(r4)
 326         addi    r4,r4,2
 327 err3;   sth     r0,0(r3)
 328         addi    r3,r3,2
 329
 330 2:      bf      cr7*4+1,3f
 331 err3;   lwz     r0,0(r4)
 332         addi    r4,r4,4
 333 err3;   stw     r0,0(r3)
 334         addi    r3,r3,4
 335
 336 3:      bf      cr7*4+0,4f
 337 err3;   ld      r0,0(r4)
 338         addi    r4,r4,8
 339 err3;   std     r0,0(r3)
 340         addi    r3,r3,8
 341
 342 4:      sub     r5,r5,r6
 343
 344         /* Get the desination 128B aligned */
 345         neg     r6,r3
 346         srdi    r7,r6,4
 347         mtocrf  0x01,r7
 348         clrldi  r6,r6,(64-7)
 349
 350         li      r9,16
 351         li      r10,32
 352         li      r11,48
 353
 354         bf      cr7*4+3,5f
 355 err3;   lvx     v1,0,r4
 356         addi    r4,r4,16
 357 err3;   stvx    v1,0,r3
 358         addi    r3,r3,16
 359
 360 5:      bf      cr7*4+2,6f
 361 err3;   lvx     v1,0,r4
 362 err3;   lvx     v0,r4,r9
 363         addi    r4,r4,32
 364 err3;   stvx    v1,0,r3
 365 err3;   stvx    v0,r3,r9
 366         addi    r3,r3,32
 367
 368 6:      bf      cr7*4+1,7f
 369 err3;   lvx     v3,0,r4
 370 err3;   lvx     v2,r4,r9
 371 err3;   lvx     v1,r4,r10
 372 err3;   lvx     v0,r4,r11
 373         addi    r4,r4,64
 374 err3;   stvx    v3,0,r3
 375 err3;   stvx    v2,r3,r9
 376 err3;   stvx    v1,r3,r10
 377 err3;   stvx    v0,r3,r11
 378         addi    r3,r3,64
 379
 380 7:      sub     r5,r5,r6
 381         srdi    r6,r5,7
 382
 383         std     r14,STK_REG(R14)(r1)
 384         std     r15,STK_REG(R15)(r1)
 385         std     r16,STK_REG(R16)(r1)
 386
 387         li      r12,64
 388         li      r14,80
 389         li      r15,96
 390         li      r16,112
 391
 392         mtctr   r6
 393
 394         /*
 395          * Now do cacheline sized loads and stores. By this stage the
 396          * cacheline stores are also cacheline aligned.
 397          */
 398         .align  5
 399 8:
 400 err4;   lvx     v7,0,r4
 401 err4;   lvx     v6,r4,r9
 402 err4;   lvx     v5,r4,r10
 403 err4;   lvx     v4,r4,r11
 404 err4;   lvx     v3,r4,r12
 405 err4;   lvx     v2,r4,r14
 406 err4;   lvx     v1,r4,r15
 407 err4;   lvx     v0,r4,r16
 408         addi    r4,r4,128
 409 err4;   stvx    v7,0,r3
 410 err4;   stvx    v6,r3,r9
 411 err4;   stvx    v5,r3,r10
 412 err4;   stvx    v4,r3,r11
 413 err4;   stvx    v3,r3,r12
 414 err4;   stvx    v2,r3,r14
 415 err4;   stvx    v1,r3,r15
 416 err4;   stvx    v0,r3,r16
 417         addi    r3,r3,128
 418         bdnz    8b
 419
 420         ld      r14,STK_REG(R14)(r1)
 421         ld      r15,STK_REG(R15)(r1)
 422         ld      r16,STK_REG(R16)(r1)
 423
 424         /* Up to 127B to go */
 425         clrldi  r5,r5,(64-7)
 426         srdi    r6,r5,4
 427         mtocrf  0x01,r6
 428
 429         bf      cr7*4+1,9f
 430 err3;   lvx     v3,0,r4
 431 err3;   lvx     v2,r4,r9
 432 err3;   lvx     v1,r4,r10
 433 err3;   lvx     v0,r4,r11
 434         addi    r4,r4,64
 435 err3;   stvx    v3,0,r3
 436 err3;   stvx    v2,r3,r9
 437 err3;   stvx    v1,r3,r10
 438 err3;   stvx    v0,r3,r11
 439         addi    r3,r3,64
 440
 441 9:      bf      cr7*4+2,10f
 442 err3;   lvx     v1,0,r4
 443 err3;   lvx     v0,r4,r9
 444         addi    r4,r4,32
 445 err3;   stvx    v1,0,r3
 446 err3;   stvx    v0,r3,r9
 447         addi    r3,r3,32
 448
 449 10:     bf      cr7*4+3,11f
 450 err3;   lvx     v1,0,r4
 451         addi    r4,r4,16
 452 err3;   stvx    v1,0,r3
 453         addi    r3,r3,16
 454
 455         /* Up to 15B to go */
 456 11:     clrldi  r5,r5,(64-4)
 457         mtocrf  0x01,r5
 458         bf      cr7*4+0,12f
 459 err3;   ld      r0,0(r4)
 460         addi    r4,r4,8
 461 err3;   std     r0,0(r3)
 462         addi    r3,r3,8
 463
 464 12:     bf      cr7*4+1,13f
 465 err3;   lwz     r0,0(r4)
 466         addi    r4,r4,4
 467 err3;   stw     r0,0(r3)
 468         addi    r3,r3,4
 469
 470 13:     bf      cr7*4+2,14f
 471 err3;   lhz     r0,0(r4)
 472         addi    r4,r4,2
 473 err3;   sth     r0,0(r3)
 474         addi    r3,r3,2
 475
 476 14:     bf      cr7*4+3,15f
 477 err3;   lbz     r0,0(r4)
 478 err3;   stb     r0,0(r3)
 479
 480 15:     addi    r1,r1,STACKFRAMESIZE
 481         b       CFUNC(exit_vmx_usercopy)        /* tail call optimise */
 482
 483 .Lvmx_unaligned_copy:
 484         /* Get the destination 16B aligned */
 485         neg     r6,r3
 486         mtocrf  0x01,r6
 487         clrldi  r6,r6,(64-4)
 488
 489         bf      cr7*4+3,1f
 490 err3;   lbz     r0,0(r4)
 491         addi    r4,r4,1
 492 err3;   stb     r0,0(r3)
 493         addi    r3,r3,1
 494
 495 1:      bf      cr7*4+2,2f
 496 err3;   lhz     r0,0(r4)
 497         addi    r4,r4,2
 498 err3;   sth     r0,0(r3)
 499         addi    r3,r3,2
 500
 501 2:      bf      cr7*4+1,3f
 502 err3;   lwz     r0,0(r4)
 503         addi    r4,r4,4
 504 err3;   stw     r0,0(r3)
 505         addi    r3,r3,4
 506
 507 3:      bf      cr7*4+0,4f
 508 err3;   lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 509 err3;   lwz     r7,4(r4)
 510         addi    r4,r4,8
 511 err3;   stw     r0,0(r3)
 512 err3;   stw     r7,4(r3)
 513         addi    r3,r3,8
 514
 515 4:      sub     r5,r5,r6
 516
 517         /* Get the desination 128B aligned */
 518         neg     r6,r3
 519         srdi    r7,r6,4
 520         mtocrf  0x01,r7
 521         clrldi  r6,r6,(64-7)
 522
 523         li      r9,16
 524         li      r10,32
 525         li      r11,48
 526
 527         LVS(v16,0,r4)           /* Setup permute control vector */
 528 err3;   lvx     v0,0,r4
 529         addi    r4,r4,16
 530
 531         bf      cr7*4+3,5f
 532 err3;   lvx     v1,0,r4
 533         VPERM(v8,v0,v1,v16)
 534         addi    r4,r4,16
 535 err3;   stvx    v8,0,r3
 536         addi    r3,r3,16
 537         vor     v0,v1,v1
 538
 539 5:      bf      cr7*4+2,6f
 540 err3;   lvx     v1,0,r4
 541         VPERM(v8,v0,v1,v16)
 542 err3;   lvx     v0,r4,r9
 543         VPERM(v9,v1,v0,v16)
 544         addi    r4,r4,32
 545 err3;   stvx    v8,0,r3
 546 err3;   stvx    v9,r3,r9
 547         addi    r3,r3,32
 548
 549 6:      bf      cr7*4+1,7f
 550 err3;   lvx     v3,0,r4
 551         VPERM(v8,v0,v3,v16)
 552 err3;   lvx     v2,r4,r9
 553         VPERM(v9,v3,v2,v16)
 554 err3;   lvx     v1,r4,r10
 555         VPERM(v10,v2,v1,v16)
 556 err3;   lvx     v0,r4,r11
 557         VPERM(v11,v1,v0,v16)
 558         addi    r4,r4,64
 559 err3;   stvx    v8,0,r3
 560 err3;   stvx    v9,r3,r9
 561 err3;   stvx    v10,r3,r10
 562 err3;   stvx    v11,r3,r11
 563         addi    r3,r3,64
 564
 565 7:      sub     r5,r5,r6
 566         srdi    r6,r5,7
 567
 568         std     r14,STK_REG(R14)(r1)
 569         std     r15,STK_REG(R15)(r1)
 570         std     r16,STK_REG(R16)(r1)
 571
 572         li      r12,64
 573         li      r14,80
 574         li      r15,96
 575         li      r16,112
 576
 577         mtctr   r6
 578
 579         /*
 580          * Now do cacheline sized loads and stores. By this stage the
 581          * cacheline stores are also cacheline aligned.
 582          */
 583         .align  5
 584 8:
 585 err4;   lvx     v7,0,r4
 586         VPERM(v8,v0,v7,v16)
 587 err4;   lvx     v6,r4,r9
 588         VPERM(v9,v7,v6,v16)
 589 err4;   lvx     v5,r4,r10
 590         VPERM(v10,v6,v5,v16)
 591 err4;   lvx     v4,r4,r11
 592         VPERM(v11,v5,v4,v16)
 593 err4;   lvx     v3,r4,r12
 594         VPERM(v12,v4,v3,v16)
 595 err4;   lvx     v2,r4,r14
 596         VPERM(v13,v3,v2,v16)
 597 err4;   lvx     v1,r4,r15
 598         VPERM(v14,v2,v1,v16)
 599 err4;   lvx     v0,r4,r16
 600         VPERM(v15,v1,v0,v16)
 601         addi    r4,r4,128
 602 err4;   stvx    v8,0,r3
 603 err4;   stvx    v9,r3,r9
 604 err4;   stvx    v10,r3,r10
 605 err4;   stvx    v11,r3,r11
 606 err4;   stvx    v12,r3,r12
 607 err4;   stvx    v13,r3,r14
 608 err4;   stvx    v14,r3,r15
 609 err4;   stvx    v15,r3,r16
 610         addi    r3,r3,128
 611         bdnz    8b
 612
 613         ld      r14,STK_REG(R14)(r1)
 614         ld      r15,STK_REG(R15)(r1)
 615         ld      r16,STK_REG(R16)(r1)
 616
 617         /* Up to 127B to go */
 618         clrldi  r5,r5,(64-7)
 619         srdi    r6,r5,4
 620         mtocrf  0x01,r6
 621
 622         bf      cr7*4+1,9f
 623 err3;   lvx     v3,0,r4
 624         VPERM(v8,v0,v3,v16)
 625 err3;   lvx     v2,r4,r9
 626         VPERM(v9,v3,v2,v16)
 627 err3;   lvx     v1,r4,r10
 628         VPERM(v10,v2,v1,v16)
 629 err3;   lvx     v0,r4,r11
 630         VPERM(v11,v1,v0,v16)
 631         addi    r4,r4,64
 632 err3;   stvx    v8,0,r3
 633 err3;   stvx    v9,r3,r9
 634 err3;   stvx    v10,r3,r10
 635 err3;   stvx    v11,r3,r11
 636         addi    r3,r3,64
 637
 638 9:      bf      cr7*4+2,10f
 639 err3;   lvx     v1,0,r4
 640         VPERM(v8,v0,v1,v16)
 641 err3;   lvx     v0,r4,r9
 642         VPERM(v9,v1,v0,v16)
 643         addi    r4,r4,32
 644 err3;   stvx    v8,0,r3
 645 err3;   stvx    v9,r3,r9
 646         addi    r3,r3,32
 647
 648 10:     bf      cr7*4+3,11f
 649 err3;   lvx     v1,0,r4
 650         VPERM(v8,v0,v1,v16)
 651         addi    r4,r4,16
 652 err3;   stvx    v8,0,r3
 653         addi    r3,r3,16
 654
 655         /* Up to 15B to go */
 656 11:     clrldi  r5,r5,(64-4)
 657         addi    r4,r4,-16       /* Unwind the +16 load offset */
 658         mtocrf  0x01,r5
 659         bf      cr7*4+0,12f
 660 err3;   lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 661 err3;   lwz     r6,4(r4)
 662         addi    r4,r4,8
 663 err3;   stw     r0,0(r3)
 664 err3;   stw     r6,4(r3)
 665         addi    r3,r3,8
 666
 667 12:     bf      cr7*4+1,13f
 668 err3;   lwz     r0,0(r4)
 669         addi    r4,r4,4
 670 err3;   stw     r0,0(r3)
 671         addi    r3,r3,4
 672
 673 13:     bf      cr7*4+2,14f
 674 err3;   lhz     r0,0(r4)
 675         addi    r4,r4,2
 676 err3;   sth     r0,0(r3)
 677         addi    r3,r3,2
 678
 679 14:     bf      cr7*4+3,15f
 680 err3;   lbz     r0,0(r4)
 681 err3;   stb     r0,0(r3)
 682
 683 15:     addi    r1,r1,STACKFRAMESIZE
 684         b       CFUNC(exit_vmx_usercopy)        /* tail call optimise */
 685 #endif /* CONFIG_ALTIVEC */