arch/powerpc/lib/memcpy_power7.S

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  *
   4  * Copyright (C) IBM Corporation, 2012
   5  *
   6  * Author: Anton Blanchard <anton@au.ibm.com>
   7  */
   8 #include <asm/ppc_asm.h>
   9
  10 #ifndef SELFTEST_CASE
  11 /* 0 == don't use VMX, 1 == use VMX */
  12 #define SELFTEST_CASE   0
  13 #endif
  14
  15 #ifdef __BIG_ENDIAN__
  16 #define LVS(VRT,RA,RB)          lvsl    VRT,RA,RB
  17 #define VPERM(VRT,VRA,VRB,VRC)  vperm   VRT,VRA,VRB,VRC
  18 #else
  19 #define LVS(VRT,RA,RB)          lvsr    VRT,RA,RB
  20 #define VPERM(VRT,VRA,VRB,VRC)  vperm   VRT,VRB,VRA,VRC
  21 #endif
  22
  23 _GLOBAL(memcpy_power7)
  24         cmpldi  r5,16
  25         cmpldi  cr1,r5,4096
  26         std     r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
  27         blt     .Lshort_copy
  28
  29 #ifdef CONFIG_ALTIVEC
  30 test_feature = SELFTEST_CASE
  31 BEGIN_FTR_SECTION
  32         bgt     cr1, .Lvmx_copy
  33 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
  34 #endif
  35
  36 .Lnonvmx_copy:
  37         /* Get the source 8B aligned */
  38         neg     r6,r4
  39         mtocrf  0x01,r6
  40         clrldi  r6,r6,(64-3)
  41
  42         bf      cr7*4+3,1f
  43         lbz     r0,0(r4)
  44         addi    r4,r4,1
  45         stb     r0,0(r3)
  46         addi    r3,r3,1
  47
  48 1:      bf      cr7*4+2,2f
  49         lhz     r0,0(r4)
  50         addi    r4,r4,2
  51         sth     r0,0(r3)
  52         addi    r3,r3,2
  53
  54 2:      bf      cr7*4+1,3f
  55         lwz     r0,0(r4)
  56         addi    r4,r4,4
  57         stw     r0,0(r3)
  58         addi    r3,r3,4
  59
  60 3:      sub     r5,r5,r6
  61         cmpldi  r5,128
  62         blt     5f
  63
  64         mflr    r0
  65         stdu    r1,-STACKFRAMESIZE(r1)
  66         std     r14,STK_REG(R14)(r1)
  67         std     r15,STK_REG(R15)(r1)
  68         std     r16,STK_REG(R16)(r1)
  69         std     r17,STK_REG(R17)(r1)
  70         std     r18,STK_REG(R18)(r1)
  71         std     r19,STK_REG(R19)(r1)
  72         std     r20,STK_REG(R20)(r1)
  73         std     r21,STK_REG(R21)(r1)
  74         std     r22,STK_REG(R22)(r1)
  75         std     r0,STACKFRAMESIZE+16(r1)
  76
  77         srdi    r6,r5,7
  78         mtctr   r6
  79
  80         /* Now do cacheline (128B) sized loads and stores. */
  81         .align  5
  82 4:
  83         ld      r0,0(r4)
  84         ld      r6,8(r4)
  85         ld      r7,16(r4)
  86         ld      r8,24(r4)
  87         ld      r9,32(r4)
  88         ld      r10,40(r4)
  89         ld      r11,48(r4)
  90         ld      r12,56(r4)
  91         ld      r14,64(r4)
  92         ld      r15,72(r4)
  93         ld      r16,80(r4)
  94         ld      r17,88(r4)
  95         ld      r18,96(r4)
  96         ld      r19,104(r4)
  97         ld      r20,112(r4)
  98         ld      r21,120(r4)
  99         addi    r4,r4,128
 100         std     r0,0(r3)
 101         std     r6,8(r3)
 102         std     r7,16(r3)
 103         std     r8,24(r3)
 104         std     r9,32(r3)
 105         std     r10,40(r3)
 106         std     r11,48(r3)
 107         std     r12,56(r3)
 108         std     r14,64(r3)
 109         std     r15,72(r3)
 110         std     r16,80(r3)
 111         std     r17,88(r3)
 112         std     r18,96(r3)
 113         std     r19,104(r3)
 114         std     r20,112(r3)
 115         std     r21,120(r3)
 116         addi    r3,r3,128
 117         bdnz    4b
 118
 119         clrldi  r5,r5,(64-7)
 120
 121         ld      r14,STK_REG(R14)(r1)
 122         ld      r15,STK_REG(R15)(r1)
 123         ld      r16,STK_REG(R16)(r1)
 124         ld      r17,STK_REG(R17)(r1)
 125         ld      r18,STK_REG(R18)(r1)
 126         ld      r19,STK_REG(R19)(r1)
 127         ld      r20,STK_REG(R20)(r1)
 128         ld      r21,STK_REG(R21)(r1)
 129         ld      r22,STK_REG(R22)(r1)
 130         addi    r1,r1,STACKFRAMESIZE
 131
 132         /* Up to 127B to go */
 133 5:      srdi    r6,r5,4
 134         mtocrf  0x01,r6
 135
 136 6:      bf      cr7*4+1,7f
 137         ld      r0,0(r4)
 138         ld      r6,8(r4)
 139         ld      r7,16(r4)
 140         ld      r8,24(r4)
 141         ld      r9,32(r4)
 142         ld      r10,40(r4)
 143         ld      r11,48(r4)
 144         ld      r12,56(r4)
 145         addi    r4,r4,64
 146         std     r0,0(r3)
 147         std     r6,8(r3)
 148         std     r7,16(r3)
 149         std     r8,24(r3)
 150         std     r9,32(r3)
 151         std     r10,40(r3)
 152         std     r11,48(r3)
 153         std     r12,56(r3)
 154         addi    r3,r3,64
 155
 156         /* Up to 63B to go */
 157 7:      bf      cr7*4+2,8f
 158         ld      r0,0(r4)
 159         ld      r6,8(r4)
 160         ld      r7,16(r4)
 161         ld      r8,24(r4)
 162         addi    r4,r4,32
 163         std     r0,0(r3)
 164         std     r6,8(r3)
 165         std     r7,16(r3)
 166         std     r8,24(r3)
 167         addi    r3,r3,32
 168
 169         /* Up to 31B to go */
 170 8:      bf      cr7*4+3,9f
 171         ld      r0,0(r4)
 172         ld      r6,8(r4)
 173         addi    r4,r4,16
 174         std     r0,0(r3)
 175         std     r6,8(r3)
 176         addi    r3,r3,16
 177
 178 9:      clrldi  r5,r5,(64-4)
 179
 180         /* Up to 15B to go */
 181 .Lshort_copy:
 182         mtocrf  0x01,r5
 183         bf      cr7*4+0,12f
 184         lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 185         lwz     r6,4(r4)
 186         addi    r4,r4,8
 187         stw     r0,0(r3)
 188         stw     r6,4(r3)
 189         addi    r3,r3,8
 190
 191 12:     bf      cr7*4+1,13f
 192         lwz     r0,0(r4)
 193         addi    r4,r4,4
 194         stw     r0,0(r3)
 195         addi    r3,r3,4
 196
 197 13:     bf      cr7*4+2,14f
 198         lhz     r0,0(r4)
 199         addi    r4,r4,2
 200         sth     r0,0(r3)
 201         addi    r3,r3,2
 202
 203 14:     bf      cr7*4+3,15f
 204         lbz     r0,0(r4)
 205         stb     r0,0(r3)
 206
 207 15:     ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
 208         blr
 209
 210 .Lunwind_stack_nonvmx_copy:
 211         addi    r1,r1,STACKFRAMESIZE
 212         b       .Lnonvmx_copy
 213
 214 .Lvmx_copy:
 215 #ifdef CONFIG_ALTIVEC
 216         mflr    r0
 217         std     r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
 218         std     r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
 219         std     r0,16(r1)
 220         stdu    r1,-STACKFRAMESIZE(r1)
 221         bl      CFUNC(enter_vmx_ops)
 222         cmpwi   cr1,r3,0
 223         ld      r0,STACKFRAMESIZE+16(r1)
 224         ld      r3,STK_REG(R31)(r1)
 225         ld      r4,STK_REG(R30)(r1)
 226         ld      r5,STK_REG(R29)(r1)
 227         mtlr    r0
 228
 229         /*
 230          * We prefetch both the source and destination using enhanced touch
 231          * instructions. We use a stream ID of 0 for the load side and
 232          * 1 for the store side.
 233          */
 234         clrrdi  r6,r4,7
 235         clrrdi  r9,r3,7
 236         ori     r9,r9,1         /* stream=1 */
 237
 238         srdi    r7,r5,7         /* length in cachelines, capped at 0x3FF */
 239         cmpldi  r7,0x3FF
 240         ble     1f
 241         li      r7,0x3FF
 242 1:      lis     r0,0x0E00       /* depth=7 */
 243         sldi    r7,r7,7
 244         or      r7,r7,r0
 245         ori     r10,r7,1        /* stream=1 */
 246
 247         DCBT_SETUP_STREAMS(r6, r7, r9, r10, r8)
 248
 249         beq     cr1,.Lunwind_stack_nonvmx_copy
 250
 251         /*
 252          * If source and destination are not relatively aligned we use a
 253          * slower permute loop.
 254          */
 255         xor     r6,r4,r3
 256         rldicl. r6,r6,0,(64-4)
 257         bne     .Lvmx_unaligned_copy
 258
 259         /* Get the destination 16B aligned */
 260         neg     r6,r3
 261         mtocrf  0x01,r6
 262         clrldi  r6,r6,(64-4)
 263
 264         bf      cr7*4+3,1f
 265         lbz     r0,0(r4)
 266         addi    r4,r4,1
 267         stb     r0,0(r3)
 268         addi    r3,r3,1
 269
 270 1:      bf      cr7*4+2,2f
 271         lhz     r0,0(r4)
 272         addi    r4,r4,2
 273         sth     r0,0(r3)
 274         addi    r3,r3,2
 275
 276 2:      bf      cr7*4+1,3f
 277         lwz     r0,0(r4)
 278         addi    r4,r4,4
 279         stw     r0,0(r3)
 280         addi    r3,r3,4
 281
 282 3:      bf      cr7*4+0,4f
 283         ld      r0,0(r4)
 284         addi    r4,r4,8
 285         std     r0,0(r3)
 286         addi    r3,r3,8
 287
 288 4:      sub     r5,r5,r6
 289
 290         /* Get the desination 128B aligned */
 291         neg     r6,r3
 292         srdi    r7,r6,4
 293         mtocrf  0x01,r7
 294         clrldi  r6,r6,(64-7)
 295
 296         li      r9,16
 297         li      r10,32
 298         li      r11,48
 299
 300         bf      cr7*4+3,5f
 301         lvx     v1,0,r4
 302         addi    r4,r4,16
 303         stvx    v1,0,r3
 304         addi    r3,r3,16
 305
 306 5:      bf      cr7*4+2,6f
 307         lvx     v1,0,r4
 308         lvx     v0,r4,r9
 309         addi    r4,r4,32
 310         stvx    v1,0,r3
 311         stvx    v0,r3,r9
 312         addi    r3,r3,32
 313
 314 6:      bf      cr7*4+1,7f
 315         lvx     v3,0,r4
 316         lvx     v2,r4,r9
 317         lvx     v1,r4,r10
 318         lvx     v0,r4,r11
 319         addi    r4,r4,64
 320         stvx    v3,0,r3
 321         stvx    v2,r3,r9
 322         stvx    v1,r3,r10
 323         stvx    v0,r3,r11
 324         addi    r3,r3,64
 325
 326 7:      sub     r5,r5,r6
 327         srdi    r6,r5,7
 328
 329         std     r14,STK_REG(R14)(r1)
 330         std     r15,STK_REG(R15)(r1)
 331         std     r16,STK_REG(R16)(r1)
 332
 333         li      r12,64
 334         li      r14,80
 335         li      r15,96
 336         li      r16,112
 337
 338         mtctr   r6
 339
 340         /*
 341          * Now do cacheline sized loads and stores. By this stage the
 342          * cacheline stores are also cacheline aligned.
 343          */
 344         .align  5
 345 8:
 346         lvx     v7,0,r4
 347         lvx     v6,r4,r9
 348         lvx     v5,r4,r10
 349         lvx     v4,r4,r11
 350         lvx     v3,r4,r12
 351         lvx     v2,r4,r14
 352         lvx     v1,r4,r15
 353         lvx     v0,r4,r16
 354         addi    r4,r4,128
 355         stvx    v7,0,r3
 356         stvx    v6,r3,r9
 357         stvx    v5,r3,r10
 358         stvx    v4,r3,r11
 359         stvx    v3,r3,r12
 360         stvx    v2,r3,r14
 361         stvx    v1,r3,r15
 362         stvx    v0,r3,r16
 363         addi    r3,r3,128
 364         bdnz    8b
 365
 366         ld      r14,STK_REG(R14)(r1)
 367         ld      r15,STK_REG(R15)(r1)
 368         ld      r16,STK_REG(R16)(r1)
 369
 370         /* Up to 127B to go */
 371         clrldi  r5,r5,(64-7)
 372         srdi    r6,r5,4
 373         mtocrf  0x01,r6
 374
 375         bf      cr7*4+1,9f
 376         lvx     v3,0,r4
 377         lvx     v2,r4,r9
 378         lvx     v1,r4,r10
 379         lvx     v0,r4,r11
 380         addi    r4,r4,64
 381         stvx    v3,0,r3
 382         stvx    v2,r3,r9
 383         stvx    v1,r3,r10
 384         stvx    v0,r3,r11
 385         addi    r3,r3,64
 386
 387 9:      bf      cr7*4+2,10f
 388         lvx     v1,0,r4
 389         lvx     v0,r4,r9
 390         addi    r4,r4,32
 391         stvx    v1,0,r3
 392         stvx    v0,r3,r9
 393         addi    r3,r3,32
 394
 395 10:     bf      cr7*4+3,11f
 396         lvx     v1,0,r4
 397         addi    r4,r4,16
 398         stvx    v1,0,r3
 399         addi    r3,r3,16
 400
 401         /* Up to 15B to go */
 402 11:     clrldi  r5,r5,(64-4)
 403         mtocrf  0x01,r5
 404         bf      cr7*4+0,12f
 405         ld      r0,0(r4)
 406         addi    r4,r4,8
 407         std     r0,0(r3)
 408         addi    r3,r3,8
 409
 410 12:     bf      cr7*4+1,13f
 411         lwz     r0,0(r4)
 412         addi    r4,r4,4
 413         stw     r0,0(r3)
 414         addi    r3,r3,4
 415
 416 13:     bf      cr7*4+2,14f
 417         lhz     r0,0(r4)
 418         addi    r4,r4,2
 419         sth     r0,0(r3)
 420         addi    r3,r3,2
 421
 422 14:     bf      cr7*4+3,15f
 423         lbz     r0,0(r4)
 424         stb     r0,0(r3)
 425
 426 15:     addi    r1,r1,STACKFRAMESIZE
 427         ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
 428         b       CFUNC(exit_vmx_ops)             /* tail call optimise */
 429
 430 .Lvmx_unaligned_copy:
 431         /* Get the destination 16B aligned */
 432         neg     r6,r3
 433         mtocrf  0x01,r6
 434         clrldi  r6,r6,(64-4)
 435
 436         bf      cr7*4+3,1f
 437         lbz     r0,0(r4)
 438         addi    r4,r4,1
 439         stb     r0,0(r3)
 440         addi    r3,r3,1
 441
 442 1:      bf      cr7*4+2,2f
 443         lhz     r0,0(r4)
 444         addi    r4,r4,2
 445         sth     r0,0(r3)
 446         addi    r3,r3,2
 447
 448 2:      bf      cr7*4+1,3f
 449         lwz     r0,0(r4)
 450         addi    r4,r4,4
 451         stw     r0,0(r3)
 452         addi    r3,r3,4
 453
 454 3:      bf      cr7*4+0,4f
 455         lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 456         lwz     r7,4(r4)
 457         addi    r4,r4,8
 458         stw     r0,0(r3)
 459         stw     r7,4(r3)
 460         addi    r3,r3,8
 461
 462 4:      sub     r5,r5,r6
 463
 464         /* Get the desination 128B aligned */
 465         neg     r6,r3
 466         srdi    r7,r6,4
 467         mtocrf  0x01,r7
 468         clrldi  r6,r6,(64-7)
 469
 470         li      r9,16
 471         li      r10,32
 472         li      r11,48
 473
 474         LVS(v16,0,r4)           /* Setup permute control vector */
 475         lvx     v0,0,r4
 476         addi    r4,r4,16
 477
 478         bf      cr7*4+3,5f
 479         lvx     v1,0,r4
 480         VPERM(v8,v0,v1,v16)
 481         addi    r4,r4,16
 482         stvx    v8,0,r3
 483         addi    r3,r3,16
 484         vor     v0,v1,v1
 485
 486 5:      bf      cr7*4+2,6f
 487         lvx     v1,0,r4
 488         VPERM(v8,v0,v1,v16)
 489         lvx     v0,r4,r9
 490         VPERM(v9,v1,v0,v16)
 491         addi    r4,r4,32
 492         stvx    v8,0,r3
 493         stvx    v9,r3,r9
 494         addi    r3,r3,32
 495
 496 6:      bf      cr7*4+1,7f
 497         lvx     v3,0,r4
 498         VPERM(v8,v0,v3,v16)
 499         lvx     v2,r4,r9
 500         VPERM(v9,v3,v2,v16)
 501         lvx     v1,r4,r10
 502         VPERM(v10,v2,v1,v16)
 503         lvx     v0,r4,r11
 504         VPERM(v11,v1,v0,v16)
 505         addi    r4,r4,64
 506         stvx    v8,0,r3
 507         stvx    v9,r3,r9
 508         stvx    v10,r3,r10
 509         stvx    v11,r3,r11
 510         addi    r3,r3,64
 511
 512 7:      sub     r5,r5,r6
 513         srdi    r6,r5,7
 514
 515         std     r14,STK_REG(R14)(r1)
 516         std     r15,STK_REG(R15)(r1)
 517         std     r16,STK_REG(R16)(r1)
 518
 519         li      r12,64
 520         li      r14,80
 521         li      r15,96
 522         li      r16,112
 523
 524         mtctr   r6
 525
 526         /*
 527          * Now do cacheline sized loads and stores. By this stage the
 528          * cacheline stores are also cacheline aligned.
 529          */
 530         .align  5
 531 8:
 532         lvx     v7,0,r4
 533         VPERM(v8,v0,v7,v16)
 534         lvx     v6,r4,r9
 535         VPERM(v9,v7,v6,v16)
 536         lvx     v5,r4,r10
 537         VPERM(v10,v6,v5,v16)
 538         lvx     v4,r4,r11
 539         VPERM(v11,v5,v4,v16)
 540         lvx     v3,r4,r12
 541         VPERM(v12,v4,v3,v16)
 542         lvx     v2,r4,r14
 543         VPERM(v13,v3,v2,v16)
 544         lvx     v1,r4,r15
 545         VPERM(v14,v2,v1,v16)
 546         lvx     v0,r4,r16
 547         VPERM(v15,v1,v0,v16)
 548         addi    r4,r4,128
 549         stvx    v8,0,r3
 550         stvx    v9,r3,r9
 551         stvx    v10,r3,r10
 552         stvx    v11,r3,r11
 553         stvx    v12,r3,r12
 554         stvx    v13,r3,r14
 555         stvx    v14,r3,r15
 556         stvx    v15,r3,r16
 557         addi    r3,r3,128
 558         bdnz    8b
 559
 560         ld      r14,STK_REG(R14)(r1)
 561         ld      r15,STK_REG(R15)(r1)
 562         ld      r16,STK_REG(R16)(r1)
 563
 564         /* Up to 127B to go */
 565         clrldi  r5,r5,(64-7)
 566         srdi    r6,r5,4
 567         mtocrf  0x01,r6
 568
 569         bf      cr7*4+1,9f
 570         lvx     v3,0,r4
 571         VPERM(v8,v0,v3,v16)
 572         lvx     v2,r4,r9
 573         VPERM(v9,v3,v2,v16)
 574         lvx     v1,r4,r10
 575         VPERM(v10,v2,v1,v16)
 576         lvx     v0,r4,r11
 577         VPERM(v11,v1,v0,v16)
 578         addi    r4,r4,64
 579         stvx    v8,0,r3
 580         stvx    v9,r3,r9
 581         stvx    v10,r3,r10
 582         stvx    v11,r3,r11
 583         addi    r3,r3,64
 584
 585 9:      bf      cr7*4+2,10f
 586         lvx     v1,0,r4
 587         VPERM(v8,v0,v1,v16)
 588         lvx     v0,r4,r9
 589         VPERM(v9,v1,v0,v16)
 590         addi    r4,r4,32
 591         stvx    v8,0,r3
 592         stvx    v9,r3,r9
 593         addi    r3,r3,32
 594
 595 10:     bf      cr7*4+3,11f
 596         lvx     v1,0,r4
 597         VPERM(v8,v0,v1,v16)
 598         addi    r4,r4,16
 599         stvx    v8,0,r3
 600         addi    r3,r3,16
 601
 602         /* Up to 15B to go */
 603 11:     clrldi  r5,r5,(64-4)
 604         addi    r4,r4,-16       /* Unwind the +16 load offset */
 605         mtocrf  0x01,r5
 606         bf      cr7*4+0,12f
 607         lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 608         lwz     r6,4(r4)
 609         addi    r4,r4,8
 610         stw     r0,0(r3)
 611         stw     r6,4(r3)
 612         addi    r3,r3,8
 613
 614 12:     bf      cr7*4+1,13f
 615         lwz     r0,0(r4)
 616         addi    r4,r4,4
 617         stw     r0,0(r3)
 618         addi    r3,r3,4
 619
 620 13:     bf      cr7*4+2,14f
 621         lhz     r0,0(r4)
 622         addi    r4,r4,2
 623         sth     r0,0(r3)
 624         addi    r3,r3,2
 625
 626 14:     bf      cr7*4+3,15f
 627         lbz     r0,0(r4)
 628         stb     r0,0(r3)
 629
 630 15:     addi    r1,r1,STACKFRAMESIZE
 631         ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
 632         b       CFUNC(exit_vmx_ops)             /* tail call optimise */
 633 #endif /* CONFIG_ALTIVEC */