arch/powerpc/lib/copyuser_64.S

   1 /*
   2  * Copyright (C) 2002 Paul Mackerras, IBM Corp.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version
   7  * 2 of the License, or (at your option) any later version.
   8  */
   9 #include <asm/processor.h>
  10 #include <asm/ppc_asm.h>
  11 #include <asm/export.h>
  12
  13 #ifdef __BIG_ENDIAN__
  14 #define sLd sld         /* Shift towards low-numbered address. */
  15 #define sHd srd         /* Shift towards high-numbered address. */
  16 #else
  17 #define sLd srd         /* Shift towards low-numbered address. */
  18 #define sHd sld         /* Shift towards high-numbered address. */
  19 #endif
  20
  21         .align  7
  22 _GLOBAL_TOC(__copy_tofrom_user)
  23 BEGIN_FTR_SECTION
  24         nop
  25 FTR_SECTION_ELSE
  26         b       __copy_tofrom_user_power7
  27 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
  28 _GLOBAL(__copy_tofrom_user_base)
  29         /* first check for a whole page copy on a page boundary */
  30         cmpldi  cr1,r5,16
  31         cmpdi   cr6,r5,4096
  32         or      r0,r3,r4
  33         neg     r6,r3           /* LS 3 bits = # bytes to 8-byte dest bdry */
  34         andi.   r0,r0,4095
  35         std     r3,-24(r1)
  36         crand   cr0*4+2,cr0*4+2,cr6*4+2
  37         std     r4,-16(r1)
  38         std     r5,-8(r1)
  39         dcbt    0,r4
  40         beq     .Lcopy_page_4K
  41         andi.   r6,r6,7
  42         PPC_MTOCRF(0x01,r5)
  43         blt     cr1,.Lshort_copy
  44 /* Below we want to nop out the bne if we're on a CPU that has the
  45  * CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
  46  * cleared.
  47  * At the time of writing the only CPU that has this combination of bits
  48  * set is Power6.
  49  */
  50 BEGIN_FTR_SECTION
  51         nop
  52 FTR_SECTION_ELSE
  53         bne     .Ldst_unaligned
  54 ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
  55                     CPU_FTR_UNALIGNED_LD_STD)
  56 .Ldst_aligned:
  57         addi    r3,r3,-16
  58 BEGIN_FTR_SECTION
  59         andi.   r0,r4,7
  60         bne     .Lsrc_unaligned
  61 END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
  62         blt     cr1,.Ldo_tail           /* if < 16 bytes to copy */
  63         srdi    r0,r5,5
  64         cmpdi   cr1,r0,0
  65 20:     ld      r7,0(r4)
  66 220:    ld      r6,8(r4)
  67         addi    r4,r4,16
  68         mtctr   r0
  69         andi.   r0,r5,0x10
  70         beq     22f
  71         addi    r3,r3,16
  72         addi    r4,r4,-16
  73         mr      r9,r7
  74         mr      r8,r6
  75         beq     cr1,72f
  76 21:     ld      r7,16(r4)
  77 221:    ld      r6,24(r4)
  78         addi    r4,r4,32
  79 70:     std     r9,0(r3)
  80 270:    std     r8,8(r3)
  81 22:     ld      r9,0(r4)
  82 222:    ld      r8,8(r4)
  83 71:     std     r7,16(r3)
  84 271:    std     r6,24(r3)
  85         addi    r3,r3,32
  86         bdnz    21b
  87 72:     std     r9,0(r3)
  88 272:    std     r8,8(r3)
  89         andi.   r5,r5,0xf
  90         beq+    3f
  91         addi    r4,r4,16
  92 .Ldo_tail:
  93         addi    r3,r3,16
  94         bf      cr7*4+0,246f
  95 244:    ld      r9,0(r4)
  96         addi    r4,r4,8
  97 245:    std     r9,0(r3)
  98         addi    r3,r3,8
  99 246:    bf      cr7*4+1,1f
 100 23:     lwz     r9,0(r4)
 101         addi    r4,r4,4
 102 73:     stw     r9,0(r3)
 103         addi    r3,r3,4
 104 1:      bf      cr7*4+2,2f
 105 44:     lhz     r9,0(r4)
 106         addi    r4,r4,2
 107 74:     sth     r9,0(r3)
 108         addi    r3,r3,2
 109 2:      bf      cr7*4+3,3f
 110 45:     lbz     r9,0(r4)
 111 75:     stb     r9,0(r3)
 112 3:      li      r3,0
 113         blr
 114
 115 .Lsrc_unaligned:
 116         srdi    r6,r5,3
 117         addi    r5,r5,-16
 118         subf    r4,r0,r4
 119         srdi    r7,r5,4
 120         sldi    r10,r0,3
 121         cmpldi  cr6,r6,3
 122         andi.   r5,r5,7
 123         mtctr   r7
 124         subfic  r11,r10,64
 125         add     r5,r5,r0
 126         bt      cr7*4+0,28f
 127
 128 24:     ld      r9,0(r4)        /* 3+2n loads, 2+2n stores */
 129 25:     ld      r0,8(r4)
 130         sLd     r6,r9,r10
 131 26:     ldu     r9,16(r4)
 132         sHd     r7,r0,r11
 133         sLd     r8,r0,r10
 134         or      r7,r7,r6
 135         blt     cr6,79f
 136 27:     ld      r0,8(r4)
 137         b       2f
 138
 139 28:     ld      r0,0(r4)        /* 4+2n loads, 3+2n stores */
 140 29:     ldu     r9,8(r4)
 141         sLd     r8,r0,r10
 142         addi    r3,r3,-8
 143         blt     cr6,5f
 144 30:     ld      r0,8(r4)
 145         sHd     r12,r9,r11
 146         sLd     r6,r9,r10
 147 31:     ldu     r9,16(r4)
 148         or      r12,r8,r12
 149         sHd     r7,r0,r11
 150         sLd     r8,r0,r10
 151         addi    r3,r3,16
 152         beq     cr6,78f
 153
 154 1:      or      r7,r7,r6
 155 32:     ld      r0,8(r4)
 156 76:     std     r12,8(r3)
 157 2:      sHd     r12,r9,r11
 158         sLd     r6,r9,r10
 159 33:     ldu     r9,16(r4)
 160         or      r12,r8,r12
 161 77:     stdu    r7,16(r3)
 162         sHd     r7,r0,r11
 163         sLd     r8,r0,r10
 164         bdnz    1b
 165
 166 78:     std     r12,8(r3)
 167         or      r7,r7,r6
 168 79:     std     r7,16(r3)
 169 5:      sHd     r12,r9,r11
 170         or      r12,r8,r12
 171 80:     std     r12,24(r3)
 172         bne     6f
 173         li      r3,0
 174         blr
 175 6:      cmpwi   cr1,r5,8
 176         addi    r3,r3,32
 177         sLd     r9,r9,r10
 178         ble     cr1,7f
 179 34:     ld      r0,8(r4)
 180         sHd     r7,r0,r11
 181         or      r9,r7,r9
 182 7:
 183         bf      cr7*4+1,1f
 184 #ifdef __BIG_ENDIAN__
 185         rotldi  r9,r9,32
 186 #endif
 187 94:     stw     r9,0(r3)
 188 #ifdef __LITTLE_ENDIAN__
 189         rotrdi  r9,r9,32
 190 #endif
 191         addi    r3,r3,4
 192 1:      bf      cr7*4+2,2f
 193 #ifdef __BIG_ENDIAN__
 194         rotldi  r9,r9,16
 195 #endif
 196 95:     sth     r9,0(r3)
 197 #ifdef __LITTLE_ENDIAN__
 198         rotrdi  r9,r9,16
 199 #endif
 200         addi    r3,r3,2
 201 2:      bf      cr7*4+3,3f
 202 #ifdef __BIG_ENDIAN__
 203         rotldi  r9,r9,8
 204 #endif
 205 96:     stb     r9,0(r3)
 206 #ifdef __LITTLE_ENDIAN__
 207         rotrdi  r9,r9,8
 208 #endif
 209 3:      li      r3,0
 210         blr
 211
 212 .Ldst_unaligned:
 213         PPC_MTOCRF(0x01,r6)             /* put #bytes to 8B bdry into cr7 */
 214         subf    r5,r6,r5
 215         li      r7,0
 216         cmpldi  cr1,r5,16
 217         bf      cr7*4+3,1f
 218 35:     lbz     r0,0(r4)
 219 81:     stb     r0,0(r3)
 220         addi    r7,r7,1
 221 1:      bf      cr7*4+2,2f
 222 36:     lhzx    r0,r7,r4
 223 82:     sthx    r0,r7,r3
 224         addi    r7,r7,2
 225 2:      bf      cr7*4+1,3f
 226 37:     lwzx    r0,r7,r4
 227 83:     stwx    r0,r7,r3
 228 3:      PPC_MTOCRF(0x01,r5)
 229         add     r4,r6,r4
 230         add     r3,r6,r3
 231         b       .Ldst_aligned
 232
 233 .Lshort_copy:
 234         bf      cr7*4+0,1f
 235 38:     lwz     r0,0(r4)
 236 39:     lwz     r9,4(r4)
 237         addi    r4,r4,8
 238 84:     stw     r0,0(r3)
 239 85:     stw     r9,4(r3)
 240         addi    r3,r3,8
 241 1:      bf      cr7*4+1,2f
 242 40:     lwz     r0,0(r4)
 243         addi    r4,r4,4
 244 86:     stw     r0,0(r3)
 245         addi    r3,r3,4
 246 2:      bf      cr7*4+2,3f
 247 41:     lhz     r0,0(r4)
 248         addi    r4,r4,2
 249 87:     sth     r0,0(r3)
 250         addi    r3,r3,2
 251 3:      bf      cr7*4+3,4f
 252 42:     lbz     r0,0(r4)
 253 88:     stb     r0,0(r3)
 254 4:      li      r3,0
 255         blr
 256
 257 /*
 258  * exception handlers follow
 259  * we have to return the number of bytes not copied
 260  * for an exception on a load, we set the rest of the destination to 0
 261  */
 262
 263 136:
 264 137:
 265         add     r3,r3,r7
 266         b       1f
 267 130:
 268 131:
 269         addi    r3,r3,8
 270 120:
 271 320:
 272 122:
 273 322:
 274 124:
 275 125:
 276 126:
 277 127:
 278 128:
 279 129:
 280 133:
 281         addi    r3,r3,8
 282 132:
 283         addi    r3,r3,8
 284 121:
 285 321:
 286 344:
 287 134:
 288 135:
 289 138:
 290 139:
 291 140:
 292 141:
 293 142:
 294 123:
 295 144:
 296 145:
 297
 298 /*
 299  * here we have had a fault on a load and r3 points to the first
 300  * unmodified byte of the destination
 301  */
 302 1:      ld      r6,-24(r1)
 303         ld      r4,-16(r1)
 304         ld      r5,-8(r1)
 305         subf    r6,r6,r3
 306         add     r4,r4,r6
 307         subf    r5,r6,r5        /* #bytes left to go */
 308
 309 /*
 310  * first see if we can copy any more bytes before hitting another exception
 311  */
 312         mtctr   r5
 313 43:     lbz     r0,0(r4)
 314         addi    r4,r4,1
 315 89:     stb     r0,0(r3)
 316         addi    r3,r3,1
 317         bdnz    43b
 318         li      r3,0            /* huh? all copied successfully this time? */
 319         blr
 320
 321 /*
 322  * here we have trapped again, amount remaining is in ctr.
 323  */
 324 143:    mfctr   r3
 325         blr
 326
 327 /*
 328  * exception handlers for stores: we just need to work
 329  * out how many bytes weren't copied
 330  */
 331 182:
 332 183:
 333         add     r3,r3,r7
 334         b       1f
 335 371:
 336 180:
 337         addi    r3,r3,8
 338 171:
 339 177:
 340 179:
 341         addi    r3,r3,8
 342 370:
 343 372:
 344 176:
 345 178:
 346         addi    r3,r3,4
 347 185:
 348         addi    r3,r3,4
 349 170:
 350 172:
 351 345:
 352 173:
 353 174:
 354 175:
 355 181:
 356 184:
 357 186:
 358 187:
 359 188:
 360 189:
 361 194:
 362 195:
 363 196:
 364 1:
 365         ld      r6,-24(r1)
 366         ld      r5,-8(r1)
 367         add     r6,r6,r5
 368         subf    r3,r3,r6        /* #bytes not copied */
 369         blr
 370
 371         EX_TABLE(20b,120b)
 372         EX_TABLE(220b,320b)
 373         EX_TABLE(21b,121b)
 374         EX_TABLE(221b,321b)
 375         EX_TABLE(70b,170b)
 376         EX_TABLE(270b,370b)
 377         EX_TABLE(22b,122b)
 378         EX_TABLE(222b,322b)
 379         EX_TABLE(71b,171b)
 380         EX_TABLE(271b,371b)
 381         EX_TABLE(72b,172b)
 382         EX_TABLE(272b,372b)
 383         EX_TABLE(244b,344b)
 384         EX_TABLE(245b,345b)
 385         EX_TABLE(23b,123b)
 386         EX_TABLE(73b,173b)
 387         EX_TABLE(44b,144b)
 388         EX_TABLE(74b,174b)
 389         EX_TABLE(45b,145b)
 390         EX_TABLE(75b,175b)
 391         EX_TABLE(24b,124b)
 392         EX_TABLE(25b,125b)
 393         EX_TABLE(26b,126b)
 394         EX_TABLE(27b,127b)
 395         EX_TABLE(28b,128b)
 396         EX_TABLE(29b,129b)
 397         EX_TABLE(30b,130b)
 398         EX_TABLE(31b,131b)
 399         EX_TABLE(32b,132b)
 400         EX_TABLE(76b,176b)
 401         EX_TABLE(33b,133b)
 402         EX_TABLE(77b,177b)
 403         EX_TABLE(78b,178b)
 404         EX_TABLE(79b,179b)
 405         EX_TABLE(80b,180b)
 406         EX_TABLE(34b,134b)
 407         EX_TABLE(94b,194b)
 408         EX_TABLE(95b,195b)
 409         EX_TABLE(96b,196b)
 410         EX_TABLE(35b,135b)
 411         EX_TABLE(81b,181b)
 412         EX_TABLE(36b,136b)
 413         EX_TABLE(82b,182b)
 414         EX_TABLE(37b,137b)
 415         EX_TABLE(83b,183b)
 416         EX_TABLE(38b,138b)
 417         EX_TABLE(39b,139b)
 418         EX_TABLE(84b,184b)
 419         EX_TABLE(85b,185b)
 420         EX_TABLE(40b,140b)
 421         EX_TABLE(86b,186b)
 422         EX_TABLE(41b,141b)
 423         EX_TABLE(87b,187b)
 424         EX_TABLE(42b,142b)
 425         EX_TABLE(88b,188b)
 426         EX_TABLE(43b,143b)
 427         EX_TABLE(89b,189b)
 428
 429 /*
 430  * Routine to copy a whole page of data, optimized for POWER4.
 431  * On POWER4 it is more than 50% faster than the simple loop
 432  * above (following the .Ldst_aligned label).
 433  */
 434 .Lcopy_page_4K:
 435         std     r31,-32(1)
 436         std     r30,-40(1)
 437         std     r29,-48(1)
 438         std     r28,-56(1)
 439         std     r27,-64(1)
 440         std     r26,-72(1)
 441         std     r25,-80(1)
 442         std     r24,-88(1)
 443         std     r23,-96(1)
 444         std     r22,-104(1)
 445         std     r21,-112(1)
 446         std     r20,-120(1)
 447         li      r5,4096/32 - 1
 448         addi    r3,r3,-8
 449         li      r0,5
 450 0:      addi    r5,r5,-24
 451         mtctr   r0
 452 20:     ld      r22,640(4)
 453 21:     ld      r21,512(4)
 454 22:     ld      r20,384(4)
 455 23:     ld      r11,256(4)
 456 24:     ld      r9,128(4)
 457 25:     ld      r7,0(4)
 458 26:     ld      r25,648(4)
 459 27:     ld      r24,520(4)
 460 28:     ld      r23,392(4)
 461 29:     ld      r10,264(4)
 462 30:     ld      r8,136(4)
 463 31:     ldu     r6,8(4)
 464         cmpwi   r5,24
 465 1:
 466 32:     std     r22,648(3)
 467 33:     std     r21,520(3)
 468 34:     std     r20,392(3)
 469 35:     std     r11,264(3)
 470 36:     std     r9,136(3)
 471 37:     std     r7,8(3)
 472 38:     ld      r28,648(4)
 473 39:     ld      r27,520(4)
 474 40:     ld      r26,392(4)
 475 41:     ld      r31,264(4)
 476 42:     ld      r30,136(4)
 477 43:     ld      r29,8(4)
 478 44:     std     r25,656(3)
 479 45:     std     r24,528(3)
 480 46:     std     r23,400(3)
 481 47:     std     r10,272(3)
 482 48:     std     r8,144(3)
 483 49:     std     r6,16(3)
 484 50:     ld      r22,656(4)
 485 51:     ld      r21,528(4)
 486 52:     ld      r20,400(4)
 487 53:     ld      r11,272(4)
 488 54:     ld      r9,144(4)
 489 55:     ld      r7,16(4)
 490 56:     std     r28,664(3)
 491 57:     std     r27,536(3)
 492 58:     std     r26,408(3)
 493 59:     std     r31,280(3)
 494 60:     std     r30,152(3)
 495 61:     stdu    r29,24(3)
 496 62:     ld      r25,664(4)
 497 63:     ld      r24,536(4)
 498 64:     ld      r23,408(4)
 499 65:     ld      r10,280(4)
 500 66:     ld      r8,152(4)
 501 67:     ldu     r6,24(4)
 502         bdnz    1b
 503 68:     std     r22,648(3)
 504 69:     std     r21,520(3)
 505 70:     std     r20,392(3)
 506 71:     std     r11,264(3)
 507 72:     std     r9,136(3)
 508 73:     std     r7,8(3)
 509 74:     addi    r4,r4,640
 510 75:     addi    r3,r3,648
 511         bge     0b
 512         mtctr   r5
 513 76:     ld      r7,0(4)
 514 77:     ld      r8,8(4)
 515 78:     ldu     r9,16(4)
 516 3:
 517 79:     ld      r10,8(4)
 518 80:     std     r7,8(3)
 519 81:     ld      r7,16(4)
 520 82:     std     r8,16(3)
 521 83:     ld      r8,24(4)
 522 84:     std     r9,24(3)
 523 85:     ldu     r9,32(4)
 524 86:     stdu    r10,32(3)
 525         bdnz    3b
 526 4:
 527 87:     ld      r10,8(4)
 528 88:     std     r7,8(3)
 529 89:     std     r8,16(3)
 530 90:     std     r9,24(3)
 531 91:     std     r10,32(3)
 532 9:      ld      r20,-120(1)
 533         ld      r21,-112(1)
 534         ld      r22,-104(1)
 535         ld      r23,-96(1)
 536         ld      r24,-88(1)
 537         ld      r25,-80(1)
 538         ld      r26,-72(1)
 539         ld      r27,-64(1)
 540         ld      r28,-56(1)
 541         ld      r29,-48(1)
 542         ld      r30,-40(1)
 543         ld      r31,-32(1)
 544         li      r3,0
 545         blr
 546
 547 /*
 548  * on an exception, reset to the beginning and jump back into the
 549  * standard __copy_tofrom_user
 550  */
 551 100:    ld      r20,-120(1)
 552         ld      r21,-112(1)
 553         ld      r22,-104(1)
 554         ld      r23,-96(1)
 555         ld      r24,-88(1)
 556         ld      r25,-80(1)
 557         ld      r26,-72(1)
 558         ld      r27,-64(1)
 559         ld      r28,-56(1)
 560         ld      r29,-48(1)
 561         ld      r30,-40(1)
 562         ld      r31,-32(1)
 563         ld      r3,-24(r1)
 564         ld      r4,-16(r1)
 565         li      r5,4096
 566         b       .Ldst_aligned
 567
 568         EX_TABLE(20b,100b)
 569         EX_TABLE(21b,100b)
 570         EX_TABLE(22b,100b)
 571         EX_TABLE(23b,100b)
 572         EX_TABLE(24b,100b)
 573         EX_TABLE(25b,100b)
 574         EX_TABLE(26b,100b)
 575         EX_TABLE(27b,100b)
 576         EX_TABLE(28b,100b)
 577         EX_TABLE(29b,100b)
 578         EX_TABLE(30b,100b)
 579         EX_TABLE(31b,100b)
 580         EX_TABLE(32b,100b)
 581         EX_TABLE(33b,100b)
 582         EX_TABLE(34b,100b)
 583         EX_TABLE(35b,100b)
 584         EX_TABLE(36b,100b)
 585         EX_TABLE(37b,100b)
 586         EX_TABLE(38b,100b)
 587         EX_TABLE(39b,100b)
 588         EX_TABLE(40b,100b)
 589         EX_TABLE(41b,100b)
 590         EX_TABLE(42b,100b)
 591         EX_TABLE(43b,100b)
 592         EX_TABLE(44b,100b)
 593         EX_TABLE(45b,100b)
 594         EX_TABLE(46b,100b)
 595         EX_TABLE(47b,100b)
 596         EX_TABLE(48b,100b)
 597         EX_TABLE(49b,100b)
 598         EX_TABLE(50b,100b)
 599         EX_TABLE(51b,100b)
 600         EX_TABLE(52b,100b)
 601         EX_TABLE(53b,100b)
 602         EX_TABLE(54b,100b)
 603         EX_TABLE(55b,100b)
 604         EX_TABLE(56b,100b)
 605         EX_TABLE(57b,100b)
 606         EX_TABLE(58b,100b)
 607         EX_TABLE(59b,100b)
 608         EX_TABLE(60b,100b)
 609         EX_TABLE(61b,100b)
 610         EX_TABLE(62b,100b)
 611         EX_TABLE(63b,100b)
 612         EX_TABLE(64b,100b)
 613         EX_TABLE(65b,100b)
 614         EX_TABLE(66b,100b)
 615         EX_TABLE(67b,100b)
 616         EX_TABLE(68b,100b)
 617         EX_TABLE(69b,100b)
 618         EX_TABLE(70b,100b)
 619         EX_TABLE(71b,100b)
 620         EX_TABLE(72b,100b)
 621         EX_TABLE(73b,100b)
 622         EX_TABLE(74b,100b)
 623         EX_TABLE(75b,100b)
 624         EX_TABLE(76b,100b)
 625         EX_TABLE(77b,100b)
 626         EX_TABLE(78b,100b)
 627         EX_TABLE(79b,100b)
 628         EX_TABLE(80b,100b)
 629         EX_TABLE(81b,100b)
 630         EX_TABLE(82b,100b)
 631         EX_TABLE(83b,100b)
 632         EX_TABLE(84b,100b)
 633         EX_TABLE(85b,100b)
 634         EX_TABLE(86b,100b)
 635         EX_TABLE(87b,100b)
 636         EX_TABLE(88b,100b)
 637         EX_TABLE(89b,100b)
 638         EX_TABLE(90b,100b)
 639         EX_TABLE(91b,100b)
 640
 641 EXPORT_SYMBOL(__copy_tofrom_user)