arch/arm/lib/memmove.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  *  linux/arch/arm/lib/memmove.S
   4  *
   5  *  Author:     Nicolas Pitre
   6  *  Created:    Sep 28, 2005
   7  *  Copyright:  (C) MontaVista Software Inc.
   8  */
   9
  10 #include <linux/linkage.h>
  11 #include <asm/assembler.h>
  12 #include <asm/unwind.h>
  13
  14                 .text
  15
  16 /*
  17  * Prototype: void *memmove(void *dest, const void *src, size_t n);
  18  *
  19  * Note:
  20  *
  21  * If the memory regions don't overlap, we simply branch to memcpy which is
  22  * normally a bit faster. Otherwise the copy is done going downwards.  This
  23  * is a transposition of the code from copy_template.S but with the copy
  24  * occurring in the opposite direction.
  25  */
  26
  27 ENTRY(__memmove)
  28 WEAK(memmove)
  29         UNWIND( .fnstart                        )
  30
  31                 subs    ip, r0, r1
  32                 cmphi   r2, ip
  33                 bls     __memcpy
  34
  35                 stmfd   sp!, {r0, r4, lr}
  36         UNWIND( .fnend                          )
  37
  38         UNWIND( .fnstart                        )
  39         UNWIND( .save   {r0, r4, lr}            ) @ in first stmfd block
  40                 add     r1, r1, r2
  41                 add     r0, r0, r2
  42                 subs    r2, r2, #4
  43                 blt     8f
  44                 ands    ip, r0, #3
  45         PLD(    pld     [r1, #-4]               )
  46                 bne     9f
  47                 ands    ip, r1, #3
  48                 bne     10f
  49
  50 1:              subs    r2, r2, #(28)
  51                 stmfd   sp!, {r5 - r8}
  52         UNWIND( .fnend                          )
  53
  54         UNWIND( .fnstart                        )
  55         UNWIND( .save   {r0, r4, lr}            )
  56         UNWIND( .save   {r5 - r8}               ) @ in second stmfd block
  57                 blt     5f
  58
  59         CALGN(  ands    ip, r0, #31             )
  60         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
  61         CALGN(  bcs     2f                      )
  62         CALGN(  adr     r4, 6f                  )
  63         CALGN(  subs    r2, r2, ip              )  @ C is set here
  64         CALGN(  rsb     ip, ip, #32             )
  65         CALGN(  add     pc, r4, ip              )
  66
  67         PLD(    pld     [r1, #-4]               )
  68 2:      PLD(    subs    r2, r2, #96             )
  69         PLD(    pld     [r1, #-32]              )
  70         PLD(    blt     4f                      )
  71         PLD(    pld     [r1, #-64]              )
  72         PLD(    pld     [r1, #-96]              )
  73
  74 3:      PLD(    pld     [r1, #-128]             )
  75 4:              ldmdb   r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
  76                 subs    r2, r2, #32
  77                 stmdb   r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
  78                 bge     3b
  79         PLD(    cmn     r2, #96                 )
  80         PLD(    bge     4b                      )
  81
  82 5:              ands    ip, r2, #28
  83                 rsb     ip, ip, #32
  84                 addne   pc, pc, ip              @ C is always clear here
  85                 b       7f
  86 6:              W(nop)
  87                 W(ldr)  r3, [r1, #-4]!
  88                 W(ldr)  r4, [r1, #-4]!
  89                 W(ldr)  r5, [r1, #-4]!
  90                 W(ldr)  r6, [r1, #-4]!
  91                 W(ldr)  r7, [r1, #-4]!
  92                 W(ldr)  r8, [r1, #-4]!
  93                 W(ldr)  lr, [r1, #-4]!
  94
  95                 add     pc, pc, ip
  96                 nop
  97                 W(nop)
  98                 W(str)  r3, [r0, #-4]!
  99                 W(str)  r4, [r0, #-4]!
 100                 W(str)  r5, [r0, #-4]!
 101                 W(str)  r6, [r0, #-4]!
 102                 W(str)  r7, [r0, #-4]!
 103                 W(str)  r8, [r0, #-4]!
 104                 W(str)  lr, [r0, #-4]!
 105
 106         CALGN(  bcs     2b                      )
 107
 108 7:              ldmfd   sp!, {r5 - r8}
 109         UNWIND( .fnend                          ) @ end of second stmfd block
 110
 111         UNWIND( .fnstart                        )
 112         UNWIND( .save   {r0, r4, lr}            ) @ still in first stmfd block
 113
 114 8:              movs    r2, r2, lsl #31
 115                 ldrbne  r3, [r1, #-1]!
 116                 ldrbcs  r4, [r1, #-1]!
 117                 ldrbcs  ip, [r1, #-1]
 118                 strbne  r3, [r0, #-1]!
 119                 strbcs  r4, [r0, #-1]!
 120                 strbcs  ip, [r0, #-1]
 121                 ldmfd   sp!, {r0, r4, pc}
 122
 123 9:              cmp     ip, #2
 124                 ldrbgt  r3, [r1, #-1]!
 125                 ldrbge  r4, [r1, #-1]!
 126                 ldrb    lr, [r1, #-1]!
 127                 strbgt  r3, [r0, #-1]!
 128                 strbge  r4, [r0, #-1]!
 129                 subs    r2, r2, ip
 130                 strb    lr, [r0, #-1]!
 131                 blt     8b
 132                 ands    ip, r1, #3
 133                 beq     1b
 134
 135 10:             bic     r1, r1, #3
 136                 cmp     ip, #2
 137                 ldr     r3, [r1, #0]
 138                 beq     17f
 139                 blt     18f
 140         UNWIND( .fnend                          )
 141
 142
 143                 .macro  backward_copy_shift push pull
 144
 145         UNWIND( .fnstart                        )
 146         UNWIND( .save   {r0, r4, lr}            ) @ still in first stmfd block
 147                 subs    r2, r2, #28
 148                 blt     14f
 149
 150         CALGN(  ands    ip, r0, #31             )
 151         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
 152         CALGN(  subcc   r2, r2, ip              )
 153         CALGN(  bcc     15f                     )
 154
 155 11:             stmfd   sp!, {r5 - r9}
 156         UNWIND( .fnend                          )
 157
 158         UNWIND( .fnstart                        )
 159         UNWIND( .save   {r0, r4, lr}            )
 160         UNWIND( .save   {r5 - r9}               ) @ in new second stmfd block
 161
 162         PLD(    pld     [r1, #-4]               )
 163         PLD(    subs    r2, r2, #96             )
 164         PLD(    pld     [r1, #-32]              )
 165         PLD(    blt     13f                     )
 166         PLD(    pld     [r1, #-64]              )
 167         PLD(    pld     [r1, #-96]              )
 168
 169 12:     PLD(    pld     [r1, #-128]             )
 170 13:             ldmdb   r1!, {r7, r8, r9, ip}
 171                 mov     lr, r3, lspush #\push
 172                 subs    r2, r2, #32
 173                 ldmdb   r1!, {r3, r4, r5, r6}
 174                 orr     lr, lr, ip, lspull #\pull
 175                 mov     ip, ip, lspush #\push
 176                 orr     ip, ip, r9, lspull #\pull
 177                 mov     r9, r9, lspush #\push
 178                 orr     r9, r9, r8, lspull #\pull
 179                 mov     r8, r8, lspush #\push
 180                 orr     r8, r8, r7, lspull #\pull
 181                 mov     r7, r7, lspush #\push
 182                 orr     r7, r7, r6, lspull #\pull
 183                 mov     r6, r6, lspush #\push
 184                 orr     r6, r6, r5, lspull #\pull
 185                 mov     r5, r5, lspush #\push
 186                 orr     r5, r5, r4, lspull #\pull
 187                 mov     r4, r4, lspush #\push
 188                 orr     r4, r4, r3, lspull #\pull
 189                 stmdb   r0!, {r4 - r9, ip, lr}
 190                 bge     12b
 191         PLD(    cmn     r2, #96                 )
 192         PLD(    bge     13b                     )
 193
 194                 ldmfd   sp!, {r5 - r9}
 195         UNWIND( .fnend                          ) @ end of the second stmfd block
 196
 197         UNWIND( .fnstart                        )
 198         UNWIND( .save {r0, r4, lr}              ) @ still in first stmfd block
 199
 200 14:             ands    ip, r2, #28
 201                 beq     16f
 202
 203 15:             mov     lr, r3, lspush #\push
 204                 ldr     r3, [r1, #-4]!
 205                 subs    ip, ip, #4
 206                 orr     lr, lr, r3, lspull #\pull
 207                 str     lr, [r0, #-4]!
 208                 bgt     15b
 209         CALGN(  cmp     r2, #0                  )
 210         CALGN(  bge     11b                     )
 211
 212 16:             add     r1, r1, #(\pull / 8)
 213                 b       8b
 214         UNWIND( .fnend                          )
 215
 216                 .endm
 217
 218
 219                 backward_copy_shift     push=8  pull=24
 220
 221 17:             backward_copy_shift     push=16 pull=16
 222
 223 18:             backward_copy_shift     push=24 pull=8
 224
 225 ENDPROC(memmove)
 226 ENDPROC(__memmove)