src/arch/arm/memmove.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /* Based on linux/arch/arm/lib/memmove.S */
   3
   4 #include <arch/asm.h>
   5 #include "asmlib.h"
   6
   7 .syntax unified
   8
   9 /*
  10  * Prototype: void *memmove(void *dest, const void *src, size_t n);
  11  *
  12  * Note:
  13  *
  14  * If the memory regions don't overlap, we simply branch to memcpy which is
  15  * normally a bit faster. Otherwise the copy is done going downwards.  This
  16  * is a transposition of the code from copy_template.S but with the copy
  17  * occurring in the opposite direction.
  18  */
  19
  20 ENTRY(memmove)
  21
  22                 subs    ip, r0, r1
  23                 cmphi   r2, ip
  24                 bls     memcpy
  25
  26                 stmfd   sp!, {r0, r4, lr}
  27                 add     r1, r1, r2
  28                 add     r0, r0, r2
  29                 subs    r2, r2, #4
  30                 blt     8f
  31                 ands    ip, r0, #3
  32         PLD(    pld     [r1, #-4]               )
  33                 bne     9f
  34                 ands    ip, r1, #3
  35                 bne     10f
  36
  37 1:              subs    r2, r2, #(28)
  38                 stmfd   sp!, {r5 - r8}
  39                 blt     5f
  40
  41         CALGN(  ands    ip, r0, #31             )
  42         CALGN(  sbcnes  r4, ip, r2              )  @ C is always set here
  43         CALGN(  bcs     2f                      )
  44         CALGN(  adr     r4, 6f                  )
  45         CALGN(  subs    r2, r2, ip              )  @ C is set here
  46         CALGN(  rsb     ip, ip, #32             )
  47         CALGN(  add     pc, r4, ip              )
  48
  49         PLD(    pld     [r1, #-4]               )
  50 2:      PLD(    subs    r2, r2, #96             )
  51         PLD(    pld     [r1, #-32]              )
  52         PLD(    blt     4f                      )
  53         PLD(    pld     [r1, #-64]              )
  54         PLD(    pld     [r1, #-96]              )
  55
  56 3:      PLD(    pld     [r1, #-128]             )
  57 4:              ldmdb   r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
  58                 subs    r2, r2, #32
  59                 stmdb   r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
  60                 bge     3b
  61         PLD(    cmn     r2, #96                 )
  62         PLD(    bge     4b                      )
  63
  64 5:              ands    ip, r2, #28
  65                 rsb     ip, ip, #32
  66                 addne   pc, pc, ip              @ C is always clear here
  67                 b       7f
  68 6:              W(nop)
  69                 W(ldr)  r3, [r1, #-4]!
  70                 W(ldr)  r4, [r1, #-4]!
  71                 W(ldr)  r5, [r1, #-4]!
  72                 W(ldr)  r6, [r1, #-4]!
  73                 W(ldr)  r7, [r1, #-4]!
  74                 W(ldr)  r8, [r1, #-4]!
  75                 W(ldr)  lr, [r1, #-4]!
  76
  77                 add     pc, pc, ip
  78                 nop
  79                 W(nop)
  80                 W(str)  r3, [r0, #-4]!
  81                 W(str)  r4, [r0, #-4]!
  82                 W(str)  r5, [r0, #-4]!
  83                 W(str)  r6, [r0, #-4]!
  84                 W(str)  r7, [r0, #-4]!
  85                 W(str)  r8, [r0, #-4]!
  86                 W(str)  lr, [r0, #-4]!
  87
  88         CALGN(  bcs     2b                      )
  89
  90 7:              ldmfd   sp!, {r5 - r8}
  91
  92 8:              movs    r2, r2, lsl #31
  93                 ldrbne  r3, [r1, #-1]!
  94                 ldrbcs  r4, [r1, #-1]!
  95                 ldrbcs  ip, [r1, #-1]
  96                 strbne  r3, [r0, #-1]!
  97                 strbcs  r4, [r0, #-1]!
  98                 strbcs  ip, [r0, #-1]
  99                 ldmfd   sp!, {r0, r4, pc}
 100
 101 9:              cmp     ip, #2
 102                 ldrbgt  r3, [r1, #-1]!
 103                 ldrbge  r4, [r1, #-1]!
 104                 ldrb    lr, [r1, #-1]!
 105                 strbgt  r3, [r0, #-1]!
 106                 strbge  r4, [r0, #-1]!
 107                 subs    r2, r2, ip
 108                 strb    lr, [r0, #-1]!
 109                 blt     8b
 110                 ands    ip, r1, #3
 111                 beq     1b
 112
 113 10:             bic     r1, r1, #3
 114                 cmp     ip, #2
 115                 ldr     r3, [r1, #0]
 116                 beq     17f
 117                 blt     18f
 118
 119
 120                 .macro  backward_copy_shift push pull
 121
 122                 subs    r2, r2, #28
 123                 blt     14f
 124
 125         CALGN(  ands    ip, r0, #31             )
 126         CALGN(  sbcnes  r4, ip, r2              )  @ C is always set here
 127         CALGN(  subcc   r2, r2, ip              )
 128         CALGN(  bcc     15f                     )
 129
 130 11:             stmfd   sp!, {r5 - r9}
 131
 132         PLD(    pld     [r1, #-4]               )
 133         PLD(    subs    r2, r2, #96             )
 134         PLD(    pld     [r1, #-32]              )
 135         PLD(    blt     13f                     )
 136         PLD(    pld     [r1, #-64]              )
 137         PLD(    pld     [r1, #-96]              )
 138
 139 12:     PLD(    pld     [r1, #-128]             )
 140 13:             ldmdb   r1!, {r7, r8, r9, ip}
 141                 mov     lr, r3, push #\push
 142                 subs    r2, r2, #32
 143                 ldmdb   r1!, {r3, r4, r5, r6}
 144                 orr     lr, lr, ip, pull #\pull
 145                 mov     ip, ip, push #\push
 146                 orr     ip, ip, r9, pull #\pull
 147                 mov     r9, r9, push #\push
 148                 orr     r9, r9, r8, pull #\pull
 149                 mov     r8, r8, push #\push
 150                 orr     r8, r8, r7, pull #\pull
 151                 mov     r7, r7, push #\push
 152                 orr     r7, r7, r6, pull #\pull
 153                 mov     r6, r6, push #\push
 154                 orr     r6, r6, r5, pull #\pull
 155                 mov     r5, r5, push #\push
 156                 orr     r5, r5, r4, pull #\pull
 157                 mov     r4, r4, push #\push
 158                 orr     r4, r4, r3, pull #\pull
 159                 stmdb   r0!, {r4 - r9, ip, lr}
 160                 bge     12b
 161         PLD(    cmn     r2, #96                 )
 162         PLD(    bge     13b                     )
 163
 164                 ldmfd   sp!, {r5 - r9}
 165
 166 14:             ands    ip, r2, #28
 167                 beq     16f
 168
 169 15:             mov     lr, r3, push #\push
 170                 ldr     r3, [r1, #-4]!
 171                 subs    ip, ip, #4
 172                 orr     lr, lr, r3, pull #\pull
 173                 str     lr, [r0, #-4]!
 174                 bgt     15b
 175         CALGN(  cmp     r2, #0                  )
 176         CALGN(  bge     11b                     )
 177
 178 16:             add     r1, r1, #(\pull / 8)
 179                 b       8b
 180
 181                 .endm
 182
 183
 184                 backward_copy_shift     push=8  pull=24
 185
 186 17:             backward_copy_shift     push=16 pull=16
 187
 188 18:             backward_copy_shift     push=24 pull=8
 189
 190 ENDPROC(memmove)