1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright 2002 Andi Kleen */
4 #include <linux/linkage.h>
6 #include <asm/cpufeatures.h>
7 #include <asm/mcsafe_test.h>
8 #include <asm/alternative-asm.h>
9 #include <asm/export.h>
12 * We build a jump to memcpy_orig by default which gets NOPped out on
13 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
14 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
15 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
21 * memcpy - Copy a memory block.
29 * rax original destination
31 SYM_FUNC_START_ALIAS(__memcpy)
32 SYM_FUNC_START_LOCAL(memcpy)
33 ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
34 "jmp memcpy_erms", X86_FEATURE_ERMS
45 SYM_FUNC_END_ALIAS(__memcpy)
47 EXPORT_SYMBOL(__memcpy)
50 * memcpy_erms() - enhanced fast string memcpy. This is faster and
51 * simpler than memcpy. Use memcpy_erms when possible.
53 SYM_FUNC_START(memcpy_erms)
58 SYM_FUNC_END(memcpy_erms)
60 SYM_FUNC_START(memcpy_orig)
67 * We check whether memory false dependence could occur,
68 * then jump to corresponding copy mode.
77 * Move in blocks of 4x8 bytes:
90 jae .Lcopy_forward_loop
96 * Calculate copy position to tail.
102 * At most 3 ALU operations in one cycle,
103 * so append NOPS in the same 16 bytes trunk.
106 .Lcopy_backward_loop:
110 movq -3*8(%rsi), %r10
111 movq -4*8(%rsi), %r11
112 leaq -4*8(%rsi), %rsi
115 movq %r10, -3*8(%rdi)
116 movq %r11, -4*8(%rdi)
117 leaq -4*8(%rdi), %rdi
118 jae .Lcopy_backward_loop
121 * Calculate copy position to head.
131 * Move data from 16 bytes to 31 bytes.
135 movq -2*8(%rsi, %rdx), %r10
136 movq -1*8(%rsi, %rdx), %r11
139 movq %r10, -2*8(%rdi, %rdx)
140 movq %r11, -1*8(%rdi, %rdx)
147 * Move data from 8 bytes to 15 bytes.
150 movq -1*8(%rsi, %rdx), %r9
152 movq %r9, -1*8(%rdi, %rdx)
160 * Move data from 4 bytes to 7 bytes.
163 movl -4(%rsi, %rdx), %r8d
165 movl %r8d, -4(%rdi, %rdx)
172 * Move data from 1 bytes to 3 bytes.
177 movzbq (%rsi, %rdx), %r9
179 movb %r9b, (%rdi, %rdx)
185 SYM_FUNC_END(memcpy_orig)
192 * __memcpy_mcsafe - memory copy with machine check exception handling
193 * Note that we only catch machine checks when reading the source addresses.
194 * Writes to target are posted and don't generate machine checks.
196 SYM_FUNC_START(__memcpy_mcsafe)
198 /* Less than 8 bytes? Go to byte copy loop */
201 /* Check for bad alignment of source */
203 /* Already aligned */
206 /* Copy one byte at a time until source is 8-byte aligned */
212 .L_read_leading_bytes:
214 MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes
215 MCSAFE_TEST_DST %rdi 1 .E_leading_bytes
216 .L_write_leading_bytes:
221 jnz .L_read_leading_bytes
231 MCSAFE_TEST_SRC %rsi 8 .E_read_words
232 MCSAFE_TEST_DST %rdi 8 .E_write_words
240 /* Any trailing bytes? */
243 jz .L_done_memcpy_trap
245 /* Copy trailing bytes */
247 .L_read_trailing_bytes:
249 MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes
250 MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes
251 .L_write_trailing_bytes:
256 jnz .L_read_trailing_bytes
258 /* Copy successful. Return zero */
263 SYM_FUNC_END(__memcpy_mcsafe)
264 EXPORT_SYMBOL_GPL(__memcpy_mcsafe)
266 .section .fixup, "ax"
268 * Return number of bytes not copied for any failure. Note that
269 * there is no "tail" handling since the source buffer is 8-byte
270 * aligned and poison is cacheline aligned.
281 * For write fault handling, given the destination is unaligned,
282 * we handle faults on multi-byte writes with a byte-by-byte
283 * copy up to the write-protected page.
289 jmp mcsafe_handle_tail
293 _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes)
294 _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words)
295 _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes)
296 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes)
297 _ASM_EXTABLE(.L_write_words, .E_write_words)
298 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes)