1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
4 #include <linux/jump_label.h>
5 #include <linux/uaccess.h>
6 #include <linux/export.h>
7 #include <linux/string.h>
8 #include <linux/types.h>
13 static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key
);
15 void enable_copy_mc_fragile(void)
17 static_branch_inc(©_mc_fragile_key
);
19 #define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key))
22 * Similar to copy_user_handle_tail, probe for the write fault point, or
23 * source exception point.
25 __visible notrace
unsigned long
26 copy_mc_fragile_handle_tail(char *to
, char *from
, unsigned len
)
28 for (; len
; --len
, to
++, from
++)
29 if (copy_mc_fragile(to
, from
, 1))
35 * No point in doing careful copying, or consulting a static key when
36 * there is no #MC handler in the CONFIG_X86_MCE=n case.
38 void enable_copy_mc_fragile(void)
41 #define copy_mc_fragile_enabled (0)
44 unsigned long copy_mc_enhanced_fast_string(void *dst
, const void *src
, unsigned len
);
47 * copy_mc_to_kernel - memory copy that handles source exceptions
49 * @dst: destination address
50 * @src: source address
51 * @len: number of bytes to copy
53 * Call into the 'fragile' version on systems that benefit from avoiding
54 * corner case poison consumption scenarios, For example, accessing
55 * poison across 2 cachelines with a single instruction. Almost all
56 * other uses case can use copy_mc_enhanced_fast_string() for a fast
57 * recoverable copy, or fallback to plain memcpy.
59 * Return 0 for success, or number of bytes not copied if there was an
62 unsigned long __must_check
copy_mc_to_kernel(void *dst
, const void *src
, unsigned len
)
64 if (copy_mc_fragile_enabled
)
65 return copy_mc_fragile(dst
, src
, len
);
66 if (static_cpu_has(X86_FEATURE_ERMS
))
67 return copy_mc_enhanced_fast_string(dst
, src
, len
);
68 memcpy(dst
, src
, len
);
71 EXPORT_SYMBOL_GPL(copy_mc_to_kernel
);
73 unsigned long __must_check
copy_mc_to_user(void *dst
, const void *src
, unsigned len
)
77 if (copy_mc_fragile_enabled
) {
79 ret
= copy_mc_fragile(dst
, src
, len
);
84 if (static_cpu_has(X86_FEATURE_ERMS
)) {
86 ret
= copy_mc_enhanced_fast_string(dst
, src
, len
);
91 return copy_user_generic(dst
, src
, len
);