include/linux/instrumented.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2
   3 /*
   4  * This header provides generic wrappers for memory access instrumentation that
   5  * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
   6  */
   7 #ifndef _LINUX_INSTRUMENTED_H
   8 #define _LINUX_INSTRUMENTED_H
   9
  10 #include <linux/compiler.h>
  11 #include <linux/kasan-checks.h>
  12 #include <linux/kcsan-checks.h>
  13 #include <linux/kmsan-checks.h>
  14 #include <linux/types.h>
  15
  16 /**
  17  * instrument_read - instrument regular read access
  18  * @v: address of access
  19  * @size: size of access
  20  *
  21  * Instrument a regular read access. The instrumentation should be inserted
  22  * before the actual read happens.
  23  */
  24 static __always_inline void instrument_read(const volatile void *v, size_t size)
  25 {
  26         kasan_check_read(v, size);
  27         kcsan_check_read(v, size);
  28 }
  29
  30 /**
  31  * instrument_write - instrument regular write access
  32  * @v: address of access
  33  * @size: size of access
  34  *
  35  * Instrument a regular write access. The instrumentation should be inserted
  36  * before the actual write happens.
  37  */
  38 static __always_inline void instrument_write(const volatile void *v, size_t size)
  39 {
  40         kasan_check_write(v, size);
  41         kcsan_check_write(v, size);
  42 }
  43
  44 /**
  45  * instrument_read_write - instrument regular read-write access
  46  * @v: address of access
  47  * @size: size of access
  48  *
  49  * Instrument a regular write access. The instrumentation should be inserted
  50  * before the actual write happens.
  51  */
  52 static __always_inline void instrument_read_write(const volatile void *v, size_t size)
  53 {
  54         kasan_check_write(v, size);
  55         kcsan_check_read_write(v, size);
  56 }
  57
  58 /**
  59  * instrument_atomic_read - instrument atomic read access
  60  * @v: address of access
  61  * @size: size of access
  62  *
  63  * Instrument an atomic read access. The instrumentation should be inserted
  64  * before the actual read happens.
  65  */
  66 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
  67 {
  68         kasan_check_read(v, size);
  69         kcsan_check_atomic_read(v, size);
  70 }
  71
  72 /**
  73  * instrument_atomic_write - instrument atomic write access
  74  * @v: address of access
  75  * @size: size of access
  76  *
  77  * Instrument an atomic write access. The instrumentation should be inserted
  78  * before the actual write happens.
  79  */
  80 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
  81 {
  82         kasan_check_write(v, size);
  83         kcsan_check_atomic_write(v, size);
  84 }
  85
  86 /**
  87  * instrument_atomic_read_write - instrument atomic read-write access
  88  * @v: address of access
  89  * @size: size of access
  90  *
  91  * Instrument an atomic read-write access. The instrumentation should be
  92  * inserted before the actual write happens.
  93  */
  94 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
  95 {
  96         kasan_check_write(v, size);
  97         kcsan_check_atomic_read_write(v, size);
  98 }
  99
 100 /**
 101  * instrument_copy_to_user - instrument reads of copy_to_user
 102  * @to: destination address
 103  * @from: source address
 104  * @n: number of bytes to copy
 105  *
 106  * Instrument reads from kernel memory, that are due to copy_to_user (and
 107  * variants). The instrumentation must be inserted before the accesses.
 108  */
 109 static __always_inline void
 110 instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
 111 {
 112         kasan_check_read(from, n);
 113         kcsan_check_read(from, n);
 114         kmsan_copy_to_user(to, from, n, 0);
 115 }
 116
 117 /**
 118  * instrument_copy_from_user_before - add instrumentation before copy_from_user
 119  * @to: destination address
 120  * @from: source address
 121  * @n: number of bytes to copy
 122  *
 123  * Instrument writes to kernel memory, that are due to copy_from_user (and
 124  * variants). The instrumentation should be inserted before the accesses.
 125  */
 126 static __always_inline void
 127 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
 128 {
 129         kasan_check_write(to, n);
 130         kcsan_check_write(to, n);
 131 }
 132
 133 /**
 134  * instrument_copy_from_user_after - add instrumentation after copy_from_user
 135  * @to: destination address
 136  * @from: source address
 137  * @n: number of bytes to copy
 138  * @left: number of bytes not copied (as returned by copy_from_user)
 139  *
 140  * Instrument writes to kernel memory, that are due to copy_from_user (and
 141  * variants). The instrumentation should be inserted after the accesses.
 142  */
 143 static __always_inline void
 144 instrument_copy_from_user_after(const void *to, const void __user *from,
 145                                 unsigned long n, unsigned long left)
 146 {
 147         kmsan_unpoison_memory(to, n - left);
 148 }
 149
 150 /**
 151  * instrument_memcpy_before - add instrumentation before non-instrumented memcpy
 152  * @to: destination address
 153  * @from: source address
 154  * @n: number of bytes to copy
 155  *
 156  * Instrument memory accesses that happen in custom memcpy implementations. The
 157  * instrumentation should be inserted before the memcpy call.
 158  */
 159 static __always_inline void instrument_memcpy_before(void *to, const void *from,
 160                                                      unsigned long n)
 161 {
 162         kasan_check_write(to, n);
 163         kasan_check_read(from, n);
 164         kcsan_check_write(to, n);
 165         kcsan_check_read(from, n);
 166 }
 167
 168 /**
 169  * instrument_memcpy_after - add instrumentation after non-instrumented memcpy
 170  * @to: destination address
 171  * @from: source address
 172  * @n: number of bytes to copy
 173  * @left: number of bytes not copied (if known)
 174  *
 175  * Instrument memory accesses that happen in custom memcpy implementations. The
 176  * instrumentation should be inserted after the memcpy call.
 177  */
 178 static __always_inline void instrument_memcpy_after(void *to, const void *from,
 179                                                     unsigned long n,
 180                                                     unsigned long left)
 181 {
 182         kmsan_memmove(to, from, n - left);
 183 }
 184
 185 /**
 186  * instrument_get_user() - add instrumentation to get_user()-like macros
 187  * @to: destination variable, may not be address-taken
 188  *
 189  * get_user() and friends are fragile, so it may depend on the implementation
 190  * whether the instrumentation happens before or after the data is copied from
 191  * the userspace.
 192  */
 193 #define instrument_get_user(to)                         \
 194 ({                                                      \
 195         u64 __tmp = (u64)(to);                          \
 196         kmsan_unpoison_memory(&__tmp, sizeof(__tmp));   \
 197         to = __tmp;                                     \
 198 })
 199
 200
 201 /**
 202  * instrument_put_user() - add instrumentation to put_user()-like macros
 203  * @from: source address
 204  * @ptr: userspace pointer to copy to
 205  * @size: number of bytes to copy
 206  *
 207  * put_user() and friends are fragile, so it may depend on the implementation
 208  * whether the instrumentation happens before or after the data is copied from
 209  * the userspace.
 210  */
 211 #define instrument_put_user(from, ptr, size)                    \
 212 ({                                                              \
 213         kmsan_copy_to_user(ptr, &from, sizeof(from), 0);        \
 214 })
 215
 216 #endif /* _LINUX_INSTRUMENTED_H */