mm/kmsan/instrumentation.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KMSAN compiler API.
   4  *
   5  * This file implements __msan_XXX hooks that Clang inserts into the code
   6  * compiled with -fsanitize=kernel-memory.
   7  * See Documentation/dev-tools/kmsan.rst for more information on how KMSAN
   8  * instrumentation works.
   9  *
  10  * Copyright (C) 2017-2022 Google LLC
  11  * Author: Alexander Potapenko <glider@google.com>
  12  *
  13  */
  14
  15 #include "kmsan.h"
  16 #include <linux/gfp.h>
  17 #include <linux/kmsan.h>
  18 #include <linux/kmsan_string.h>
  19 #include <linux/mm.h>
  20 #include <linux/uaccess.h>
  21
  22 static inline bool is_bad_asm_addr(void *addr, uintptr_t size, bool is_store)
  23 {
  24         if (IS_ENABLED(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) &&
  25             (u64)addr < TASK_SIZE)
  26                 return true;
  27         if (!kmsan_get_metadata(addr, KMSAN_META_SHADOW))
  28                 return true;
  29         return false;
  30 }
  31
  32 static inline struct shadow_origin_ptr
  33 get_shadow_origin_ptr(void *addr, u64 size, bool store)
  34 {
  35         unsigned long ua_flags = user_access_save();
  36         struct shadow_origin_ptr ret;
  37
  38         ret = kmsan_get_shadow_origin_ptr(addr, size, store);
  39         user_access_restore(ua_flags);
  40         return ret;
  41 }
  42
  43 /*
  44  * KMSAN instrumentation functions follow. They are not declared elsewhere in
  45  * the kernel code, so they are preceded by prototypes, to silence
  46  * -Wmissing-prototypes warnings.
  47  */
  48
  49 /* Get shadow and origin pointers for a memory load with non-standard size. */
  50 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
  51                                                         uintptr_t size);
  52 struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr,
  53                                                         uintptr_t size)
  54 {
  55         return get_shadow_origin_ptr(addr, size, /*store*/ false);
  56 }
  57 EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);
  58
  59 /* Get shadow and origin pointers for a memory store with non-standard size. */
  60 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
  61                                                          uintptr_t size);
  62 struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr,
  63                                                          uintptr_t size)
  64 {
  65         return get_shadow_origin_ptr(addr, size, /*store*/ true);
  66 }
  67 EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);
  68
  69 /*
  70  * Declare functions that obtain shadow/origin pointers for loads and stores
  71  * with fixed size.
  72  */
  73 #define DECLARE_METADATA_PTR_GETTER(size)                                  \
  74         struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(      \
  75                 void *addr);                                               \
  76         struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(      \
  77                 void *addr)                                                \
  78         {                                                                  \
  79                 return get_shadow_origin_ptr(addr, size, /*store*/ false); \
  80         }                                                                  \
  81         EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size);                \
  82         struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(     \
  83                 void *addr);                                               \
  84         struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(     \
  85                 void *addr)                                                \
  86         {                                                                  \
  87                 return get_shadow_origin_ptr(addr, size, /*store*/ true);  \
  88         }                                                                  \
  89         EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)
  90
  91 DECLARE_METADATA_PTR_GETTER(1);
  92 DECLARE_METADATA_PTR_GETTER(2);
  93 DECLARE_METADATA_PTR_GETTER(4);
  94 DECLARE_METADATA_PTR_GETTER(8);
  95
  96 /*
  97  * Handle a memory store performed by inline assembly. KMSAN conservatively
  98  * attempts to unpoison the outputs of asm() directives to prevent false
  99  * positives caused by missed stores.
 100  *
 101  * __msan_instrument_asm_store() may be called for inline assembly code when
 102  * entering or leaving IRQ. We omit the check for kmsan_in_runtime() to ensure
 103  * the memory written to in these cases is also marked as initialized.
 104  */
 105 void __msan_instrument_asm_store(void *addr, uintptr_t size);
 106 void __msan_instrument_asm_store(void *addr, uintptr_t size)
 107 {
 108         unsigned long ua_flags;
 109
 110         if (!kmsan_enabled)
 111                 return;
 112
 113         ua_flags = user_access_save();
 114         /*
 115          * Most of the accesses are below 32 bytes. The exceptions so far are
 116          * clwb() (64 bytes), FPU state (512 bytes) and chsc() (4096 bytes).
 117          */
 118         if (size > 4096) {
 119                 WARN_ONCE(1, "assembly store size too big: %ld\n", size);
 120                 size = 8;
 121         }
 122         if (is_bad_asm_addr(addr, size, /*is_store*/ true)) {
 123                 user_access_restore(ua_flags);
 124                 return;
 125         }
 126         /* Unpoisoning the memory on best effort. */
 127         kmsan_internal_unpoison_memory(addr, size, /*checked*/ false);
 128         user_access_restore(ua_flags);
 129 }
 130 EXPORT_SYMBOL(__msan_instrument_asm_store);
 131
 132 /*
 133  * KMSAN instrumentation pass replaces LLVM memcpy, memmove and memset
 134  * intrinsics with calls to respective __msan_ functions. We use
 135  * get_param0_metadata() and set_retval_metadata() to store the shadow/origin
 136  * values for the destination argument of these functions and use them for the
 137  * functions' return values.
 138  */
 139 static inline void get_param0_metadata(u64 *shadow,
 140                                        depot_stack_handle_t *origin)
 141 {
 142         struct kmsan_ctx *ctx = kmsan_get_context();
 143
 144         *shadow = *(u64 *)(ctx->cstate.param_tls);
 145         *origin = ctx->cstate.param_origin_tls[0];
 146 }
 147
 148 static inline void set_retval_metadata(u64 shadow, depot_stack_handle_t origin)
 149 {
 150         struct kmsan_ctx *ctx = kmsan_get_context();
 151
 152         *(u64 *)(ctx->cstate.retval_tls) = shadow;
 153         ctx->cstate.retval_origin_tls = origin;
 154 }
 155
 156 /* Handle llvm.memmove intrinsic. */
 157 void *__msan_memmove(void *dst, const void *src, uintptr_t n);
 158 void *__msan_memmove(void *dst, const void *src, uintptr_t n)
 159 {
 160         depot_stack_handle_t origin;
 161         void *result;
 162         u64 shadow;
 163
 164         get_param0_metadata(&shadow, &origin);
 165         result = __memmove(dst, src, n);
 166         if (!n)
 167                 /* Some people call memmove() with zero length. */
 168                 return result;
 169         if (!kmsan_enabled || kmsan_in_runtime())
 170                 return result;
 171
 172         kmsan_enter_runtime();
 173         kmsan_internal_memmove_metadata(dst, (void *)src, n);
 174         kmsan_leave_runtime();
 175
 176         set_retval_metadata(shadow, origin);
 177         return result;
 178 }
 179 EXPORT_SYMBOL(__msan_memmove);
 180
 181 /* Handle llvm.memcpy intrinsic. */
 182 void *__msan_memcpy(void *dst, const void *src, uintptr_t n);
 183 void *__msan_memcpy(void *dst, const void *src, uintptr_t n)
 184 {
 185         depot_stack_handle_t origin;
 186         void *result;
 187         u64 shadow;
 188
 189         get_param0_metadata(&shadow, &origin);
 190         result = __memcpy(dst, src, n);
 191         if (!n)
 192                 /* Some people call memcpy() with zero length. */
 193                 return result;
 194
 195         if (!kmsan_enabled || kmsan_in_runtime())
 196                 return result;
 197
 198         kmsan_enter_runtime();
 199         /* Using memmove instead of memcpy doesn't affect correctness. */
 200         kmsan_internal_memmove_metadata(dst, (void *)src, n);
 201         kmsan_leave_runtime();
 202
 203         set_retval_metadata(shadow, origin);
 204         return result;
 205 }
 206 EXPORT_SYMBOL(__msan_memcpy);
 207
 208 /* Handle llvm.memset intrinsic. */
 209 void *__msan_memset(void *dst, int c, uintptr_t n);
 210 void *__msan_memset(void *dst, int c, uintptr_t n)
 211 {
 212         depot_stack_handle_t origin;
 213         void *result;
 214         u64 shadow;
 215
 216         get_param0_metadata(&shadow, &origin);
 217         result = __memset(dst, c, n);
 218         if (!kmsan_enabled || kmsan_in_runtime())
 219                 return result;
 220
 221         kmsan_enter_runtime();
 222         /*
 223          * Clang doesn't pass parameter metadata here, so it is impossible to
 224          * use shadow of @c to set up the shadow for @dst.
 225          */
 226         kmsan_internal_unpoison_memory(dst, n, /*checked*/ false);
 227         kmsan_leave_runtime();
 228
 229         set_retval_metadata(shadow, origin);
 230         return result;
 231 }
 232 EXPORT_SYMBOL(__msan_memset);
 233
 234 /*
 235  * Create a new origin from an old one. This is done when storing an
 236  * uninitialized value to memory. When reporting an error, KMSAN unrolls and
 237  * prints the whole chain of stores that preceded the use of this value.
 238  */
 239 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin);
 240 depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)
 241 {
 242         depot_stack_handle_t ret = 0;
 243         unsigned long ua_flags;
 244
 245         if (!kmsan_enabled || kmsan_in_runtime())
 246                 return ret;
 247
 248         ua_flags = user_access_save();
 249
 250         /* Creating new origins may allocate memory. */
 251         kmsan_enter_runtime();
 252         ret = kmsan_internal_chain_origin(origin);
 253         kmsan_leave_runtime();
 254         user_access_restore(ua_flags);
 255         return ret;
 256 }
 257 EXPORT_SYMBOL(__msan_chain_origin);
 258
 259 /* Poison a local variable when entering a function. */
 260 void __msan_poison_alloca(void *address, uintptr_t size, char *descr);
 261 void __msan_poison_alloca(void *address, uintptr_t size, char *descr)
 262 {
 263         depot_stack_handle_t handle;
 264         unsigned long entries[4];
 265         unsigned long ua_flags;
 266
 267         if (!kmsan_enabled || kmsan_in_runtime())
 268                 return;
 269
 270         ua_flags = user_access_save();
 271         entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;
 272         entries[1] = (u64)descr;
 273         entries[2] = (u64)__builtin_return_address(0);
 274         /*
 275          * With frame pointers enabled, it is possible to quickly fetch the
 276          * second frame of the caller stack without calling the unwinder.
 277          * Without them, simply do not bother.
 278          */
 279         if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER))
 280                 entries[3] = (u64)__builtin_return_address(1);
 281         else
 282                 entries[3] = 0;
 283
 284         /* stack_depot_save() may allocate memory. */
 285         kmsan_enter_runtime();
 286         handle = stack_depot_save(entries, ARRAY_SIZE(entries), __GFP_HIGH);
 287         kmsan_leave_runtime();
 288
 289         kmsan_internal_set_shadow_origin(address, size, -1, handle,
 290                                          /*checked*/ true);
 291         user_access_restore(ua_flags);
 292 }
 293 EXPORT_SYMBOL(__msan_poison_alloca);
 294
 295 /* Unpoison a local variable. */
 296 void __msan_unpoison_alloca(void *address, uintptr_t size);
 297 void __msan_unpoison_alloca(void *address, uintptr_t size)
 298 {
 299         if (!kmsan_enabled || kmsan_in_runtime())
 300                 return;
 301
 302         kmsan_enter_runtime();
 303         kmsan_internal_unpoison_memory(address, size, /*checked*/ true);
 304         kmsan_leave_runtime();
 305 }
 306 EXPORT_SYMBOL(__msan_unpoison_alloca);
 307
 308 /*
 309  * Report that an uninitialized value with the given origin was used in a way
 310  * that constituted undefined behavior.
 311  */
 312 void __msan_warning(u32 origin);
 313 void __msan_warning(u32 origin)
 314 {
 315         if (!kmsan_enabled || kmsan_in_runtime())
 316                 return;
 317         kmsan_enter_runtime();
 318         kmsan_report(origin, /*address*/ NULL, /*size*/ 0,
 319                      /*off_first*/ 0, /*off_last*/ 0, /*user_addr*/ NULL,
 320                      REASON_ANY);
 321         kmsan_leave_runtime();
 322 }
 323 EXPORT_SYMBOL(__msan_warning);
 324
 325 /*
 326  * At the beginning of an instrumented function, obtain the pointer to
 327  * `struct kmsan_context_state` holding the metadata for function parameters.
 328  */
 329 struct kmsan_context_state *__msan_get_context_state(void);
 330 struct kmsan_context_state *__msan_get_context_state(void)
 331 {
 332         return &kmsan_get_context()->cstate;
 333 }
 334 EXPORT_SYMBOL(__msan_get_context_state);