mm/kmsan/hooks.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KMSAN hooks for kernel subsystems.
   4  *
   5  * These functions handle creation of KMSAN metadata for memory allocations.
   6  *
   7  * Copyright (C) 2018-2022 Google LLC
   8  * Author: Alexander Potapenko <glider@google.com>
   9  *
  10  */
  11
  12 #include <linux/cacheflush.h>
  13 #include <linux/dma-direction.h>
  14 #include <linux/gfp.h>
  15 #include <linux/kmsan.h>
  16 #include <linux/mm.h>
  17 #include <linux/mm_types.h>
  18 #include <linux/scatterlist.h>
  19 #include <linux/slab.h>
  20 #include <linux/uaccess.h>
  21 #include <linux/usb.h>
  22
  23 #include "../internal.h"
  24 #include "../slab.h"
  25 #include "kmsan.h"
  26
  27 /*
  28  * Instrumented functions shouldn't be called under
  29  * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
  30  * skipping effects of functions like memset() inside instrumented code.
  31  */
  32
  33 void kmsan_task_create(struct task_struct *task)
  34 {
  35         kmsan_enter_runtime();
  36         kmsan_internal_task_create(task);
  37         kmsan_leave_runtime();
  38 }
  39
  40 void kmsan_task_exit(struct task_struct *task)
  41 {
  42         if (!kmsan_enabled || kmsan_in_runtime())
  43                 return;
  44
  45         kmsan_disable_current();
  46 }
  47
  48 void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
  49 {
  50         if (unlikely(object == NULL))
  51                 return;
  52         if (!kmsan_enabled || kmsan_in_runtime())
  53                 return;
  54         /*
  55          * There's a ctor or this is an RCU cache - do nothing. The memory
  56          * status hasn't changed since last use.
  57          */
  58         if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
  59                 return;
  60
  61         kmsan_enter_runtime();
  62         if (flags & __GFP_ZERO)
  63                 kmsan_internal_unpoison_memory(object, s->object_size,
  64                                                KMSAN_POISON_CHECK);
  65         else
  66                 kmsan_internal_poison_memory(object, s->object_size, flags,
  67                                              KMSAN_POISON_CHECK);
  68         kmsan_leave_runtime();
  69 }
  70
  71 void kmsan_slab_free(struct kmem_cache *s, void *object)
  72 {
  73         if (!kmsan_enabled || kmsan_in_runtime())
  74                 return;
  75
  76         /* RCU slabs could be legally used after free within the RCU period */
  77         if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
  78                 return;
  79         /*
  80          * If there's a constructor, freed memory must remain in the same state
  81          * until the next allocation. We cannot save its state to detect
  82          * use-after-free bugs, instead we just keep it unpoisoned.
  83          */
  84         if (s->ctor)
  85                 return;
  86         kmsan_enter_runtime();
  87         kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
  88                                      KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
  89         kmsan_leave_runtime();
  90 }
  91
  92 void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
  93 {
  94         if (unlikely(ptr == NULL))
  95                 return;
  96         if (!kmsan_enabled || kmsan_in_runtime())
  97                 return;
  98         kmsan_enter_runtime();
  99         if (flags & __GFP_ZERO)
 100                 kmsan_internal_unpoison_memory((void *)ptr, size,
 101                                                /*checked*/ true);
 102         else
 103                 kmsan_internal_poison_memory((void *)ptr, size, flags,
 104                                              KMSAN_POISON_CHECK);
 105         kmsan_leave_runtime();
 106 }
 107
 108 void kmsan_kfree_large(const void *ptr)
 109 {
 110         struct page *page;
 111
 112         if (!kmsan_enabled || kmsan_in_runtime())
 113                 return;
 114         kmsan_enter_runtime();
 115         page = virt_to_head_page((void *)ptr);
 116         KMSAN_WARN_ON(ptr != page_address(page));
 117         kmsan_internal_poison_memory((void *)ptr,
 118                                      page_size(page),
 119                                      GFP_KERNEL,
 120                                      KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
 121         kmsan_leave_runtime();
 122 }
 123
 124 static unsigned long vmalloc_shadow(unsigned long addr)
 125 {
 126         return (unsigned long)kmsan_get_metadata((void *)addr,
 127                                                  KMSAN_META_SHADOW);
 128 }
 129
 130 static unsigned long vmalloc_origin(unsigned long addr)
 131 {
 132         return (unsigned long)kmsan_get_metadata((void *)addr,
 133                                                  KMSAN_META_ORIGIN);
 134 }
 135
 136 void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
 137 {
 138         __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
 139         __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
 140         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
 141         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
 142 }
 143
 144 /*
 145  * This function creates new shadow/origin pages for the physical pages mapped
 146  * into the virtual memory. If those physical pages already had shadow/origin,
 147  * those are ignored.
 148  */
 149 int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
 150                              phys_addr_t phys_addr, pgprot_t prot,
 151                              unsigned int page_shift)
 152 {
 153         gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;
 154         struct page *shadow, *origin;
 155         unsigned long off = 0;
 156         int nr, err = 0, clean = 0, mapped;
 157
 158         if (!kmsan_enabled || kmsan_in_runtime())
 159                 return 0;
 160
 161         nr = (end - start) / PAGE_SIZE;
 162         kmsan_enter_runtime();
 163         for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
 164                 shadow = alloc_pages(gfp_mask, 1);
 165                 origin = alloc_pages(gfp_mask, 1);
 166                 if (!shadow || !origin) {
 167                         err = -ENOMEM;
 168                         goto ret;
 169                 }
 170                 mapped = __vmap_pages_range_noflush(
 171                         vmalloc_shadow(start + off),
 172                         vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
 173                         PAGE_SHIFT);
 174                 if (mapped) {
 175                         err = mapped;
 176                         goto ret;
 177                 }
 178                 shadow = NULL;
 179                 mapped = __vmap_pages_range_noflush(
 180                         vmalloc_origin(start + off),
 181                         vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
 182                         PAGE_SHIFT);
 183                 if (mapped) {
 184                         __vunmap_range_noflush(
 185                                 vmalloc_shadow(start + off),
 186                                 vmalloc_shadow(start + off + PAGE_SIZE));
 187                         err = mapped;
 188                         goto ret;
 189                 }
 190                 origin = NULL;
 191         }
 192         /* Page mapping loop finished normally, nothing to clean up. */
 193         clean = 0;
 194
 195 ret:
 196         if (clean > 0) {
 197                 /*
 198                  * Something went wrong. Clean up shadow/origin pages allocated
 199                  * on the last loop iteration, then delete mappings created
 200                  * during the previous iterations.
 201                  */
 202                 if (shadow)
 203                         __free_pages(shadow, 1);
 204                 if (origin)
 205                         __free_pages(origin, 1);
 206                 __vunmap_range_noflush(
 207                         vmalloc_shadow(start),
 208                         vmalloc_shadow(start + clean * PAGE_SIZE));
 209                 __vunmap_range_noflush(
 210                         vmalloc_origin(start),
 211                         vmalloc_origin(start + clean * PAGE_SIZE));
 212         }
 213         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
 214         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
 215         kmsan_leave_runtime();
 216         return err;
 217 }
 218
 219 void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
 220 {
 221         unsigned long v_shadow, v_origin;
 222         struct page *shadow, *origin;
 223         int nr;
 224
 225         if (!kmsan_enabled || kmsan_in_runtime())
 226                 return;
 227
 228         nr = (end - start) / PAGE_SIZE;
 229         kmsan_enter_runtime();
 230         v_shadow = (unsigned long)vmalloc_shadow(start);
 231         v_origin = (unsigned long)vmalloc_origin(start);
 232         for (int i = 0; i < nr;
 233              i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
 234                 shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
 235                 origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
 236                 __vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
 237                 __vunmap_range_noflush(v_origin, vmalloc_origin(end));
 238                 if (shadow)
 239                         __free_pages(shadow, 1);
 240                 if (origin)
 241                         __free_pages(origin, 1);
 242         }
 243         flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
 244         flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
 245         kmsan_leave_runtime();
 246 }
 247
 248 void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
 249                         size_t left)
 250 {
 251         unsigned long ua_flags;
 252
 253         if (!kmsan_enabled || kmsan_in_runtime())
 254                 return;
 255         /*
 256          * At this point we've copied the memory already. It's hard to check it
 257          * before copying, as the size of actually copied buffer is unknown.
 258          */
 259
 260         /* copy_to_user() may copy zero bytes. No need to check. */
 261         if (!to_copy)
 262                 return;
 263         /* Or maybe copy_to_user() failed to copy anything. */
 264         if (to_copy <= left)
 265                 return;
 266
 267         ua_flags = user_access_save();
 268         if (!IS_ENABLED(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) ||
 269             (u64)to < TASK_SIZE) {
 270                 /* This is a user memory access, check it. */
 271                 kmsan_internal_check_memory((void *)from, to_copy - left, to,
 272                                             REASON_COPY_TO_USER);
 273         } else {
 274                 /* Otherwise this is a kernel memory access. This happens when a
 275                  * compat syscall passes an argument allocated on the kernel
 276                  * stack to a real syscall.
 277                  * Don't check anything, just copy the shadow of the copied
 278                  * bytes.
 279                  */
 280                 kmsan_internal_memmove_metadata((void *)to, (void *)from,
 281                                                 to_copy - left);
 282         }
 283         user_access_restore(ua_flags);
 284 }
 285 EXPORT_SYMBOL(kmsan_copy_to_user);
 286
 287 void kmsan_memmove(void *to, const void *from, size_t size)
 288 {
 289         if (!kmsan_enabled || kmsan_in_runtime())
 290                 return;
 291
 292         kmsan_enter_runtime();
 293         kmsan_internal_memmove_metadata(to, (void *)from, size);
 294         kmsan_leave_runtime();
 295 }
 296 EXPORT_SYMBOL(kmsan_memmove);
 297
 298 /* Helper function to check an URB. */
 299 void kmsan_handle_urb(const struct urb *urb, bool is_out)
 300 {
 301         if (!urb)
 302                 return;
 303         if (is_out)
 304                 kmsan_internal_check_memory(urb->transfer_buffer,
 305                                             urb->transfer_buffer_length,
 306                                             /*user_addr*/ NULL,
 307                                             REASON_SUBMIT_URB);
 308         else
 309                 kmsan_internal_unpoison_memory(urb->transfer_buffer,
 310                                                urb->transfer_buffer_length,
 311                                                /*checked*/ false);
 312 }
 313 EXPORT_SYMBOL_GPL(kmsan_handle_urb);
 314
 315 static void kmsan_handle_dma_page(const void *addr, size_t size,
 316                                   enum dma_data_direction dir)
 317 {
 318         switch (dir) {
 319         case DMA_BIDIRECTIONAL:
 320                 kmsan_internal_check_memory((void *)addr, size,
 321                                             /*user_addr*/ NULL, REASON_ANY);
 322                 kmsan_internal_unpoison_memory((void *)addr, size,
 323                                                /*checked*/ false);
 324                 break;
 325         case DMA_TO_DEVICE:
 326                 kmsan_internal_check_memory((void *)addr, size,
 327                                             /*user_addr*/ NULL, REASON_ANY);
 328                 break;
 329         case DMA_FROM_DEVICE:
 330                 kmsan_internal_unpoison_memory((void *)addr, size,
 331                                                /*checked*/ false);
 332                 break;
 333         case DMA_NONE:
 334                 break;
 335         }
 336 }
 337
 338 /* Helper function to handle DMA data transfers. */
 339 void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
 340                       enum dma_data_direction dir)
 341 {
 342         u64 page_offset, to_go, addr;
 343
 344         if (PageHighMem(page))
 345                 return;
 346         addr = (u64)page_address(page) + offset;
 347         /*
 348          * The kernel may occasionally give us adjacent DMA pages not belonging
 349          * to the same allocation. Process them separately to avoid triggering
 350          * internal KMSAN checks.
 351          */
 352         while (size > 0) {
 353                 page_offset = offset_in_page(addr);
 354                 to_go = min(PAGE_SIZE - page_offset, (u64)size);
 355                 kmsan_handle_dma_page((void *)addr, to_go, dir);
 356                 addr += to_go;
 357                 size -= to_go;
 358         }
 359 }
 360
 361 void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
 362                          enum dma_data_direction dir)
 363 {
 364         struct scatterlist *item;
 365         int i;
 366
 367         for_each_sg(sg, item, nents, i)
 368                 kmsan_handle_dma(sg_page(item), item->offset, item->length,
 369                                  dir);
 370 }
 371
 372 /* Functions from kmsan-checks.h follow. */
 373
 374 /*
 375  * To create an origin, kmsan_poison_memory() unwinds the stacks and stores it
 376  * into the stack depot. This may cause deadlocks if done from within KMSAN
 377  * runtime, therefore we bail out if kmsan_in_runtime().
 378  */
 379 void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
 380 {
 381         if (!kmsan_enabled || kmsan_in_runtime())
 382                 return;
 383         kmsan_enter_runtime();
 384         /* The users may want to poison/unpoison random memory. */
 385         kmsan_internal_poison_memory((void *)address, size, flags,
 386                                      KMSAN_POISON_NOCHECK);
 387         kmsan_leave_runtime();
 388 }
 389 EXPORT_SYMBOL(kmsan_poison_memory);
 390
 391 /*
 392  * Unlike kmsan_poison_memory(), this function can be used from within KMSAN
 393  * runtime, because it does not trigger allocations or call instrumented code.
 394  */
 395 void kmsan_unpoison_memory(const void *address, size_t size)
 396 {
 397         unsigned long ua_flags;
 398
 399         if (!kmsan_enabled)
 400                 return;
 401
 402         ua_flags = user_access_save();
 403         /* The users may want to poison/unpoison random memory. */
 404         kmsan_internal_unpoison_memory((void *)address, size,
 405                                        KMSAN_POISON_NOCHECK);
 406         user_access_restore(ua_flags);
 407 }
 408 EXPORT_SYMBOL(kmsan_unpoison_memory);
 409
 410 /*
 411  * Version of kmsan_unpoison_memory() called from IRQ entry functions.
 412  */
 413 void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
 414 {
 415         kmsan_unpoison_memory((void *)regs, sizeof(*regs));
 416 }
 417
 418 void kmsan_check_memory(const void *addr, size_t size)
 419 {
 420         if (!kmsan_enabled)
 421                 return;
 422         return kmsan_internal_check_memory((void *)addr, size,
 423                                            /*user_addr*/ NULL, REASON_ANY);
 424 }
 425 EXPORT_SYMBOL(kmsan_check_memory);
 426
 427 void kmsan_enable_current(void)
 428 {
 429         KMSAN_WARN_ON(current->kmsan_ctx.depth == 0);
 430         current->kmsan_ctx.depth--;
 431 }
 432 EXPORT_SYMBOL(kmsan_enable_current);
 433
 434 void kmsan_disable_current(void)
 435 {
 436         current->kmsan_ctx.depth++;
 437         KMSAN_WARN_ON(current->kmsan_ctx.depth == 0);
 438 }
 439 EXPORT_SYMBOL(kmsan_disable_current);