mm/kfence/report.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * KFENCE reporting.
   4  *
   5  * Copyright (C) 2020, Google LLC.
   6  */
   7
   8 #include <linux/stdarg.h>
   9
  10 #include <linux/kernel.h>
  11 #include <linux/lockdep.h>
  12 #include <linux/math.h>
  13 #include <linux/printk.h>
  14 #include <linux/sched/debug.h>
  15 #include <linux/seq_file.h>
  16 #include <linux/sprintf.h>
  17 #include <linux/stacktrace.h>
  18 #include <linux/string.h>
  19 #include <linux/sched/clock.h>
  20 #include <trace/events/error_report.h>
  21
  22 #include <asm/kfence.h>
  23
  24 #include "kfence.h"
  25
  26 /* May be overridden by <asm/kfence.h>. */
  27 #ifndef ARCH_FUNC_PREFIX
  28 #define ARCH_FUNC_PREFIX ""
  29 #endif
  30
  31 /* Helper function to either print to a seq_file or to console. */
  32 __printf(2, 3)
  33 static void seq_con_printf(struct seq_file *seq, const char *fmt, ...)
  34 {
  35         va_list args;
  36
  37         va_start(args, fmt);
  38         if (seq)
  39                 seq_vprintf(seq, fmt, args);
  40         else
  41                 vprintk(fmt, args);
  42         va_end(args);
  43 }
  44
  45 /*
  46  * Get the number of stack entries to skip to get out of MM internals. @type is
  47  * optional, and if set to NULL, assumes an allocation or free stack.
  48  */
  49 static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries,
  50                             const enum kfence_error_type *type)
  51 {
  52         char buf[64];
  53         int skipnr, fallback = 0;
  54
  55         if (type) {
  56                 /* Depending on error type, find different stack entries. */
  57                 switch (*type) {
  58                 case KFENCE_ERROR_UAF:
  59                 case KFENCE_ERROR_OOB:
  60                 case KFENCE_ERROR_INVALID:
  61                         /*
  62                          * kfence_handle_page_fault() may be called with pt_regs
  63                          * set to NULL; in that case we'll simply show the full
  64                          * stack trace.
  65                          */
  66                         return 0;
  67                 case KFENCE_ERROR_CORRUPTION:
  68                 case KFENCE_ERROR_INVALID_FREE:
  69                         break;
  70                 }
  71         }
  72
  73         for (skipnr = 0; skipnr < num_entries; skipnr++) {
  74                 int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]);
  75
  76                 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") ||
  77                     str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") ||
  78                     str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||
  79                     !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) {
  80                         /*
  81                          * In case of tail calls from any of the below to any of
  82                          * the above, optimized by the compiler such that the
  83                          * stack trace would omit the initial entry point below.
  84                          */
  85                         fallback = skipnr + 1;
  86                 }
  87
  88                 /*
  89                  * The below list should only include the initial entry points
  90                  * into the slab allocators. Includes the *_bulk() variants by
  91                  * checking prefixes.
  92                  */
  93                 if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||
  94                     str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||
  95                     str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||
  96                     str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))
  97                         goto found;
  98         }
  99         if (fallback < num_entries)
 100                 return fallback;
 101 found:
 102         skipnr++;
 103         return skipnr < num_entries ? skipnr : 0;
 104 }
 105
 106 static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta,
 107                                bool show_alloc)
 108 {
 109         const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track;
 110         u64 ts_sec = track->ts_nsec;
 111         unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC);
 112         u64 interval_nsec = local_clock() - track->ts_nsec;
 113         unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC);
 114
 115         /* Timestamp matches printk timestamp format. */
 116         seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n",
 117                        show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ?
 118                        "rcu freeing" : "freed", track->pid,
 119                        track->cpu, (unsigned long)ts_sec, rem_nsec / 1000,
 120                        (unsigned long)interval_nsec, rem_interval_nsec / 1000);
 121
 122         if (track->num_stack_entries) {
 123                 /* Skip allocation/free internals stack. */
 124                 int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
 125
 126                 /* stack_trace_seq_print() does not exist; open code our own. */
 127                 for (; i < track->num_stack_entries; i++)
 128                         seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]);
 129         } else {
 130                 seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation");
 131         }
 132 }
 133
 134 void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta)
 135 {
 136         const int size = abs(meta->size);
 137         const unsigned long start = meta->addr;
 138         const struct kmem_cache *const cache = meta->cache;
 139
 140         lockdep_assert_held(&meta->lock);
 141
 142         if (meta->state == KFENCE_OBJECT_UNUSED) {
 143                 seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata);
 144                 return;
 145         }
 146
 147         seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n",
 148                        meta - kfence_metadata, (void *)start, (void *)(start + size - 1),
 149                        size, (cache && cache->name) ? cache->name : "<destroyed>");
 150
 151         kfence_print_stack(seq, meta, true);
 152
 153         if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) {
 154                 seq_con_printf(seq, "\n");
 155                 kfence_print_stack(seq, meta, false);
 156         }
 157 }
 158
 159 /*
 160  * Show bytes at @addr that are different from the expected canary values, up to
 161  * @max_bytes.
 162  */
 163 static void print_diff_canary(unsigned long address, size_t bytes_to_show,
 164                               const struct kfence_metadata *meta)
 165 {
 166         const unsigned long show_until_addr = address + bytes_to_show;
 167         const u8 *cur, *end;
 168
 169         /* Do not show contents of object nor read into following guard page. */
 170         end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr)
 171                                                 : min(show_until_addr, PAGE_ALIGN(address)));
 172
 173         pr_cont("[");
 174         for (cur = (const u8 *)address; cur < end; cur++) {
 175                 if (*cur == KFENCE_CANARY_PATTERN_U8(cur))
 176                         pr_cont(" .");
 177                 else if (no_hash_pointers)
 178                         pr_cont(" 0x%02x", *cur);
 179                 else /* Do not leak kernel memory in non-debug builds. */
 180                         pr_cont(" !");
 181         }
 182         pr_cont(" ]");
 183 }
 184
 185 static const char *get_access_type(bool is_write)
 186 {
 187         return is_write ? "write" : "read";
 188 }
 189
 190 void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
 191                          const struct kfence_metadata *meta, enum kfence_error_type type)
 192 {
 193         unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 };
 194         const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1;
 195         int num_stack_entries;
 196         int skipnr = 0;
 197
 198         if (regs) {
 199                 num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0);
 200         } else {
 201                 num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1);
 202                 skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type);
 203         }
 204
 205         /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */
 206         if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta))
 207                 return;
 208
 209         if (meta)
 210                 lockdep_assert_held(&meta->lock);
 211         /*
 212          * Because we may generate reports in printk-unfriendly parts of the
 213          * kernel, such as scheduler code, the use of printk() could deadlock.
 214          * Until such time that all printing code here is safe in all parts of
 215          * the kernel, accept the risk, and just get our message out (given the
 216          * system might already behave unpredictably due to the memory error).
 217          * As such, also disable lockdep to hide warnings, and avoid disabling
 218          * lockdep for the rest of the kernel.
 219          */
 220         lockdep_off();
 221
 222         pr_err("==================================================================\n");
 223         /* Print report header. */
 224         switch (type) {
 225         case KFENCE_ERROR_OOB: {
 226                 const bool left_of_object = address < meta->addr;
 227
 228                 pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write),
 229                        (void *)stack_entries[skipnr]);
 230                 pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n",
 231                        get_access_type(is_write), (void *)address,
 232                        left_of_object ? meta->addr - address : address - meta->addr,
 233                        left_of_object ? "left" : "right", object_index);
 234                 break;
 235         }
 236         case KFENCE_ERROR_UAF:
 237                 pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write),
 238                        (void *)stack_entries[skipnr]);
 239                 pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n",
 240                        get_access_type(is_write), (void *)address, object_index);
 241                 break;
 242         case KFENCE_ERROR_CORRUPTION:
 243                 pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]);
 244                 pr_err("Corrupted memory at 0x%p ", (void *)address);
 245                 print_diff_canary(address, 16, meta);
 246                 pr_cont(" (in kfence-#%td):\n", object_index);
 247                 break;
 248         case KFENCE_ERROR_INVALID:
 249                 pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write),
 250                        (void *)stack_entries[skipnr]);
 251                 pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write),
 252                        (void *)address);
 253                 break;
 254         case KFENCE_ERROR_INVALID_FREE:
 255                 pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]);
 256                 pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address,
 257                        object_index);
 258                 break;
 259         }
 260
 261         /* Print stack trace and object info. */
 262         stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0);
 263
 264         if (meta) {
 265                 pr_err("\n");
 266                 kfence_print_object(NULL, meta);
 267         }
 268
 269         /* Print report footer. */
 270         pr_err("\n");
 271         if (no_hash_pointers && regs)
 272                 show_regs(regs);
 273         else
 274                 dump_stack_print_info(KERN_ERR);
 275         trace_error_report_end(ERROR_DETECTOR_KFENCE, address);
 276         pr_err("==================================================================\n");
 277
 278         lockdep_on();
 279
 280         check_panic_on_warn("KFENCE");
 281
 282         /* We encountered a memory safety error, taint the kernel! */
 283         add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK);
 284 }
 285
 286 #ifdef CONFIG_PRINTK
 287 static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack)
 288 {
 289         int i, j;
 290
 291         i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL);
 292         for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j)
 293                 kp_stack[j] = (void *)track->stack_entries[i];
 294         if (j < KS_ADDRS_COUNT)
 295                 kp_stack[j] = NULL;
 296 }
 297
 298 bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
 299 {
 300         struct kfence_metadata *meta = addr_to_metadata((unsigned long)object);
 301         unsigned long flags;
 302
 303         if (!meta)
 304                 return false;
 305
 306         /*
 307          * If state is UNUSED at least show the pointer requested; the rest
 308          * would be garbage data.
 309          */
 310         kpp->kp_ptr = object;
 311
 312         /* Requesting info an a never-used object is almost certainly a bug. */
 313         if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED))
 314                 return true;
 315
 316         raw_spin_lock_irqsave(&meta->lock, flags);
 317
 318         kpp->kp_slab = slab;
 319         kpp->kp_slab_cache = meta->cache;
 320         kpp->kp_objp = (void *)meta->addr;
 321         kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack);
 322         if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING)
 323                 kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack);
 324         /* get_stack_skipnr() ensures the first entry is outside allocator. */
 325         kpp->kp_ret = kpp->kp_stack[0];
 326
 327         raw_spin_unlock_irqrestore(&meta->lock, flags);
 328
 329         return true;
 330 }
 331 #endif