kernel/events/callchain.c

   1 /*
   2  * Performance events callchain code, extracted from core.c:
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
   7  *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
   8  *
   9  * For licensing details see kernel-base/COPYING
  10  */
  11
  12 #include <linux/perf_event.h>
  13 #include <linux/slab.h>
  14 #include "internal.h"
  15
  16 struct callchain_cpus_entries {
  17         struct rcu_head                 rcu_head;
  18         struct perf_callchain_entry     *cpu_entries[0];
  19 };
  20
  21 static DEFINE_PER_CPU(int, callchain_recursion[PERF_NR_CONTEXTS]);
  22 static atomic_t nr_callchain_events;
  23 static DEFINE_MUTEX(callchain_mutex);
  24 static struct callchain_cpus_entries *callchain_cpus_entries;
  25
  26
  27 __weak void perf_callchain_kernel(struct perf_callchain_entry *entry,
  28                                   struct pt_regs *regs)
  29 {
  30 }
  31
  32 __weak void perf_callchain_user(struct perf_callchain_entry *entry,
  33                                 struct pt_regs *regs)
  34 {
  35 }
  36
  37 static void release_callchain_buffers_rcu(struct rcu_head *head)
  38 {
  39         struct callchain_cpus_entries *entries;
  40         int cpu;
  41
  42         entries = container_of(head, struct callchain_cpus_entries, rcu_head);
  43
  44         for_each_possible_cpu(cpu)
  45                 kfree(entries->cpu_entries[cpu]);
  46
  47         kfree(entries);
  48 }
  49
  50 static void release_callchain_buffers(void)
  51 {
  52         struct callchain_cpus_entries *entries;
  53
  54         entries = callchain_cpus_entries;
  55         RCU_INIT_POINTER(callchain_cpus_entries, NULL);
  56         call_rcu(&entries->rcu_head, release_callchain_buffers_rcu);
  57 }
  58
  59 static int alloc_callchain_buffers(void)
  60 {
  61         int cpu;
  62         int size;
  63         struct callchain_cpus_entries *entries;
  64
  65         /*
  66          * We can't use the percpu allocation API for data that can be
  67          * accessed from NMI. Use a temporary manual per cpu allocation
  68          * until that gets sorted out.
  69          */
  70         size = offsetof(struct callchain_cpus_entries, cpu_entries[nr_cpu_ids]);
  71
  72         entries = kzalloc(size, GFP_KERNEL);
  73         if (!entries)
  74                 return -ENOMEM;
  75
  76         size = sizeof(struct perf_callchain_entry) * PERF_NR_CONTEXTS;
  77
  78         for_each_possible_cpu(cpu) {
  79                 entries->cpu_entries[cpu] = kmalloc_node(size, GFP_KERNEL,
  80                                                          cpu_to_node(cpu));
  81                 if (!entries->cpu_entries[cpu])
  82                         goto fail;
  83         }
  84
  85         rcu_assign_pointer(callchain_cpus_entries, entries);
  86
  87         return 0;
  88
  89 fail:
  90         for_each_possible_cpu(cpu)
  91                 kfree(entries->cpu_entries[cpu]);
  92         kfree(entries);
  93
  94         return -ENOMEM;
  95 }
  96
  97 int get_callchain_buffers(void)
  98 {
  99         int err = 0;
 100         int count;
 101
 102         mutex_lock(&callchain_mutex);
 103
 104         count = atomic_inc_return(&nr_callchain_events);
 105         if (WARN_ON_ONCE(count < 1)) {
 106                 err = -EINVAL;
 107                 goto exit;
 108         }
 109
 110         if (count > 1) {
 111                 /* If the allocation failed, give up */
 112                 if (!callchain_cpus_entries)
 113                         err = -ENOMEM;
 114                 goto exit;
 115         }
 116
 117         err = alloc_callchain_buffers();
 118 exit:
 119         if (err)
 120                 atomic_dec(&nr_callchain_events);
 121
 122         mutex_unlock(&callchain_mutex);
 123
 124         return err;
 125 }
 126
 127 void put_callchain_buffers(void)
 128 {
 129         if (atomic_dec_and_mutex_lock(&nr_callchain_events, &callchain_mutex)) {
 130                 release_callchain_buffers();
 131                 mutex_unlock(&callchain_mutex);
 132         }
 133 }
 134
 135 static struct perf_callchain_entry *get_callchain_entry(int *rctx)
 136 {
 137         int cpu;
 138         struct callchain_cpus_entries *entries;
 139
 140         *rctx = get_recursion_context(this_cpu_ptr(callchain_recursion));
 141         if (*rctx == -1)
 142                 return NULL;
 143
 144         entries = rcu_dereference(callchain_cpus_entries);
 145         if (!entries)
 146                 return NULL;
 147
 148         cpu = smp_processor_id();
 149
 150         return &entries->cpu_entries[cpu][*rctx];
 151 }
 152
 153 static void
 154 put_callchain_entry(int rctx)
 155 {
 156         put_recursion_context(this_cpu_ptr(callchain_recursion), rctx);
 157 }
 158
 159 struct perf_callchain_entry *
 160 perf_callchain(struct perf_event *event, struct pt_regs *regs)
 161 {
 162         bool kernel = !event->attr.exclude_callchain_kernel;
 163         bool user   = !event->attr.exclude_callchain_user;
 164         /* Disallow cross-task user callchains. */
 165         bool crosstask = event->ctx->task && event->ctx->task != current;
 166
 167         if (!kernel && !user)
 168                 return NULL;
 169
 170         return get_perf_callchain(regs, 0, kernel, user, crosstask, true);
 171 }
 172
 173 struct perf_callchain_entry *
 174 get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
 175                    bool crosstask, bool add_mark)
 176 {
 177         struct perf_callchain_entry *entry;
 178         int rctx;
 179
 180         entry = get_callchain_entry(&rctx);
 181         if (rctx == -1)
 182                 return NULL;
 183
 184         if (!entry)
 185                 goto exit_put;
 186
 187         entry->nr = init_nr;
 188
 189         if (kernel && !user_mode(regs)) {
 190                 if (add_mark)
 191                         perf_callchain_store(entry, PERF_CONTEXT_KERNEL);
 192                 perf_callchain_kernel(entry, regs);
 193         }
 194
 195         if (user) {
 196                 if (!user_mode(regs)) {
 197                         if  (current->mm)
 198                                 regs = task_pt_regs(current);
 199                         else
 200                                 regs = NULL;
 201                 }
 202
 203                 if (regs) {
 204                         if (crosstask)
 205                                 goto exit_put;
 206
 207                         if (add_mark)
 208                                 perf_callchain_store(entry, PERF_CONTEXT_USER);
 209                         perf_callchain_user(entry, regs);
 210                 }
 211         }
 212
 213 exit_put:
 214         put_callchain_entry(rctx);
 215
 216         return entry;
 217 }