Merge tag 'block-5.9-2020-08-14' of git://git.kernel.dk/linux-block

[linux/fpc-iii.git] / tools / perf / util / thread-stack.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * thread-stack.c: Synthesize a thread's stack using call / return events
 * Copyright (c) 2014, Intel Corporation.
 */

#include <linux/rbtree.h>
#include <linux/list.h>
#include <linux/log2.h>
#include <linux/zalloc.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include "thread.h"
#include "event.h"
#include "machine.h"
#include "env.h"
#include "debug.h"
#include "symbol.h"
#include "comm.h"
#include "call-path.h"
#include "thread-stack.h"

#define STACK_GROWTH 2048

/*
 * State of retpoline detection.
 *
 * RETPOLINE_NONE: no retpoline detection
 * X86_RETPOLINE_POSSIBLE: x86 retpoline possible
 * X86_RETPOLINE_DETECTED: x86 retpoline detected
 */
enum retpoline_state_t {
        RETPOLINE_NONE,
        X86_RETPOLINE_POSSIBLE,
        X86_RETPOLINE_DETECTED,
};

/**
 * struct thread_stack_entry - thread stack entry.
 * @ret_addr: return address
 * @timestamp: timestamp (if known)
 * @ref: external reference (e.g. db_id of sample)
 * @branch_count: the branch count when the entry was created
 * @insn_count: the instruction count when the entry was created
 * @cyc_count the cycle count when the entry was created
 * @db_id: id used for db-export
 * @cp: call path
 * @no_call: a 'call' was not seen
 * @trace_end: a 'call' but trace ended
 * @non_call: a branch but not a 'call' to the start of a different symbol
 */
struct thread_stack_entry {
        u64 ret_addr;
        u64 timestamp;
        u64 ref;
        u64 branch_count;
        u64 insn_count;
        u64 cyc_count;
        u64 db_id;
        struct call_path *cp;
        bool no_call;
        bool trace_end;
        bool non_call;
};

/**
 * struct thread_stack - thread stack constructed from 'call' and 'return'
 *                       branch samples.
 * @stack: array that holds the stack
 * @cnt: number of entries in the stack
 * @sz: current maximum stack size
 * @trace_nr: current trace number
 * @branch_count: running branch count
 * @insn_count: running  instruction count
 * @cyc_count running  cycle count
 * @kernel_start: kernel start address
 * @last_time: last timestamp
 * @crp: call/return processor
 * @comm: current comm
 * @arr_sz: size of array if this is the first element of an array
 * @rstate: used to detect retpolines
 * @br_stack_rb: branch stack (ring buffer)
 * @br_stack_sz: maximum branch stack size
 * @br_stack_pos: current position in @br_stack_rb
 * @mispred_all: mark all branches as mispredicted
 */
struct thread_stack {
        struct thread_stack_entry *stack;
        size_t cnt;
        size_t sz;
        u64 trace_nr;
        u64 branch_count;
        u64 insn_count;
        u64 cyc_count;
        u64 kernel_start;
        u64 last_time;
        struct call_return_processor *crp;
        struct comm *comm;
        unsigned int arr_sz;
        enum retpoline_state_t rstate;
        struct branch_stack *br_stack_rb;
        unsigned int br_stack_sz;
        unsigned int br_stack_pos;
        bool mispred_all;
};

/*
 * Assume pid == tid == 0 identifies the idle task as defined by
 * perf_session__register_idle_thread(). The idle task is really 1 task per cpu,
 * and therefore requires a stack for each cpu.
 */
static inline bool thread_stack__per_cpu(struct thread *thread)
{
        return !(thread->tid || thread->pid_);
}

static int thread_stack__grow(struct thread_stack *ts)
{
        struct thread_stack_entry *new_stack;
        size_t sz, new_sz;

        new_sz = ts->sz + STACK_GROWTH;
        sz = new_sz * sizeof(struct thread_stack_entry);

        new_stack = realloc(ts->stack, sz);
        if (!new_stack)
                return -ENOMEM;

        ts->stack = new_stack;
        ts->sz = new_sz;

        return 0;
}

static int thread_stack__init(struct thread_stack *ts, struct thread *thread,
                              struct call_return_processor *crp,
                              bool callstack, unsigned int br_stack_sz)
{
        int err;

        if (callstack) {
                err = thread_stack__grow(ts);
                if (err)
                        return err;
        }

        if (br_stack_sz) {
                size_t sz = sizeof(struct branch_stack);

                sz += br_stack_sz * sizeof(struct branch_entry);
                ts->br_stack_rb = zalloc(sz);
                if (!ts->br_stack_rb)
                        return -ENOMEM;
                ts->br_stack_sz = br_stack_sz;
        }

        if (thread->maps && thread->maps->machine) {
                struct machine *machine = thread->maps->machine;
                const char *arch = perf_env__arch(machine->env);

                ts->kernel_start = machine__kernel_start(machine);
                if (!strcmp(arch, "x86"))
                        ts->rstate = X86_RETPOLINE_POSSIBLE;
        } else {
                ts->kernel_start = 1ULL << 63;
        }
        ts->crp = crp;

        return 0;
}

static struct thread_stack *thread_stack__new(struct thread *thread, int cpu,
                                              struct call_return_processor *crp,
                                              bool callstack,
                                              unsigned int br_stack_sz)
{
        struct thread_stack *ts = thread->ts, *new_ts;
        unsigned int old_sz = ts ? ts->arr_sz : 0;
        unsigned int new_sz = 1;

        if (thread_stack__per_cpu(thread) && cpu > 0)
                new_sz = roundup_pow_of_two(cpu + 1);

        if (!ts || new_sz > old_sz) {
                new_ts = calloc(new_sz, sizeof(*ts));
                if (!new_ts)
                        return NULL;
                if (ts)
                        memcpy(new_ts, ts, old_sz * sizeof(*ts));
                new_ts->arr_sz = new_sz;
                zfree(&thread->ts);
                thread->ts = new_ts;
                ts = new_ts;
        }

        if (thread_stack__per_cpu(thread) && cpu > 0 &&
            (unsigned int)cpu < ts->arr_sz)
                ts += cpu;

        if (!ts->stack &&
            thread_stack__init(ts, thread, crp, callstack, br_stack_sz))
                return NULL;

        return ts;
}

static struct thread_stack *thread__cpu_stack(struct thread *thread, int cpu)
{
        struct thread_stack *ts = thread->ts;

        if (cpu < 0)
                cpu = 0;

        if (!ts || (unsigned int)cpu >= ts->arr_sz)
                return NULL;

        ts += cpu;

        if (!ts->stack)
                return NULL;

        return ts;
}

static inline struct thread_stack *thread__stack(struct thread *thread,
                                                    int cpu)
{
        if (!thread)
                return NULL;

        if (thread_stack__per_cpu(thread))
                return thread__cpu_stack(thread, cpu);

        return thread->ts;
}

static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,
                              bool trace_end)
{
        int err = 0;

        if (ts->cnt == ts->sz) {
                err = thread_stack__grow(ts);
                if (err) {
                        pr_warning("Out of memory: discarding thread stack\n");
                        ts->cnt = 0;
                }
        }

        ts->stack[ts->cnt].trace_end = trace_end;
        ts->stack[ts->cnt++].ret_addr = ret_addr;

        return err;
}

static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)
{
        size_t i;

        /*
         * In some cases there may be functions which are not seen to return.
         * For example when setjmp / longjmp has been used.  Or the perf context
         * switch in the kernel which doesn't stop and start tracing in exactly
         * the same code path.  When that happens the return address will be
         * further down the stack.  If the return address is not found at all,
         * we assume the opposite (i.e. this is a return for a call that wasn't
         * seen for some reason) and leave the stack alone.
         */
        for (i = ts->cnt; i; ) {
                if (ts->stack[--i].ret_addr == ret_addr) {
                        ts->cnt = i;
                        return;
                }
        }
}

static void thread_stack__pop_trace_end(struct thread_stack *ts)
{
        size_t i;

        for (i = ts->cnt; i; ) {
                if (ts->stack[--i].trace_end)
                        ts->cnt = i;
                else
                        return;
        }
}

static bool thread_stack__in_kernel(struct thread_stack *ts)
{
        if (!ts->cnt)
                return false;

        return ts->stack[ts->cnt - 1].cp->in_kernel;
}

static int thread_stack__call_return(struct thread *thread,
                                     struct thread_stack *ts, size_t idx,
                                     u64 timestamp, u64 ref, bool no_return)
{
        struct call_return_processor *crp = ts->crp;
        struct thread_stack_entry *tse;
        struct call_return cr = {
                .thread = thread,
                .comm = ts->comm,
                .db_id = 0,
        };
        u64 *parent_db_id;

        tse = &ts->stack[idx];
        cr.cp = tse->cp;
        cr.call_time = tse->timestamp;
        cr.return_time = timestamp;
        cr.branch_count = ts->branch_count - tse->branch_count;
        cr.insn_count = ts->insn_count - tse->insn_count;
        cr.cyc_count = ts->cyc_count - tse->cyc_count;
        cr.db_id = tse->db_id;
        cr.call_ref = tse->ref;
        cr.return_ref = ref;
        if (tse->no_call)
                cr.flags |= CALL_RETURN_NO_CALL;
        if (no_return)
                cr.flags |= CALL_RETURN_NO_RETURN;
        if (tse->non_call)
                cr.flags |= CALL_RETURN_NON_CALL;

        /*
         * The parent db_id must be assigned before exporting the child. Note
         * it is not possible to export the parent first because its information
         * is not yet complete because its 'return' has not yet been processed.
         */
        parent_db_id = idx ? &(tse - 1)->db_id : NULL;

        return crp->process(&cr, parent_db_id, crp->data);
}

static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)
{
        struct call_return_processor *crp = ts->crp;
        int err;

        if (!crp) {
                ts->cnt = 0;
                ts->br_stack_pos = 0;
                if (ts->br_stack_rb)
                        ts->br_stack_rb->nr = 0;
                return 0;
        }

        while (ts->cnt) {
                err = thread_stack__call_return(thread, ts, --ts->cnt,
                                                ts->last_time, 0, true);
                if (err) {
                        pr_err("Error flushing thread stack!\n");
                        ts->cnt = 0;
                        return err;
                }
        }

        return 0;
}

int thread_stack__flush(struct thread *thread)
{
        struct thread_stack *ts = thread->ts;
        unsigned int pos;
        int err = 0;

        if (ts) {
                for (pos = 0; pos < ts->arr_sz; pos++) {
                        int ret = __thread_stack__flush(thread, ts + pos);

                        if (ret)
                                err = ret;
                }
        }

        return err;
}

static void thread_stack__update_br_stack(struct thread_stack *ts, u32 flags,
                                          u64 from_ip, u64 to_ip)
{
        struct branch_stack *bs = ts->br_stack_rb;
        struct branch_entry *be;

        if (!ts->br_stack_pos)
                ts->br_stack_pos = ts->br_stack_sz;

        ts->br_stack_pos -= 1;

        be              = &bs->entries[ts->br_stack_pos];
        be->from        = from_ip;
        be->to          = to_ip;
        be->flags.value = 0;
        be->flags.abort = !!(flags & PERF_IP_FLAG_TX_ABORT);
        be->flags.in_tx = !!(flags & PERF_IP_FLAG_IN_TX);
        /* No support for mispredict */
        be->flags.mispred = ts->mispred_all;

        if (bs->nr < ts->br_stack_sz)
                bs->nr += 1;
}

int thread_stack__event(struct thread *thread, int cpu, u32 flags, u64 from_ip,
                        u64 to_ip, u16 insn_len, u64 trace_nr, bool callstack,
                        unsigned int br_stack_sz, bool mispred_all)
{
        struct thread_stack *ts = thread__stack(thread, cpu);

        if (!thread)
                return -EINVAL;

        if (!ts) {
                ts = thread_stack__new(thread, cpu, NULL, callstack, br_stack_sz);
                if (!ts) {
                        pr_warning("Out of memory: no thread stack\n");
                        return -ENOMEM;
                }
                ts->trace_nr = trace_nr;
                ts->mispred_all = mispred_all;
        }

        /*
         * When the trace is discontinuous, the trace_nr changes.  In that case
         * the stack might be completely invalid.  Better to report nothing than
         * to report something misleading, so flush the stack.
         */
        if (trace_nr != ts->trace_nr) {
                if (ts->trace_nr)
                        __thread_stack__flush(thread, ts);
                ts->trace_nr = trace_nr;
        }

        if (br_stack_sz)
                thread_stack__update_br_stack(ts, flags, from_ip, to_ip);

        /*
         * Stop here if thread_stack__process() is in use, or not recording call
         * stack.
         */
        if (ts->crp || !callstack)
                return 0;

        if (flags & PERF_IP_FLAG_CALL) {
                u64 ret_addr;

                if (!to_ip)
                        return 0;
                ret_addr = from_ip + insn_len;
                if (ret_addr == to_ip)
                        return 0; /* Zero-length calls are excluded */
                return thread_stack__push(ts, ret_addr,
                                          flags & PERF_IP_FLAG_TRACE_END);
        } else if (flags & PERF_IP_FLAG_TRACE_BEGIN) {
                /*
                 * If the caller did not change the trace number (which would
                 * have flushed the stack) then try to make sense of the stack.
                 * Possibly, tracing began after returning to the current
                 * address, so try to pop that. Also, do not expect a call made
                 * when the trace ended, to return, so pop that.
                 */
                thread_stack__pop(ts, to_ip);
                thread_stack__pop_trace_end(ts);
        } else if ((flags & PERF_IP_FLAG_RETURN) && from_ip) {
                thread_stack__pop(ts, to_ip);
        }

        return 0;
}

void thread_stack__set_trace_nr(struct thread *thread, int cpu, u64 trace_nr)
{
        struct thread_stack *ts = thread__stack(thread, cpu);

        if (!ts)
                return;

        if (trace_nr != ts->trace_nr) {
                if (ts->trace_nr)
                        __thread_stack__flush(thread, ts);
                ts->trace_nr = trace_nr;
        }
}

static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)
{
        __thread_stack__flush(thread, ts);
        zfree(&ts->stack);
        zfree(&ts->br_stack_rb);
}

static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)
{
        unsigned int arr_sz = ts->arr_sz;

        __thread_stack__free(thread, ts);
        memset(ts, 0, sizeof(*ts));
        ts->arr_sz = arr_sz;
}

void thread_stack__free(struct thread *thread)
{
        struct thread_stack *ts = thread->ts;
        unsigned int pos;

        if (ts) {
                for (pos = 0; pos < ts->arr_sz; pos++)
                        __thread_stack__free(thread, ts + pos);
                zfree(&thread->ts);
        }
}

static inline u64 callchain_context(u64 ip, u64 kernel_start)
{
        return ip < kernel_start ? PERF_CONTEXT_USER : PERF_CONTEXT_KERNEL;
}

void thread_stack__sample(struct thread *thread, int cpu,
                          struct ip_callchain *chain,
                          size_t sz, u64 ip, u64 kernel_start)
{
        struct thread_stack *ts = thread__stack(thread, cpu);
        u64 context = callchain_context(ip, kernel_start);
        u64 last_context;
        size_t i, j;

        if (sz < 2) {
                chain->nr = 0;
                return;
        }

        chain->ips[0] = context;
        chain->ips[1] = ip;

        if (!ts) {
                chain->nr = 2;
                return;
        }

        last_context = context;

        for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {
                ip = ts->stack[ts->cnt - j].ret_addr;
                context = callchain_context(ip, kernel_start);
                if (context != last_context) {
                        if (i >= sz - 1)
                                break;
                        chain->ips[i++] = context;
                        last_context = context;
                }
                chain->ips[i] = ip;
        }

        chain->nr = i;
}

/*
 * Hardware sample records, created some time after the event occurred, need to
 * have subsequent addresses removed from the call chain.
 */
void thread_stack__sample_late(struct thread *thread, int cpu,
                               struct ip_callchain *chain, size_t sz,
                               u64 sample_ip, u64 kernel_start)
{
        struct thread_stack *ts = thread__stack(thread, cpu);
        u64 sample_context = callchain_context(sample_ip, kernel_start);
        u64 last_context, context, ip;
        size_t nr = 0, j;

        if (sz < 2) {
                chain->nr = 0;
                return;
        }

        if (!ts)
                goto out;

        /*
         * When tracing kernel space, kernel addresses occur at the top of the
         * call chain after the event occurred but before tracing stopped.
         * Skip them.
         */
        for (j = 1; j <= ts->cnt; j++) {
                ip = ts->stack[ts->cnt - j].ret_addr;
                context = callchain_context(ip, kernel_start);
                if (context == PERF_CONTEXT_USER ||
                    (context == sample_context && ip == sample_ip))
                        break;
        }

        last_context = sample_ip; /* Use sample_ip as an invalid context */

        for (; nr < sz && j <= ts->cnt; nr++, j++) {
                ip = ts->stack[ts->cnt - j].ret_addr;
                context = callchain_context(ip, kernel_start);
                if (context != last_context) {
                        if (nr >= sz - 1)
                                break;
                        chain->ips[nr++] = context;
                        last_context = context;
                }
                chain->ips[nr] = ip;
        }
out:
        if (nr) {
                chain->nr = nr;
        } else {
                chain->ips[0] = sample_context;
                chain->ips[1] = sample_ip;
                chain->nr = 2;
        }
}

void thread_stack__br_sample(struct thread *thread, int cpu,
                             struct branch_stack *dst, unsigned int sz)
{
        struct thread_stack *ts = thread__stack(thread, cpu);
        const size_t bsz = sizeof(struct branch_entry);
        struct branch_stack *src;
        struct branch_entry *be;
        unsigned int nr;

        dst->nr = 0;

        if (!ts)
                return;

        src = ts->br_stack_rb;
        if (!src->nr)
                return;

        dst->nr = min((unsigned int)src->nr, sz);

        be = &dst->entries[0];
        nr = min(ts->br_stack_sz - ts->br_stack_pos, (unsigned int)dst->nr);
        memcpy(be, &src->entries[ts->br_stack_pos], bsz * nr);

        if (src->nr >= ts->br_stack_sz) {
                sz -= nr;
                be = &dst->entries[nr];
                nr = min(ts->br_stack_pos, sz);
                memcpy(be, &src->entries[0], bsz * ts->br_stack_pos);
        }
}

/* Start of user space branch entries */
static bool us_start(struct branch_entry *be, u64 kernel_start, bool *start)
{
        if (!*start)
                *start = be->to && be->to < kernel_start;

        return *start;
}

/*
 * Start of branch entries after the ip fell in between 2 branches, or user
 * space branch entries.
 */
static bool ks_start(struct branch_entry *be, u64 sample_ip, u64 kernel_start,
                     bool *start, struct branch_entry *nb)
{
        if (!*start) {
                *start = (nb && sample_ip >= be->to && sample_ip <= nb->from) ||
                         be->from < kernel_start ||
                         (be->to && be->to < kernel_start);
        }

        return *start;
}

/*
 * Hardware sample records, created some time after the event occurred, need to
 * have subsequent addresses removed from the branch stack.
 */
void thread_stack__br_sample_late(struct thread *thread, int cpu,
                                  struct branch_stack *dst, unsigned int sz,
                                  u64 ip, u64 kernel_start)
{
        struct thread_stack *ts = thread__stack(thread, cpu);
        struct branch_entry *d, *s, *spos, *ssz;
        struct branch_stack *src;
        unsigned int nr = 0;
        bool start = false;

        dst->nr = 0;

        if (!ts)
                return;

        src = ts->br_stack_rb;
        if (!src->nr)
                return;

        spos = &src->entries[ts->br_stack_pos];
        ssz  = &src->entries[ts->br_stack_sz];

        d = &dst->entries[0];
        s = spos;

        if (ip < kernel_start) {
                /*
                 * User space sample: start copying branch entries when the
                 * branch is in user space.
                 */
                for (s = spos; s < ssz && nr < sz; s++) {
                        if (us_start(s, kernel_start, &start)) {
                                *d++ = *s;
                                nr += 1;
                        }
                }

                if (src->nr >= ts->br_stack_sz) {
                        for (s = &src->entries[0]; s < spos && nr < sz; s++) {
                                if (us_start(s, kernel_start, &start)) {
                                        *d++ = *s;
                                        nr += 1;
                                }
                        }
                }
        } else {
                struct branch_entry *nb = NULL;

                /*
                 * Kernel space sample: start copying branch entries when the ip
                 * falls in between 2 branches (or the branch is in user space
                 * because then the start must have been missed).
                 */
                for (s = spos; s < ssz && nr < sz; s++) {
                        if (ks_start(s, ip, kernel_start, &start, nb)) {
                                *d++ = *s;
                                nr += 1;
                        }
                        nb = s;
                }

                if (src->nr >= ts->br_stack_sz) {
                        for (s = &src->entries[0]; s < spos && nr < sz; s++) {
                                if (ks_start(s, ip, kernel_start, &start, nb)) {
                                        *d++ = *s;
                                        nr += 1;
                                }
                                nb = s;
                        }
                }
        }

        dst->nr = nr;
}

struct call_return_processor *
call_return_processor__new(int (*process)(struct call_return *cr, u64 *parent_db_id, void *data),
                           void *data)
{
        struct call_return_processor *crp;

        crp = zalloc(sizeof(struct call_return_processor));
        if (!crp)
                return NULL;
        crp->cpr = call_path_root__new();
        if (!crp->cpr)
                goto out_free;
        crp->process = process;
        crp->data = data;
        return crp;

out_free:
        free(crp);
        return NULL;
}

void call_return_processor__free(struct call_return_processor *crp)
{
        if (crp) {
                call_path_root__free(crp->cpr);
                free(crp);
        }
}

static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,
                                 u64 timestamp, u64 ref, struct call_path *cp,
                                 bool no_call, bool trace_end)
{
        struct thread_stack_entry *tse;
        int err;

        if (!cp)
                return -ENOMEM;

        if (ts->cnt == ts->sz) {
                err = thread_stack__grow(ts);
                if (err)
                        return err;
        }

        tse = &ts->stack[ts->cnt++];
        tse->ret_addr = ret_addr;
        tse->timestamp = timestamp;
        tse->ref = ref;
        tse->branch_count = ts->branch_count;
        tse->insn_count = ts->insn_count;
        tse->cyc_count = ts->cyc_count;
        tse->cp = cp;
        tse->no_call = no_call;
        tse->trace_end = trace_end;
        tse->non_call = false;
        tse->db_id = 0;

        return 0;
}

static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,
                                u64 ret_addr, u64 timestamp, u64 ref,
                                struct symbol *sym)
{
        int err;

        if (!ts->cnt)
                return 1;

        if (ts->cnt == 1) {
                struct thread_stack_entry *tse = &ts->stack[0];

                if (tse->cp->sym == sym)
                        return thread_stack__call_return(thread, ts, --ts->cnt,
                                                         timestamp, ref, false);
        }

        if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&
            !ts->stack[ts->cnt - 1].non_call) {
                return thread_stack__call_return(thread, ts, --ts->cnt,
                                                 timestamp, ref, false);
        } else {
                size_t i = ts->cnt - 1;

                while (i--) {
                        if (ts->stack[i].ret_addr != ret_addr ||
                            ts->stack[i].non_call)
                                continue;
                        i += 1;
                        while (ts->cnt > i) {
                                err = thread_stack__call_return(thread, ts,
                                                                --ts->cnt,
                                                                timestamp, ref,
                                                                true);
                                if (err)
                                        return err;
                        }
                        return thread_stack__call_return(thread, ts, --ts->cnt,
                                                         timestamp, ref, false);
                }
        }

        return 1;
}

static int thread_stack__bottom(struct thread_stack *ts,
                                struct perf_sample *sample,
                                struct addr_location *from_al,
                                struct addr_location *to_al, u64 ref)
{
        struct call_path_root *cpr = ts->crp->cpr;
        struct call_path *cp;
        struct symbol *sym;
        u64 ip;

        if (sample->ip) {
                ip = sample->ip;
                sym = from_al->sym;
        } else if (sample->addr) {
                ip = sample->addr;
                sym = to_al->sym;
        } else {
                return 0;
        }

        cp = call_path__findnew(cpr, &cpr->call_path, sym, ip,
                                ts->kernel_start);

        return thread_stack__push_cp(ts, ip, sample->time, ref, cp,
                                     true, false);
}

static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,
                                struct perf_sample *sample, u64 ref)
{
        u64 tm = sample->time;
        int err;

        /* Return to userspace, so pop all kernel addresses */
        while (thread_stack__in_kernel(ts)) {
                err = thread_stack__call_return(thread, ts, --ts->cnt,
                                                tm, ref, true);
                if (err)
                        return err;
        }

        return 0;
}

static int thread_stack__no_call_return(struct thread *thread,
                                        struct thread_stack *ts,
                                        struct perf_sample *sample,
                                        struct addr_location *from_al,
                                        struct addr_location *to_al, u64 ref)
{
        struct call_path_root *cpr = ts->crp->cpr;
        struct call_path *root = &cpr->call_path;
        struct symbol *fsym = from_al->sym;
        struct symbol *tsym = to_al->sym;
        struct call_path *cp, *parent;
        u64 ks = ts->kernel_start;
        u64 addr = sample->addr;
        u64 tm = sample->time;
        u64 ip = sample->ip;
        int err;

        if (ip >= ks && addr < ks) {
                /* Return to userspace, so pop all kernel addresses */
                err = thread_stack__pop_ks(thread, ts, sample, ref);
                if (err)
                        return err;

                /* If the stack is empty, push the userspace address */
                if (!ts->cnt) {
                        cp = call_path__findnew(cpr, root, tsym, addr, ks);
                        return thread_stack__push_cp(ts, 0, tm, ref, cp, true,
                                                     false);
                }
        } else if (thread_stack__in_kernel(ts) && ip < ks) {
                /* Return to userspace, so pop all kernel addresses */
                err = thread_stack__pop_ks(thread, ts, sample, ref);
                if (err)
                        return err;
        }

        if (ts->cnt)
                parent = ts->stack[ts->cnt - 1].cp;
        else
                parent = root;

        if (parent->sym == from_al->sym) {
                /*
                 * At the bottom of the stack, assume the missing 'call' was
                 * before the trace started. So, pop the current symbol and push
                 * the 'to' symbol.
                 */
                if (ts->cnt == 1) {
                        err = thread_stack__call_return(thread, ts, --ts->cnt,
                                                        tm, ref, false);
                        if (err)
                                return err;
                }

                if (!ts->cnt) {
                        cp = call_path__findnew(cpr, root, tsym, addr, ks);

                        return thread_stack__push_cp(ts, addr, tm, ref, cp,
                                                     true, false);
                }

                /*
                 * Otherwise assume the 'return' is being used as a jump (e.g.
                 * retpoline) and just push the 'to' symbol.
                 */
                cp = call_path__findnew(cpr, parent, tsym, addr, ks);

                err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);
                if (!err)
                        ts->stack[ts->cnt - 1].non_call = true;

                return err;
        }

        /*
         * Assume 'parent' has not yet returned, so push 'to', and then push and
         * pop 'from'.
         */

        cp = call_path__findnew(cpr, parent, tsym, addr, ks);

        err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);
        if (err)
                return err;

        cp = call_path__findnew(cpr, cp, fsym, ip, ks);

        err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);
        if (err)
                return err;

        return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);
}

static int thread_stack__trace_begin(struct thread *thread,
                                     struct thread_stack *ts, u64 timestamp,
                                     u64 ref)
{
        struct thread_stack_entry *tse;
        int err;

        if (!ts->cnt)
                return 0;

        /* Pop trace end */
        tse = &ts->stack[ts->cnt - 1];
        if (tse->trace_end) {
                err = thread_stack__call_return(thread, ts, --ts->cnt,
                                                timestamp, ref, false);
                if (err)
                        return err;
        }

        return 0;
}

static int thread_stack__trace_end(struct thread_stack *ts,
                                   struct perf_sample *sample, u64 ref)
{
        struct call_path_root *cpr = ts->crp->cpr;
        struct call_path *cp;
        u64 ret_addr;

        /* No point having 'trace end' on the bottom of the stack */
        if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))
                return 0;

        cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,
                                ts->kernel_start);

        ret_addr = sample->ip + sample->insn_len;

        return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,
                                     false, true);
}

static bool is_x86_retpoline(const char *name)
{
        const char *p = strstr(name, "__x86_indirect_thunk_");

        return p == name || !strcmp(name, "__indirect_thunk_start");
}

/*
 * x86 retpoline functions pollute the call graph. This function removes them.
 * This does not handle function return thunks, nor is there any improvement
 * for the handling of inline thunks or extern thunks.
 */
static int thread_stack__x86_retpoline(struct thread_stack *ts,
                                       struct perf_sample *sample,
                                       struct addr_location *to_al)
{
        struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];
        struct call_path_root *cpr = ts->crp->cpr;
        struct symbol *sym = tse->cp->sym;
        struct symbol *tsym = to_al->sym;
        struct call_path *cp;

        if (sym && is_x86_retpoline(sym->name)) {
                /*
                 * This is a x86 retpoline fn. It pollutes the call graph by
                 * showing up everywhere there is an indirect branch, but does
                 * not itself mean anything. Here the top-of-stack is removed,
                 * by decrementing the stack count, and then further down, the
                 * resulting top-of-stack is replaced with the actual target.
                 * The result is that the retpoline functions will no longer
                 * appear in the call graph. Note this only affects the call
                 * graph, since all the original branches are left unchanged.
                 */
                ts->cnt -= 1;
                sym = ts->stack[ts->cnt - 2].cp->sym;
                if (sym && sym == tsym && to_al->addr != tsym->start) {
                        /*
                         * Target is back to the middle of the symbol we came
                         * from so assume it is an indirect jmp and forget it
                         * altogether.
                         */
                        ts->cnt -= 1;
                        return 0;
                }
        } else if (sym && sym == tsym) {
                /*
                 * Target is back to the symbol we came from so assume it is an
                 * indirect jmp and forget it altogether.
                 */
                ts->cnt -= 1;
                return 0;
        }

        cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,
                                sample->addr, ts->kernel_start);
        if (!cp)
                return -ENOMEM;

        /* Replace the top-of-stack with the actual target */
        ts->stack[ts->cnt - 1].cp = cp;

        return 0;
}

int thread_stack__process(struct thread *thread, struct comm *comm,
                          struct perf_sample *sample,
                          struct addr_location *from_al,
                          struct addr_location *to_al, u64 ref,
                          struct call_return_processor *crp)
{
        struct thread_stack *ts = thread__stack(thread, sample->cpu);
        enum retpoline_state_t rstate;
        int err = 0;

        if (ts && !ts->crp) {
                /* Supersede thread_stack__event() */
                thread_stack__reset(thread, ts);
                ts = NULL;
        }

        if (!ts) {
                ts = thread_stack__new(thread, sample->cpu, crp, true, 0);
                if (!ts)
                        return -ENOMEM;
                ts->comm = comm;
        }

        rstate = ts->rstate;
        if (rstate == X86_RETPOLINE_DETECTED)
                ts->rstate = X86_RETPOLINE_POSSIBLE;

        /* Flush stack on exec */
        if (ts->comm != comm && thread->pid_ == thread->tid) {
                err = __thread_stack__flush(thread, ts);
                if (err)
                        return err;
                ts->comm = comm;
        }

        /* If the stack is empty, put the current symbol on the stack */
        if (!ts->cnt) {
                err = thread_stack__bottom(ts, sample, from_al, to_al, ref);
                if (err)
                        return err;
        }

        ts->branch_count += 1;
        ts->insn_count += sample->insn_cnt;
        ts->cyc_count += sample->cyc_cnt;
        ts->last_time = sample->time;

        if (sample->flags & PERF_IP_FLAG_CALL) {
                bool trace_end = sample->flags & PERF_IP_FLAG_TRACE_END;
                struct call_path_root *cpr = ts->crp->cpr;
                struct call_path *cp;
                u64 ret_addr;

                if (!sample->ip || !sample->addr)
                        return 0;

                ret_addr = sample->ip + sample->insn_len;
                if (ret_addr == sample->addr)
                        return 0; /* Zero-length calls are excluded */

                cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
                                        to_al->sym, sample->addr,
                                        ts->kernel_start);
                err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,
                                            cp, false, trace_end);

                /*
                 * A call to the same symbol but not the start of the symbol,
                 * may be the start of a x86 retpoline.
                 */
                if (!err && rstate == X86_RETPOLINE_POSSIBLE && to_al->sym &&
                    from_al->sym == to_al->sym &&
                    to_al->addr != to_al->sym->start)
                        ts->rstate = X86_RETPOLINE_DETECTED;

        } else if (sample->flags & PERF_IP_FLAG_RETURN) {
                if (!sample->addr) {
                        u32 return_from_kernel = PERF_IP_FLAG_SYSCALLRET |
                                                 PERF_IP_FLAG_INTERRUPT;

                        if (!(sample->flags & return_from_kernel))
                                return 0;

                        /* Pop kernel stack */
                        return thread_stack__pop_ks(thread, ts, sample, ref);
                }

                if (!sample->ip)
                        return 0;

                /* x86 retpoline 'return' doesn't match the stack */
                if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&
                    ts->stack[ts->cnt - 1].ret_addr != sample->addr)
                        return thread_stack__x86_retpoline(ts, sample, to_al);

                err = thread_stack__pop_cp(thread, ts, sample->addr,
                                           sample->time, ref, from_al->sym);
                if (err) {
                        if (err < 0)
                                return err;
                        err = thread_stack__no_call_return(thread, ts, sample,
                                                           from_al, to_al, ref);
                }
        } else if (sample->flags & PERF_IP_FLAG_TRACE_BEGIN) {
                err = thread_stack__trace_begin(thread, ts, sample->time, ref);
        } else if (sample->flags & PERF_IP_FLAG_TRACE_END) {
                err = thread_stack__trace_end(ts, sample, ref);
        } else if (sample->flags & PERF_IP_FLAG_BRANCH &&
                   from_al->sym != to_al->sym && to_al->sym &&
                   to_al->addr == to_al->sym->start) {
                struct call_path_root *cpr = ts->crp->cpr;
                struct call_path *cp;

                /*
                 * The compiler might optimize a call/ret combination by making
                 * it a jmp. Make that visible by recording on the stack a
                 * branch to the start of a different symbol. Note, that means
                 * when a ret pops the stack, all jmps must be popped off first.
                 */
                cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,
                                        to_al->sym, sample->addr,
                                        ts->kernel_start);
                err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,
                                            false);
                if (!err)
                        ts->stack[ts->cnt - 1].non_call = true;
        }

        return err;
}

size_t thread_stack__depth(struct thread *thread, int cpu)
{
        struct thread_stack *ts = thread__stack(thread, cpu);

        if (!ts)
                return 0;
        return ts->cnt;
}