ARM: 7409/1: Do not call flush_cache_user_range with mmap_sem held

[linux/fpc-iii.git] / arch / mips / kernel / perf_event.c

/*
 * Linux performance counter support for MIPS.
 *
 * Copyright (C) 2010 MIPS Technologies, Inc.
 * Author: Deng-Cheng Zhu
 *
 * This code is based on the implementation for ARM, which is in turn
 * based on the sparc64 perf event code and the x86 code. Performance
 * counter access is based on the MIPS Oprofile code. And the callchain
 * support references the code of MIPS stacktrace.c.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/cpumask.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
#include <linux/kernel.h>
#include <linux/perf_event.h>
#include <linux/uaccess.h>

#include <asm/irq.h>
#include <asm/irq_regs.h>
#include <asm/stacktrace.h>
#include <asm/time.h> /* For perf_irq */

/* These are for 32bit counters. For 64bit ones, define them accordingly. */
#define MAX_PERIOD      ((1ULL << 32) - 1)
#define VALID_COUNT     0x7fffffff
#define TOTAL_BITS      32
#define HIGHEST_BIT     31

#define MIPS_MAX_HWEVENTS 4

struct cpu_hw_events {
        /* Array of events on this cpu. */
        struct perf_event       *events[MIPS_MAX_HWEVENTS];

        /*
         * Set the bit (indexed by the counter number) when the counter
         * is used for an event.
         */
        unsigned long           used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];

        /*
         * The borrowed MSB for the performance counter. A MIPS performance
         * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit
         * counters) as a factor of determining whether a counter overflow
         * should be signaled. So here we use a separate MSB for each
         * counter to make things easy.
         */
        unsigned long           msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];

        /*
         * Software copy of the control register for each performance counter.
         * MIPS CPUs vary in performance counters. They use this differently,
         * and even may not use it.
         */
        unsigned int            saved_ctrl[MIPS_MAX_HWEVENTS];
};
DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
        .saved_ctrl = {0},
};

/* The description of MIPS performance events. */
struct mips_perf_event {
        unsigned int event_id;
        /*
         * MIPS performance counters are indexed starting from 0.
         * CNTR_EVEN indicates the indexes of the counters to be used are
         * even numbers.
         */
        unsigned int cntr_mask;
        #define CNTR_EVEN       0x55555555
        #define CNTR_ODD        0xaaaaaaaa
#ifdef CONFIG_MIPS_MT_SMP
        enum {
                T  = 0,
                V  = 1,
                P  = 2,
        } range;
#else
        #define T
        #define V
        #define P
#endif
};

static struct mips_perf_event raw_event;
static DEFINE_MUTEX(raw_event_mutex);

#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff
#define C(x) PERF_COUNT_HW_CACHE_##x

struct mips_pmu {
        const char      *name;
        int             irq;
        irqreturn_t     (*handle_irq)(int irq, void *dev);
        int             (*handle_shared_irq)(void);
        void            (*start)(void);
        void            (*stop)(void);
        int             (*alloc_counter)(struct cpu_hw_events *cpuc,
                                        struct hw_perf_event *hwc);
        u64             (*read_counter)(unsigned int idx);
        void            (*write_counter)(unsigned int idx, u64 val);
        void            (*enable_event)(struct hw_perf_event *evt, int idx);
        void            (*disable_event)(int idx);
        const struct mips_perf_event *(*map_raw_event)(u64 config);
        const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
        const struct mips_perf_event (*cache_event_map)
                                [PERF_COUNT_HW_CACHE_MAX]
                                [PERF_COUNT_HW_CACHE_OP_MAX]
                                [PERF_COUNT_HW_CACHE_RESULT_MAX];
        unsigned int    num_counters;
};

static const struct mips_pmu *mipspmu;

static int
mipspmu_event_set_period(struct perf_event *event,
                        struct hw_perf_event *hwc,
                        int idx)
{
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        s64 left = local64_read(&hwc->period_left);
        s64 period = hwc->sample_period;
        int ret = 0;
        u64 uleft;
        unsigned long flags;

        if (unlikely(left <= -period)) {
                left = period;
                local64_set(&hwc->period_left, left);
                hwc->last_period = period;
                ret = 1;
        }

        if (unlikely(left <= 0)) {
                left += period;
                local64_set(&hwc->period_left, left);
                hwc->last_period = period;
                ret = 1;
        }

        if (left > (s64)MAX_PERIOD)
                left = MAX_PERIOD;

        local64_set(&hwc->prev_count, (u64)-left);

        local_irq_save(flags);
        uleft = (u64)(-left) & MAX_PERIOD;
        uleft > VALID_COUNT ?
                set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs);
        mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT);
        local_irq_restore(flags);

        perf_event_update_userpage(event);

        return ret;
}

static void mipspmu_event_update(struct perf_event *event,
                        struct hw_perf_event *hwc,
                        int idx)
{
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        unsigned long flags;
        int shift = 64 - TOTAL_BITS;
        s64 prev_raw_count, new_raw_count;
        u64 delta;

again:
        prev_raw_count = local64_read(&hwc->prev_count);
        local_irq_save(flags);
        /* Make the counter value be a "real" one. */
        new_raw_count = mipspmu->read_counter(idx);
        if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) {
                new_raw_count &= VALID_COUNT;
                clear_bit(idx, cpuc->msbs);
        } else
                new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT);
        local_irq_restore(flags);

        if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
                                new_raw_count) != prev_raw_count)
                goto again;

        delta = (new_raw_count << shift) - (prev_raw_count << shift);
        delta >>= shift;

        local64_add(delta, &event->count);
        local64_sub(delta, &hwc->period_left);

        return;
}

static void mipspmu_start(struct perf_event *event, int flags)
{
        struct hw_perf_event *hwc = &event->hw;

        if (!mipspmu)
                return;

        if (flags & PERF_EF_RELOAD)
                WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));

        hwc->state = 0;

        /* Set the period for the event. */
        mipspmu_event_set_period(event, hwc, hwc->idx);

        /* Enable the event. */
        mipspmu->enable_event(hwc, hwc->idx);
}

static void mipspmu_stop(struct perf_event *event, int flags)
{
        struct hw_perf_event *hwc = &event->hw;

        if (!mipspmu)
                return;

        if (!(hwc->state & PERF_HES_STOPPED)) {
                /* We are working on a local event. */
                mipspmu->disable_event(hwc->idx);
                barrier();
                mipspmu_event_update(event, hwc, hwc->idx);
                hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
        }
}

static int mipspmu_add(struct perf_event *event, int flags)
{
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
        int idx;
        int err = 0;

        perf_pmu_disable(event->pmu);

        /* To look for a free counter for this event. */
        idx = mipspmu->alloc_counter(cpuc, hwc);
        if (idx < 0) {
                err = idx;
                goto out;
        }

        /*
         * If there is an event in the counter we are going to use then
         * make sure it is disabled.
         */
        event->hw.idx = idx;
        mipspmu->disable_event(idx);
        cpuc->events[idx] = event;

        hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
        if (flags & PERF_EF_START)
                mipspmu_start(event, PERF_EF_RELOAD);

        /* Propagate our changes to the userspace mapping. */
        perf_event_update_userpage(event);

out:
        perf_pmu_enable(event->pmu);
        return err;
}

static void mipspmu_del(struct perf_event *event, int flags)
{
        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
        struct hw_perf_event *hwc = &event->hw;
        int idx = hwc->idx;

        WARN_ON(idx < 0 || idx >= mipspmu->num_counters);

        mipspmu_stop(event, PERF_EF_UPDATE);
        cpuc->events[idx] = NULL;
        clear_bit(idx, cpuc->used_mask);

        perf_event_update_userpage(event);
}

static void mipspmu_read(struct perf_event *event)
{
        struct hw_perf_event *hwc = &event->hw;

        /* Don't read disabled counters! */
        if (hwc->idx < 0)
                return;

        mipspmu_event_update(event, hwc, hwc->idx);
}

static void mipspmu_enable(struct pmu *pmu)
{
        if (mipspmu)
                mipspmu->start();
}

static void mipspmu_disable(struct pmu *pmu)
{
        if (mipspmu)
                mipspmu->stop();
}

static atomic_t active_events = ATOMIC_INIT(0);
static DEFINE_MUTEX(pmu_reserve_mutex);
static int (*save_perf_irq)(void);

static int mipspmu_get_irq(void)
{
        int err;

        if (mipspmu->irq >= 0) {
                /* Request my own irq handler. */
                err = request_irq(mipspmu->irq, mipspmu->handle_irq,
                        IRQF_DISABLED | IRQF_NOBALANCING,
                        "mips_perf_pmu", NULL);
                if (err) {
                        pr_warning("Unable to request IRQ%d for MIPS "
                           "performance counters!\n", mipspmu->irq);
                }
        } else if (cp0_perfcount_irq < 0) {
                /*
                 * We are sharing the irq number with the timer interrupt.
                 */
                save_perf_irq = perf_irq;
                perf_irq = mipspmu->handle_shared_irq;
                err = 0;
        } else {
                pr_warning("The platform hasn't properly defined its "
                        "interrupt controller.\n");
                err = -ENOENT;
        }

        return err;
}

static void mipspmu_free_irq(void)
{
        if (mipspmu->irq >= 0)
                free_irq(mipspmu->irq, NULL);
        else if (cp0_perfcount_irq < 0)
                perf_irq = save_perf_irq;
}

/*
 * mipsxx/rm9000/loongson2 have different performance counters, they have
 * specific low-level init routines.
 */
static void reset_counters(void *arg);
static int __hw_perf_event_init(struct perf_event *event);

static void hw_perf_event_destroy(struct perf_event *event)
{
        if (atomic_dec_and_mutex_lock(&active_events,
                                &pmu_reserve_mutex)) {
                /*
                 * We must not call the destroy function with interrupts
                 * disabled.
                 */
                on_each_cpu(reset_counters,
                        (void *)(long)mipspmu->num_counters, 1);
                mipspmu_free_irq();
                mutex_unlock(&pmu_reserve_mutex);
        }
}

static int mipspmu_event_init(struct perf_event *event)
{
        int err = 0;

        switch (event->attr.type) {
        case PERF_TYPE_RAW:
        case PERF_TYPE_HARDWARE:
        case PERF_TYPE_HW_CACHE:
                break;

        default:
                return -ENOENT;
        }

        if (!mipspmu || event->cpu >= nr_cpumask_bits ||
                (event->cpu >= 0 && !cpu_online(event->cpu)))
                return -ENODEV;

        if (!atomic_inc_not_zero(&active_events)) {
                if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
                        atomic_dec(&active_events);
                        return -ENOSPC;
                }

                mutex_lock(&pmu_reserve_mutex);
                if (atomic_read(&active_events) == 0)
                        err = mipspmu_get_irq();

                if (!err)
                        atomic_inc(&active_events);
                mutex_unlock(&pmu_reserve_mutex);
        }

        if (err)
                return err;

        err = __hw_perf_event_init(event);
        if (err)
                hw_perf_event_destroy(event);

        return err;
}

static struct pmu pmu = {
        .pmu_enable     = mipspmu_enable,
        .pmu_disable    = mipspmu_disable,
        .event_init     = mipspmu_event_init,
        .add            = mipspmu_add,
        .del            = mipspmu_del,
        .start          = mipspmu_start,
        .stop           = mipspmu_stop,
        .read           = mipspmu_read,
};

static inline unsigned int
mipspmu_perf_event_encode(const struct mips_perf_event *pev)
{
/*
 * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
 * event_id.
 */
#ifdef CONFIG_MIPS_MT_SMP
        return ((unsigned int)pev->range << 24) |
                (pev->cntr_mask & 0xffff00) |
                (pev->event_id & 0xff);
#else
        return (pev->cntr_mask & 0xffff00) |
                (pev->event_id & 0xff);
#endif
}

static const struct mips_perf_event *
mipspmu_map_general_event(int idx)
{
        const struct mips_perf_event *pev;

        pev = ((*mipspmu->general_event_map)[idx].event_id ==
                UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) :
                &(*mipspmu->general_event_map)[idx]);

        return pev;
}

static const struct mips_perf_event *
mipspmu_map_cache_event(u64 config)
{
        unsigned int cache_type, cache_op, cache_result;
        const struct mips_perf_event *pev;

        cache_type = (config >> 0) & 0xff;
        if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
                return ERR_PTR(-EINVAL);

        cache_op = (config >> 8) & 0xff;
        if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
                return ERR_PTR(-EINVAL);

        cache_result = (config >> 16) & 0xff;
        if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
                return ERR_PTR(-EINVAL);

        pev = &((*mipspmu->cache_event_map)
                                        [cache_type]
                                        [cache_op]
                                        [cache_result]);

        if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID)
                return ERR_PTR(-EOPNOTSUPP);

        return pev;

}

static int validate_event(struct cpu_hw_events *cpuc,
               struct perf_event *event)
{
        struct hw_perf_event fake_hwc = event->hw;

        /* Allow mixed event group. So return 1 to pass validation. */
        if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF)
                return 1;

        return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0;
}

static int validate_group(struct perf_event *event)
{
        struct perf_event *sibling, *leader = event->group_leader;
        struct cpu_hw_events fake_cpuc;

        memset(&fake_cpuc, 0, sizeof(fake_cpuc));

        if (!validate_event(&fake_cpuc, leader))
                return -ENOSPC;

        list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
                if (!validate_event(&fake_cpuc, sibling))
                        return -ENOSPC;
        }

        if (!validate_event(&fake_cpuc, event))
                return -ENOSPC;

        return 0;
}

/* This is needed by specific irq handlers in perf_event_*.c */
static void
handle_associated_event(struct cpu_hw_events *cpuc,
        int idx, struct perf_sample_data *data, struct pt_regs *regs)
{
        struct perf_event *event = cpuc->events[idx];
        struct hw_perf_event *hwc = &event->hw;

        mipspmu_event_update(event, hwc, idx);
        data->period = event->hw.last_period;
        if (!mipspmu_event_set_period(event, hwc, idx))
                return;

        if (perf_event_overflow(event, 0, data, regs))
                mipspmu->disable_event(idx);
}

#include "perf_event_mipsxx.c"

/* Callchain handling code. */

/*
 * Leave userspace callchain empty for now. When we find a way to trace
 * the user stack callchains, we add here.
 */
void perf_callchain_user(struct perf_callchain_entry *entry,
                    struct pt_regs *regs)
{
}

static void save_raw_perf_callchain(struct perf_callchain_entry *entry,
        unsigned long reg29)
{
        unsigned long *sp = (unsigned long *)reg29;
        unsigned long addr;

        while (!kstack_end(sp)) {
                addr = *sp++;
                if (__kernel_text_address(addr)) {
                        perf_callchain_store(entry, addr);
                        if (entry->nr >= PERF_MAX_STACK_DEPTH)
                                break;
                }
        }
}

void perf_callchain_kernel(struct perf_callchain_entry *entry,
                      struct pt_regs *regs)
{
        unsigned long sp = regs->regs[29];
#ifdef CONFIG_KALLSYMS
        unsigned long ra = regs->regs[31];
        unsigned long pc = regs->cp0_epc;

        if (raw_show_trace || !__kernel_text_address(pc)) {
                unsigned long stack_page =
                        (unsigned long)task_stack_page(current);
                if (stack_page && sp >= stack_page &&
                    sp <= stack_page + THREAD_SIZE - 32)
                        save_raw_perf_callchain(entry, sp);
                return;
        }
        do {
                perf_callchain_store(entry, pc);
                if (entry->nr >= PERF_MAX_STACK_DEPTH)
                        break;
                pc = unwind_stack(current, &sp, pc, &ra);
        } while (pc);
#else
        save_raw_perf_callchain(entry, sp);
#endif
}