treewide: remove redundant IS_ERR() before error code check

[linux/fpc-iii.git] / arch / nds32 / kernel / perf_event_cpu.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2008-2017 Andes Technology Corporation
 *
 * Reference ARMv7: Jean Pihet <jpihet@mvista.com>
 * 2010 (c) MontaVista Software, LLC.
 */

#include <linux/perf_event.h>
#include <linux/bitmap.h>
#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/of.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/pm_runtime.h>
#include <linux/ftrace.h>
#include <linux/uaccess.h>
#include <linux/sched/clock.h>
#include <linux/percpu-defs.h>

#include <asm/pmu.h>
#include <asm/irq_regs.h>
#include <asm/nds32.h>
#include <asm/stacktrace.h>
#include <asm/perf_event.h>
#include <nds32_intrinsic.h>

/* Set at runtime when we know what CPU type we are. */
static struct nds32_pmu *cpu_pmu;

static DEFINE_PER_CPU(struct pmu_hw_events, cpu_hw_events);
static void nds32_pmu_start(struct nds32_pmu *cpu_pmu);
static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu);
static struct platform_device_id cpu_pmu_plat_device_ids[] = {
        {.name = "nds32-pfm"},
        {},
};

static int nds32_pmu_map_cache_event(const unsigned int (*cache_map)
                                  [PERF_COUNT_HW_CACHE_MAX]
                                  [PERF_COUNT_HW_CACHE_OP_MAX]
                                  [PERF_COUNT_HW_CACHE_RESULT_MAX], u64 config)
{
        unsigned int cache_type, cache_op, cache_result, ret;

        cache_type = (config >> 0) & 0xff;
        if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
                return -EINVAL;

        cache_op = (config >> 8) & 0xff;
        if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
                return -EINVAL;

        cache_result = (config >> 16) & 0xff;
        if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
                return -EINVAL;

        ret = (int)(*cache_map)[cache_type][cache_op][cache_result];

        if (ret == CACHE_OP_UNSUPPORTED)
                return -ENOENT;

        return ret;
}

static int
nds32_pmu_map_hw_event(const unsigned int (*event_map)[PERF_COUNT_HW_MAX],
                       u64 config)
{
        int mapping;

        if (config >= PERF_COUNT_HW_MAX)
                return -ENOENT;

        mapping = (*event_map)[config];
        return mapping == HW_OP_UNSUPPORTED ? -ENOENT : mapping;
}

static int nds32_pmu_map_raw_event(u32 raw_event_mask, u64 config)
{
        int ev_type = (int)(config & raw_event_mask);
        int idx = config >> 8;

        switch (idx) {
        case 0:
                ev_type = PFM_OFFSET_MAGIC_0 + ev_type;
                if (ev_type >= SPAV3_0_SEL_LAST || ev_type <= SPAV3_0_SEL_BASE)
                        return -ENOENT;
                break;
        case 1:
                ev_type = PFM_OFFSET_MAGIC_1 + ev_type;
                if (ev_type >= SPAV3_1_SEL_LAST || ev_type <= SPAV3_1_SEL_BASE)
                        return -ENOENT;
                break;
        case 2:
                ev_type = PFM_OFFSET_MAGIC_2 + ev_type;
                if (ev_type >= SPAV3_2_SEL_LAST || ev_type <= SPAV3_2_SEL_BASE)
                        return -ENOENT;
                break;
        default:
                return -ENOENT;
        }

        return ev_type;
}

int
nds32_pmu_map_event(struct perf_event *event,
                    const unsigned int (*event_map)[PERF_COUNT_HW_MAX],
                    const unsigned int (*cache_map)
                    [PERF_COUNT_HW_CACHE_MAX]
                    [PERF_COUNT_HW_CACHE_OP_MAX]
                    [PERF_COUNT_HW_CACHE_RESULT_MAX], u32 raw_event_mask)
{
        u64 config = event->attr.config;

        switch (event->attr.type) {
        case PERF_TYPE_HARDWARE:
                return nds32_pmu_map_hw_event(event_map, config);
        case PERF_TYPE_HW_CACHE:
                return nds32_pmu_map_cache_event(cache_map, config);
        case PERF_TYPE_RAW:
                return nds32_pmu_map_raw_event(raw_event_mask, config);
        }

        return -ENOENT;
}

static int nds32_spav3_map_event(struct perf_event *event)
{
        return nds32_pmu_map_event(event, &nds32_pfm_perf_map,
                                &nds32_pfm_perf_cache_map, SOFTWARE_EVENT_MASK);
}

static inline u32 nds32_pfm_getreset_flags(void)
{
        /* Read overflow status */
        u32 val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        u32 old_val = val;

        /* Write overflow bit to clear status, and others keep it 0 */
        u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2];

        __nds32__mtsr(val | ov_flag, NDS32_SR_PFM_CTL);

        return old_val;
}

static inline int nds32_pfm_has_overflowed(u32 pfm)
{
        u32 ov_flag = PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2];

        return pfm & ov_flag;
}

static inline int nds32_pfm_counter_has_overflowed(u32 pfm, int idx)
{
        u32 mask = 0;

        switch (idx) {
        case 0:
                mask = PFM_CTL_OVF[0];
                break;
        case 1:
                mask = PFM_CTL_OVF[1];
                break;
        case 2:
                mask = PFM_CTL_OVF[2];
                break;
        default:
                pr_err("%s index wrong\n", __func__);
                break;
        }
        return pfm & mask;
}

/*
 * Set the next IRQ period, based on the hwc->period_left value.
 * To be called with the event disabled in hw:
 */
int nds32_pmu_event_set_period(struct perf_event *event)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        s64 left = local64_read(&hwc->period_left);
        s64 period = hwc->sample_period;
        int ret = 0;

        /* The period may have been changed by PERF_EVENT_IOC_PERIOD */
        if (unlikely(period != hwc->last_period))
                left = period - (hwc->last_period - left);

        if (unlikely(left <= -period)) {
                left = period;
                local64_set(&hwc->period_left, left);
                hwc->last_period = period;
                ret = 1;
        }

        if (unlikely(left <= 0)) {
                left += period;
                local64_set(&hwc->period_left, left);
                hwc->last_period = period;
                ret = 1;
        }

        if (left > (s64)nds32_pmu->max_period)
                left = nds32_pmu->max_period;

        /*
         * The hw event starts counting from this event offset,
         * mark it to be able to extract future "deltas":
         */
        local64_set(&hwc->prev_count, (u64)(-left));

        nds32_pmu->write_counter(event, (u64)(-left) & nds32_pmu->max_period);

        perf_event_update_userpage(event);

        return ret;
}

static irqreturn_t nds32_pmu_handle_irq(int irq_num, void *dev)
{
        u32 pfm;
        struct perf_sample_data data;
        struct nds32_pmu *cpu_pmu = (struct nds32_pmu *)dev;
        struct pmu_hw_events *cpuc = cpu_pmu->get_hw_events();
        struct pt_regs *regs;
        int idx;
        /*
         * Get and reset the IRQ flags
         */
        pfm = nds32_pfm_getreset_flags();

        /*
         * Did an overflow occur?
         */
        if (!nds32_pfm_has_overflowed(pfm))
                return IRQ_NONE;

        /*
         * Handle the counter(s) overflow(s)
         */
        regs = get_irq_regs();

        nds32_pmu_stop(cpu_pmu);
        for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
                struct perf_event *event = cpuc->events[idx];
                struct hw_perf_event *hwc;

                /* Ignore if we don't have an event. */
                if (!event)
                        continue;

                /*
                 * We have a single interrupt for all counters. Check that
                 * each counter has overflowed before we process it.
                 */
                if (!nds32_pfm_counter_has_overflowed(pfm, idx))
                        continue;

                hwc = &event->hw;
                nds32_pmu_event_update(event);
                perf_sample_data_init(&data, 0, hwc->last_period);
                if (!nds32_pmu_event_set_period(event))
                        continue;

                if (perf_event_overflow(event, &data, regs))
                        cpu_pmu->disable(event);
        }
        nds32_pmu_start(cpu_pmu);
        /*
         * Handle the pending perf events.
         *
         * Note: this call *must* be run with interrupts disabled. For
         * platforms that can have the PMU interrupts raised as an NMI, this
         * will not work.
         */
        irq_work_run();

        return IRQ_HANDLED;
}

static inline int nds32_pfm_counter_valid(struct nds32_pmu *cpu_pmu, int idx)
{
        return ((idx >= 0) && (idx < cpu_pmu->num_events));
}

static inline int nds32_pfm_disable_counter(int idx)
{
        unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        u32 mask = 0;

        mask = PFM_CTL_EN[idx];
        val &= ~mask;
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);
        return idx;
}

/*
 * Add an event filter to a given event.
 */
static int nds32_pmu_set_event_filter(struct hw_perf_event *event,
                                      struct perf_event_attr *attr)
{
        unsigned long config_base = 0;
        int idx = event->idx;
        unsigned long no_kernel_tracing = 0;
        unsigned long no_user_tracing = 0;
        /* If index is -1, do not do anything */
        if (idx == -1)
                return 0;

        no_kernel_tracing = PFM_CTL_KS[idx];
        no_user_tracing = PFM_CTL_KU[idx];
        /*
         * Default: enable both kernel and user mode tracing.
         */
        if (attr->exclude_user)
                config_base |= no_user_tracing;

        if (attr->exclude_kernel)
                config_base |= no_kernel_tracing;

        /*
         * Install the filter into config_base as this is used to
         * construct the event type.
         */
        event->config_base |= config_base;
        return 0;
}

static inline void nds32_pfm_write_evtsel(int idx, u32 evnum)
{
        u32 offset = 0;
        u32 ori_val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        u32 ev_mask = 0;
        u32 no_kernel_mask = 0;
        u32 no_user_mask = 0;
        u32 val;

        offset = PFM_CTL_OFFSEL[idx];
        /* Clear previous mode selection, and write new one */
        no_kernel_mask = PFM_CTL_KS[idx];
        no_user_mask = PFM_CTL_KU[idx];
        ori_val &= ~no_kernel_mask;
        ori_val &= ~no_user_mask;
        if (evnum & no_kernel_mask)
                ori_val |= no_kernel_mask;

        if (evnum & no_user_mask)
                ori_val |= no_user_mask;

        /* Clear previous event selection */
        ev_mask = PFM_CTL_SEL[idx];
        ori_val &= ~ev_mask;
        evnum &= SOFTWARE_EVENT_MASK;

        /* undo the linear mapping */
        evnum = get_converted_evet_hw_num(evnum);
        val = ori_val | (evnum << offset);
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);
}

static inline int nds32_pfm_enable_counter(int idx)
{
        unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        u32 mask = 0;

        mask = PFM_CTL_EN[idx];
        val |= mask;
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);
        return idx;
}

static inline int nds32_pfm_enable_intens(int idx)
{
        unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        u32 mask = 0;

        mask = PFM_CTL_IE[idx];
        val |= mask;
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);
        return idx;
}

static inline int nds32_pfm_disable_intens(int idx)
{
        unsigned int val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        u32 mask = 0;

        mask = PFM_CTL_IE[idx];
        val &= ~mask;
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);
        return idx;
}

static int event_requires_mode_exclusion(struct perf_event_attr *attr)
{
        /* Other modes NDS32 does not support */
        return attr->exclude_user || attr->exclude_kernel;
}

static void nds32_pmu_enable_event(struct perf_event *event)
{
        unsigned long flags;
        unsigned int evnum = 0;
        struct hw_perf_event *hwc = &event->hw;
        struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu);
        struct pmu_hw_events *events = cpu_pmu->get_hw_events();
        int idx = hwc->idx;

        if (!nds32_pfm_counter_valid(cpu_pmu, idx)) {
                pr_err("CPU enabling wrong pfm counter IRQ enable\n");
                return;
        }

        /*
         * Enable counter and interrupt, and set the counter to count
         * the event that we're interested in.
         */
        raw_spin_lock_irqsave(&events->pmu_lock, flags);

        /*
         * Disable counter
         */
        nds32_pfm_disable_counter(idx);

        /*
         * Check whether we need to exclude the counter from certain modes.
         */
        if ((!cpu_pmu->set_event_filter ||
             cpu_pmu->set_event_filter(hwc, &event->attr)) &&
             event_requires_mode_exclusion(&event->attr)) {
                pr_notice
                ("NDS32 performance counters do not support mode exclusion\n");
                hwc->config_base = 0;
        }
        /* Write event */
        evnum = hwc->config_base;
        nds32_pfm_write_evtsel(idx, evnum);

        /*
         * Enable interrupt for this counter
         */
        nds32_pfm_enable_intens(idx);

        /*
         * Enable counter
         */
        nds32_pfm_enable_counter(idx);

        raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}

static void nds32_pmu_disable_event(struct perf_event *event)
{
        unsigned long flags;
        struct hw_perf_event *hwc = &event->hw;
        struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu);
        struct pmu_hw_events *events = cpu_pmu->get_hw_events();
        int idx = hwc->idx;

        if (!nds32_pfm_counter_valid(cpu_pmu, idx)) {
                pr_err("CPU disabling wrong pfm counter IRQ enable %d\n", idx);
                return;
        }

        /*
         * Disable counter and interrupt
         */
        raw_spin_lock_irqsave(&events->pmu_lock, flags);

        /*
         * Disable counter
         */
        nds32_pfm_disable_counter(idx);

        /*
         * Disable interrupt for this counter
         */
        nds32_pfm_disable_intens(idx);

        raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}

static inline u32 nds32_pmu_read_counter(struct perf_event *event)
{
        struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        int idx = hwc->idx;
        u32 count = 0;

        if (!nds32_pfm_counter_valid(cpu_pmu, idx)) {
                pr_err("CPU reading wrong counter %d\n", idx);
        } else {
                switch (idx) {
                case PFMC0:
                        count = __nds32__mfsr(NDS32_SR_PFMC0);
                        break;
                case PFMC1:
                        count = __nds32__mfsr(NDS32_SR_PFMC1);
                        break;
                case PFMC2:
                        count = __nds32__mfsr(NDS32_SR_PFMC2);
                        break;
                default:
                        pr_err
                            ("%s: CPU has no performance counters %d\n",
                             __func__, idx);
                }
        }
        return count;
}

static inline void nds32_pmu_write_counter(struct perf_event *event, u32 value)
{
        struct nds32_pmu *cpu_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        int idx = hwc->idx;

        if (!nds32_pfm_counter_valid(cpu_pmu, idx)) {
                pr_err("CPU writing wrong counter %d\n", idx);
        } else {
                switch (idx) {
                case PFMC0:
                        __nds32__mtsr_isb(value, NDS32_SR_PFMC0);
                        break;
                case PFMC1:
                        __nds32__mtsr_isb(value, NDS32_SR_PFMC1);
                        break;
                case PFMC2:
                        __nds32__mtsr_isb(value, NDS32_SR_PFMC2);
                        break;
                default:
                        pr_err
                            ("%s: CPU has no performance counters %d\n",
                             __func__, idx);
                }
        }
}

static int nds32_pmu_get_event_idx(struct pmu_hw_events *cpuc,
                                   struct perf_event *event)
{
        int idx;
        struct hw_perf_event *hwc = &event->hw;
        /*
         * Current implementation maps cycles, instruction count and cache-miss
         * to specific counter.
         * However, multiple of the 3 counters are able to count these events.
         *
         *
         * SOFTWARE_EVENT_MASK mask for getting event num ,
         * This is defined by Jia-Rung, you can change the polocies.
         * However, do not exceed 8 bits. This is hardware specific.
         * The last number is SPAv3_2_SEL_LAST.
         */
        unsigned long evtype = hwc->config_base & SOFTWARE_EVENT_MASK;

        idx = get_converted_event_idx(evtype);
        /*
         * Try to get the counter for correpsonding event
         */
        if (evtype == SPAV3_0_SEL_TOTAL_CYCLES) {
                if (!test_and_set_bit(idx, cpuc->used_mask))
                        return idx;
                if (!test_and_set_bit(NDS32_IDX_COUNTER0, cpuc->used_mask))
                        return NDS32_IDX_COUNTER0;
                if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask))
                        return NDS32_IDX_COUNTER1;
        } else if (evtype == SPAV3_1_SEL_COMPLETED_INSTRUCTION) {
                if (!test_and_set_bit(idx, cpuc->used_mask))
                        return idx;
                else if (!test_and_set_bit(NDS32_IDX_COUNTER1, cpuc->used_mask))
                        return NDS32_IDX_COUNTER1;
                else if (!test_and_set_bit
                         (NDS32_IDX_CYCLE_COUNTER, cpuc->used_mask))
                        return NDS32_IDX_CYCLE_COUNTER;
        } else {
                if (!test_and_set_bit(idx, cpuc->used_mask))
                        return idx;
        }
        return -EAGAIN;
}

static void nds32_pmu_start(struct nds32_pmu *cpu_pmu)
{
        unsigned long flags;
        unsigned int val;
        struct pmu_hw_events *events = cpu_pmu->get_hw_events();

        raw_spin_lock_irqsave(&events->pmu_lock, flags);

        /* Enable all counters , NDS PFM has 3 counters */
        val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        val |= (PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]);
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);

        raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}

static void nds32_pmu_stop(struct nds32_pmu *cpu_pmu)
{
        unsigned long flags;
        unsigned int val;
        struct pmu_hw_events *events = cpu_pmu->get_hw_events();

        raw_spin_lock_irqsave(&events->pmu_lock, flags);

        /* Disable all counters , NDS PFM has 3 counters */
        val = __nds32__mfsr(NDS32_SR_PFM_CTL);
        val &= ~(PFM_CTL_EN[0] | PFM_CTL_EN[1] | PFM_CTL_EN[2]);
        val &= ~(PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr_isb(val, NDS32_SR_PFM_CTL);

        raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
}

static void nds32_pmu_reset(void *info)
{
        u32 val = 0;

        val |= (PFM_CTL_OVF[0] | PFM_CTL_OVF[1] | PFM_CTL_OVF[2]);
        __nds32__mtsr(val, NDS32_SR_PFM_CTL);
        __nds32__mtsr(0, NDS32_SR_PFM_CTL);
        __nds32__mtsr(0, NDS32_SR_PFMC0);
        __nds32__mtsr(0, NDS32_SR_PFMC1);
        __nds32__mtsr(0, NDS32_SR_PFMC2);
}

static void nds32_pmu_init(struct nds32_pmu *cpu_pmu)
{
        cpu_pmu->handle_irq = nds32_pmu_handle_irq;
        cpu_pmu->enable = nds32_pmu_enable_event;
        cpu_pmu->disable = nds32_pmu_disable_event;
        cpu_pmu->read_counter = nds32_pmu_read_counter;
        cpu_pmu->write_counter = nds32_pmu_write_counter;
        cpu_pmu->get_event_idx = nds32_pmu_get_event_idx;
        cpu_pmu->start = nds32_pmu_start;
        cpu_pmu->stop = nds32_pmu_stop;
        cpu_pmu->reset = nds32_pmu_reset;
        cpu_pmu->max_period = 0xFFFFFFFF;       /* Maximum counts */
};

static u32 nds32_read_num_pfm_events(void)
{
        /* NDS32 SPAv3 PMU support 3 counter */
        return 3;
}

static int device_pmu_init(struct nds32_pmu *cpu_pmu)
{
        nds32_pmu_init(cpu_pmu);
        /*
         * This name should be devive-specific name, whatever you like :)
         * I think "PMU" will be a good generic name.
         */
        cpu_pmu->name = "nds32v3-pmu";
        cpu_pmu->map_event = nds32_spav3_map_event;
        cpu_pmu->num_events = nds32_read_num_pfm_events();
        cpu_pmu->set_event_filter = nds32_pmu_set_event_filter;
        return 0;
}

/*
 * CPU PMU identification and probing.
 */
static int probe_current_pmu(struct nds32_pmu *pmu)
{
        int ret;

        get_cpu();
        ret = -ENODEV;
        /*
         * If ther are various CPU types with its own PMU, initialize with
         *
         * the corresponding one
         */
        device_pmu_init(pmu);
        put_cpu();
        return ret;
}

static void nds32_pmu_enable(struct pmu *pmu)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu);
        struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events();
        int enabled = bitmap_weight(hw_events->used_mask,
                                    nds32_pmu->num_events);

        if (enabled)
                nds32_pmu->start(nds32_pmu);
}

static void nds32_pmu_disable(struct pmu *pmu)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(pmu);

        nds32_pmu->stop(nds32_pmu);
}

static void nds32_pmu_release_hardware(struct nds32_pmu *nds32_pmu)
{
        nds32_pmu->free_irq(nds32_pmu);
        pm_runtime_put_sync(&nds32_pmu->plat_device->dev);
}

static irqreturn_t nds32_pmu_dispatch_irq(int irq, void *dev)
{
        struct nds32_pmu *nds32_pmu = (struct nds32_pmu *)dev;
        int ret;
        u64 start_clock, finish_clock;

        start_clock = local_clock();
        ret = nds32_pmu->handle_irq(irq, dev);
        finish_clock = local_clock();

        perf_sample_event_took(finish_clock - start_clock);
        return ret;
}

static int nds32_pmu_reserve_hardware(struct nds32_pmu *nds32_pmu)
{
        int err;
        struct platform_device *pmu_device = nds32_pmu->plat_device;

        if (!pmu_device)
                return -ENODEV;

        pm_runtime_get_sync(&pmu_device->dev);
        err = nds32_pmu->request_irq(nds32_pmu, nds32_pmu_dispatch_irq);
        if (err) {
                nds32_pmu_release_hardware(nds32_pmu);
                return err;
        }

        return 0;
}

static int
validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events,
               struct perf_event *event)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);

        if (is_software_event(event))
                return 1;

        if (event->pmu != pmu)
                return 0;

        if (event->state < PERF_EVENT_STATE_OFF)
                return 1;

        if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
                return 1;

        return nds32_pmu->get_event_idx(hw_events, event) >= 0;
}

static int validate_group(struct perf_event *event)
{
        struct perf_event *sibling, *leader = event->group_leader;
        struct pmu_hw_events fake_pmu;
        DECLARE_BITMAP(fake_used_mask, MAX_COUNTERS);
        /*
         * Initialize the fake PMU. We only need to populate the
         * used_mask for the purposes of validation.
         */
        memset(fake_used_mask, 0, sizeof(fake_used_mask));

        if (!validate_event(event->pmu, &fake_pmu, leader))
                return -EINVAL;

        for_each_sibling_event(sibling, leader) {
                if (!validate_event(event->pmu, &fake_pmu, sibling))
                        return -EINVAL;
        }

        if (!validate_event(event->pmu, &fake_pmu, event))
                return -EINVAL;

        return 0;
}

static int __hw_perf_event_init(struct perf_event *event)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        int mapping;

        mapping = nds32_pmu->map_event(event);

        if (mapping < 0) {
                pr_debug("event %x:%llx not supported\n", event->attr.type,
                         event->attr.config);
                return mapping;
        }

        /*
         * We don't assign an index until we actually place the event onto
         * hardware. Use -1 to signify that we haven't decided where to put it
         * yet. For SMP systems, each core has it's own PMU so we can't do any
         * clever allocation or constraints checking at this point.
         */
        hwc->idx = -1;
        hwc->config_base = 0;
        hwc->config = 0;
        hwc->event_base = 0;

        /*
         * Check whether we need to exclude the counter from certain modes.
         */
        if ((!nds32_pmu->set_event_filter ||
             nds32_pmu->set_event_filter(hwc, &event->attr)) &&
            event_requires_mode_exclusion(&event->attr)) {
                pr_debug
                        ("NDS performance counters do not support mode exclusion\n");
                return -EOPNOTSUPP;
        }

        /*
         * Store the event encoding into the config_base field.
         */
        hwc->config_base |= (unsigned long)mapping;

        if (!hwc->sample_period) {
                /*
                 * For non-sampling runs, limit the sample_period to half
                 * of the counter width. That way, the new counter value
                 * is far less likely to overtake the previous one unless
                 * you have some serious IRQ latency issues.
                 */
                hwc->sample_period = nds32_pmu->max_period >> 1;
                hwc->last_period = hwc->sample_period;
                local64_set(&hwc->period_left, hwc->sample_period);
        }

        if (event->group_leader != event) {
                if (validate_group(event) != 0)
                        return -EINVAL;
        }

        return 0;
}

static int nds32_pmu_event_init(struct perf_event *event)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        int err = 0;
        atomic_t *active_events = &nds32_pmu->active_events;

        /* does not support taken branch sampling */
        if (has_branch_stack(event))
                return -EOPNOTSUPP;

        if (nds32_pmu->map_event(event) == -ENOENT)
                return -ENOENT;

        if (!atomic_inc_not_zero(active_events)) {
                if (atomic_read(active_events) == 0) {
                        /* Register irq handler */
                        err = nds32_pmu_reserve_hardware(nds32_pmu);
                }

                if (!err)
                        atomic_inc(active_events);
        }

        if (err)
                return err;

        err = __hw_perf_event_init(event);

        return err;
}

static void nds32_start(struct perf_event *event, int flags)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        /*
         * NDS pmu always has to reprogram the period, so ignore
         * PERF_EF_RELOAD, see the comment below.
         */
        if (flags & PERF_EF_RELOAD)
                WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));

        hwc->state = 0;
        /* Set the period for the event. */
        nds32_pmu_event_set_period(event);

        nds32_pmu->enable(event);
}

static int nds32_pmu_add(struct perf_event *event, int flags)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events();
        struct hw_perf_event *hwc = &event->hw;
        int idx;
        int err = 0;

        perf_pmu_disable(event->pmu);

        /* If we don't have a space for the counter then finish early. */
        idx = nds32_pmu->get_event_idx(hw_events, event);
        if (idx < 0) {
                err = idx;
                goto out;
        }

        /*
         * If there is an event in the counter we are going to use then make
         * sure it is disabled.
         */
        event->hw.idx = idx;
        nds32_pmu->disable(event);
        hw_events->events[idx] = event;

        hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
        if (flags & PERF_EF_START)
                nds32_start(event, PERF_EF_RELOAD);

        /* Propagate our changes to the userspace mapping. */
        perf_event_update_userpage(event);

out:
        perf_pmu_enable(event->pmu);
        return err;
}

u64 nds32_pmu_event_update(struct perf_event *event)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        u64 delta, prev_raw_count, new_raw_count;

again:
        prev_raw_count = local64_read(&hwc->prev_count);
        new_raw_count = nds32_pmu->read_counter(event);

        if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
                            new_raw_count) != prev_raw_count) {
                goto again;
        }
        /*
         * Whether overflow or not, "unsigned substraction"
         * will always get their delta
         */
        delta = (new_raw_count - prev_raw_count) & nds32_pmu->max_period;

        local64_add(delta, &event->count);
        local64_sub(delta, &hwc->period_left);

        return new_raw_count;
}

static void nds32_stop(struct perf_event *event, int flags)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct hw_perf_event *hwc = &event->hw;
        /*
         * NDS pmu always has to update the counter, so ignore
         * PERF_EF_UPDATE, see comments in nds32_start().
         */
        if (!(hwc->state & PERF_HES_STOPPED)) {
                nds32_pmu->disable(event);
                nds32_pmu_event_update(event);
                hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
        }
}

static void nds32_pmu_del(struct perf_event *event, int flags)
{
        struct nds32_pmu *nds32_pmu = to_nds32_pmu(event->pmu);
        struct pmu_hw_events *hw_events = nds32_pmu->get_hw_events();
        struct hw_perf_event *hwc = &event->hw;
        int idx = hwc->idx;

        nds32_stop(event, PERF_EF_UPDATE);
        hw_events->events[idx] = NULL;
        clear_bit(idx, hw_events->used_mask);

        perf_event_update_userpage(event);
}

static void nds32_pmu_read(struct perf_event *event)
{
        nds32_pmu_event_update(event);
}

/* Please refer to SPAv3 for more hardware specific details */
PMU_FORMAT_ATTR(event, "config:0-63");

static struct attribute *nds32_arch_formats_attr[] = {
        &format_attr_event.attr,
        NULL,
};

static struct attribute_group nds32_pmu_format_group = {
        .name = "format",
        .attrs = nds32_arch_formats_attr,
};

static ssize_t nds32_pmu_cpumask_show(struct device *dev,
                                      struct device_attribute *attr,
                                      char *buf)
{
        return 0;
}

static DEVICE_ATTR(cpus, 0444, nds32_pmu_cpumask_show, NULL);

static struct attribute *nds32_pmu_common_attrs[] = {
        &dev_attr_cpus.attr,
        NULL,
};

static struct attribute_group nds32_pmu_common_group = {
        .attrs = nds32_pmu_common_attrs,
};

static const struct attribute_group *nds32_pmu_attr_groups[] = {
        &nds32_pmu_format_group,
        &nds32_pmu_common_group,
        NULL,
};

static void nds32_init(struct nds32_pmu *nds32_pmu)
{
        atomic_set(&nds32_pmu->active_events, 0);

        nds32_pmu->pmu = (struct pmu) {
                .pmu_enable = nds32_pmu_enable,
                .pmu_disable = nds32_pmu_disable,
                .attr_groups = nds32_pmu_attr_groups,
                .event_init = nds32_pmu_event_init,
                .add = nds32_pmu_add,
                .del = nds32_pmu_del,
                .start = nds32_start,
                .stop = nds32_stop,
                .read = nds32_pmu_read,
        };
}

int nds32_pmu_register(struct nds32_pmu *nds32_pmu, int type)
{
        nds32_init(nds32_pmu);
        pm_runtime_enable(&nds32_pmu->plat_device->dev);
        pr_info("enabled with %s PMU driver, %d counters available\n",
                nds32_pmu->name, nds32_pmu->num_events);
        return perf_pmu_register(&nds32_pmu->pmu, nds32_pmu->name, type);
}

static struct pmu_hw_events *cpu_pmu_get_cpu_events(void)
{
        return this_cpu_ptr(&cpu_hw_events);
}

static int cpu_pmu_request_irq(struct nds32_pmu *cpu_pmu, irq_handler_t handler)
{
        int err, irq, irqs;
        struct platform_device *pmu_device = cpu_pmu->plat_device;

        if (!pmu_device)
                return -ENODEV;

        irqs = min(pmu_device->num_resources, num_possible_cpus());
        if (irqs < 1) {
                pr_err("no irqs for PMUs defined\n");
                return -ENODEV;
        }

        irq = platform_get_irq(pmu_device, 0);
        err = request_irq(irq, handler, IRQF_NOBALANCING, "nds32-pfm",
                          cpu_pmu);
        if (err) {
                pr_err("unable to request IRQ%d for NDS PMU counters\n",
                       irq);
                return err;
        }
        return 0;
}

static void cpu_pmu_free_irq(struct nds32_pmu *cpu_pmu)
{
        int irq;
        struct platform_device *pmu_device = cpu_pmu->plat_device;

        irq = platform_get_irq(pmu_device, 0);
        if (irq >= 0)
                free_irq(irq, cpu_pmu);
}

static void cpu_pmu_init(struct nds32_pmu *cpu_pmu)
{
        int cpu;
        struct pmu_hw_events *events = &per_cpu(cpu_hw_events, cpu);

        raw_spin_lock_init(&events->pmu_lock);

        cpu_pmu->get_hw_events = cpu_pmu_get_cpu_events;
        cpu_pmu->request_irq = cpu_pmu_request_irq;
        cpu_pmu->free_irq = cpu_pmu_free_irq;

        /* Ensure the PMU has sane values out of reset. */
        if (cpu_pmu->reset)
                on_each_cpu(cpu_pmu->reset, cpu_pmu, 1);
}

static const struct of_device_id cpu_pmu_of_device_ids[] = {
        {.compatible = "andestech,nds32v3-pmu",
         .data = device_pmu_init},
        {},
};

static int cpu_pmu_device_probe(struct platform_device *pdev)
{
        const struct of_device_id *of_id;
        int (*init_fn)(struct nds32_pmu *nds32_pmu);
        struct device_node *node = pdev->dev.of_node;
        struct nds32_pmu *pmu;
        int ret = -ENODEV;

        if (cpu_pmu) {
                pr_notice("[perf] attempt to register multiple PMU devices!\n");
                return -ENOSPC;
        }

        pmu = kzalloc(sizeof(*pmu), GFP_KERNEL);
        if (!pmu)
                return -ENOMEM;

        of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node);
        if (node && of_id) {
                init_fn = of_id->data;
                ret = init_fn(pmu);
        } else {
                ret = probe_current_pmu(pmu);
        }

        if (ret) {
                pr_notice("[perf] failed to probe PMU!\n");
                goto out_free;
        }

        cpu_pmu = pmu;
        cpu_pmu->plat_device = pdev;
        cpu_pmu_init(cpu_pmu);
        ret = nds32_pmu_register(cpu_pmu, PERF_TYPE_RAW);

        if (!ret)
                return 0;

out_free:
        pr_notice("[perf] failed to register PMU devices!\n");
        kfree(pmu);
        return ret;
}

static struct platform_driver cpu_pmu_driver = {
        .driver = {
                   .name = "nds32-pfm",
                   .of_match_table = cpu_pmu_of_device_ids,
                   },
        .probe = cpu_pmu_device_probe,
        .id_table = cpu_pmu_plat_device_ids,
};

static int __init register_pmu_driver(void)
{
        int err = 0;

        err = platform_driver_register(&cpu_pmu_driver);
        if (err)
                pr_notice("[perf] PMU initialization failed\n");
        else
                pr_notice("[perf] PMU initialization done\n");

        return err;
}

device_initcall(register_pmu_driver);

/*
 * References: arch/nds32/kernel/traps.c:__dump()
 * You will need to know the NDS ABI first.
 */
static int unwind_frame_kernel(struct stackframe *frame)
{
        int graph = 0;
#ifdef CONFIG_FRAME_POINTER
        /* 0x3 means misalignment */
        if (!kstack_end((void *)frame->fp) &&
            !((unsigned long)frame->fp & 0x3) &&
            ((unsigned long)frame->fp >= TASK_SIZE)) {
                /*
                 *      The array index is based on the ABI, the below graph
                 *      illustrate the reasons.
                 *      Function call procedure: "smw" and "lmw" will always
                 *      update SP and FP for you automatically.
                 *
                 *      Stack                                 Relative Address
                 *      |  |                                          0
                 *      ----
                 *      |LP| <-- SP(before smw)  <-- FP(after smw)   -1
                 *      ----
                 *      |FP|                                         -2
                 *      ----
                 *      |  | <-- SP(after smw)                       -3
                 */
                frame->lp = ((unsigned long *)frame->fp)[-1];
                frame->fp = ((unsigned long *)frame->fp)[FP_OFFSET];
                /* make sure CONFIG_FUNCTION_GRAPH_TRACER is turned on */
                if (__kernel_text_address(frame->lp))
                        frame->lp = ftrace_graph_ret_addr
                                                (NULL, &graph, frame->lp, NULL);

                return 0;
        } else {
                return -EPERM;
        }
#else
        /*
         * You can refer to arch/nds32/kernel/traps.c:__dump()
         * Treat "sp" as "fp", but the "sp" is one frame ahead of "fp".
         * And, the "sp" is not always correct.
         *
         *   Stack                                 Relative Address
         *   |  |                                          0
         *   ----
         *   |LP| <-- SP(before smw)                      -1
         *   ----
         *   |  | <-- SP(after smw)                       -2
         *   ----
         */
        if (!kstack_end((void *)frame->sp)) {
                frame->lp = ((unsigned long *)frame->sp)[1];
                /* TODO: How to deal with the value in first
                 * "sp" is not correct?
                 */
                if (__kernel_text_address(frame->lp))
                        frame->lp = ftrace_graph_ret_addr
                                                (tsk, &graph, frame->lp, NULL);

                frame->sp = ((unsigned long *)frame->sp) + 1;

                return 0;
        } else {
                return -EPERM;
        }
#endif
}

static void notrace
walk_stackframe(struct stackframe *frame,
                int (*fn_record)(struct stackframe *, void *),
                void *data)
{
        while (1) {
                int ret;

                if (fn_record(frame, data))
                        break;

                ret = unwind_frame_kernel(frame);
                if (ret < 0)
                        break;
        }
}

/*
 * Gets called by walk_stackframe() for every stackframe. This will be called
 * whist unwinding the stackframe and is like a subroutine return so we use
 * the PC.
 */
static int callchain_trace(struct stackframe *fr, void *data)
{
        struct perf_callchain_entry_ctx *entry = data;

        perf_callchain_store(entry, fr->lp);
        return 0;
}

/*
 * Get the return address for a single stackframe and return a pointer to the
 * next frame tail.
 */
static unsigned long
user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp)
{
        struct frame_tail buftail;
        unsigned long lp = 0;
        unsigned long *user_frame_tail =
                (unsigned long *)(fp - (unsigned long)sizeof(buftail));

        /* Check accessibility of one struct frame_tail beyond */
        if (!access_ok(user_frame_tail, sizeof(buftail)))
                return 0;
        if (__copy_from_user_inatomic
                (&buftail, user_frame_tail, sizeof(buftail)))
                return 0;

        /*
         * Refer to unwind_frame_kernel() for more illurstration
         */
        lp = buftail.stack_lp;  /* ((unsigned long *)fp)[-1] */
        fp = buftail.stack_fp;  /* ((unsigned long *)fp)[FP_OFFSET] */
        perf_callchain_store(entry, lp);
        return fp;
}

static unsigned long
user_backtrace_opt_size(struct perf_callchain_entry_ctx *entry,
                        unsigned long fp)
{
        struct frame_tail_opt_size buftail;
        unsigned long lp = 0;

        unsigned long *user_frame_tail =
                (unsigned long *)(fp - (unsigned long)sizeof(buftail));

        /* Check accessibility of one struct frame_tail beyond */
        if (!access_ok(user_frame_tail, sizeof(buftail)))
                return 0;
        if (__copy_from_user_inatomic
                (&buftail, user_frame_tail, sizeof(buftail)))
                return 0;

        /*
         * Refer to unwind_frame_kernel() for more illurstration
         */
        lp = buftail.stack_lp;  /* ((unsigned long *)fp)[-1] */
        fp = buftail.stack_fp;  /* ((unsigned long *)fp)[FP_OFFSET] */

        perf_callchain_store(entry, lp);
        return fp;
}

/*
 * This will be called when the target is in user mode
 * This function will only be called when we use
 * "PERF_SAMPLE_CALLCHAIN" in
 * kernel/events/core.c:perf_prepare_sample()
 *
 * How to trigger perf_callchain_[user/kernel] :
 * $ perf record -e cpu-clock --call-graph fp ./program
 * $ perf report --call-graph
 */
unsigned long leaf_fp;
void
perf_callchain_user(struct perf_callchain_entry_ctx *entry,
                    struct pt_regs *regs)
{
        unsigned long fp = 0;
        unsigned long gp = 0;
        unsigned long lp = 0;
        unsigned long sp = 0;
        unsigned long *user_frame_tail;

        leaf_fp = 0;

        if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
                /* We don't support guest os callchain now */
                return;
        }

        perf_callchain_store(entry, regs->ipc);
        fp = regs->fp;
        gp = regs->gp;
        lp = regs->lp;
        sp = regs->sp;
        if (entry->nr < PERF_MAX_STACK_DEPTH &&
            (unsigned long)fp && !((unsigned long)fp & 0x7) && fp > sp) {
                user_frame_tail =
                        (unsigned long *)(fp - (unsigned long)sizeof(fp));

                if (!access_ok(user_frame_tail, sizeof(fp)))
                        return;

                if (__copy_from_user_inatomic
                        (&leaf_fp, user_frame_tail, sizeof(fp)))
                        return;

                if (leaf_fp == lp) {
                        /*
                         * Maybe this is non leaf function
                         * with optimize for size,
                         * or maybe this is the function
                         * with optimize for size
                         */
                        struct frame_tail buftail;

                        user_frame_tail =
                                (unsigned long *)(fp -
                                        (unsigned long)sizeof(buftail));

                        if (!access_ok(user_frame_tail, sizeof(buftail)))
                                return;

                        if (__copy_from_user_inatomic
                                (&buftail, user_frame_tail, sizeof(buftail)))
                                return;

                        if (buftail.stack_fp == gp) {
                                /* non leaf function with optimize
                                 * for size condition
                                 */
                                struct frame_tail_opt_size buftail_opt_size;

                                user_frame_tail =
                                        (unsigned long *)(fp - (unsigned long)
                                                sizeof(buftail_opt_size));

                                if (!access_ok(user_frame_tail,
                                               sizeof(buftail_opt_size)))
                                        return;

                                if (__copy_from_user_inatomic
                                   (&buftail_opt_size, user_frame_tail,
                                   sizeof(buftail_opt_size)))
                                        return;

                                perf_callchain_store(entry, lp);
                                fp = buftail_opt_size.stack_fp;

                                while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
                                       (unsigned long)fp &&
                                                !((unsigned long)fp & 0x7) &&
                                                fp > sp) {
                                        sp = fp;
                                        fp = user_backtrace_opt_size(entry, fp);
                                }

                        } else {
                                /* this is the function
                                 * without optimize for size
                                 */
                                fp = buftail.stack_fp;
                                perf_callchain_store(entry, lp);
                                while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
                                       (unsigned long)fp &&
                                                !((unsigned long)fp & 0x7) &&
                                                fp > sp) {
                                        sp = fp;
                                        fp = user_backtrace(entry, fp);
                                }
                        }
                } else {
                        /* this is leaf function */
                        fp = leaf_fp;
                        perf_callchain_store(entry, lp);

                        /* previous function callcahin  */
                        while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
                               (unsigned long)fp &&
                                   !((unsigned long)fp & 0x7) && fp > sp) {
                                sp = fp;
                                fp = user_backtrace(entry, fp);
                        }
                }
                return;
        }
}

/* This will be called when the target is in kernel mode */
void
perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
                      struct pt_regs *regs)
{
        struct stackframe fr;

        if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
                /* We don't support guest os callchain now */
                return;
        }
        fr.fp = regs->fp;
        fr.lp = regs->lp;
        fr.sp = regs->sp;
        walk_stackframe(&fr, callchain_trace, entry);
}

unsigned long perf_instruction_pointer(struct pt_regs *regs)
{
        /* However, NDS32 does not support virtualization */
        if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
                return perf_guest_cbs->get_guest_ip();

        return instruction_pointer(regs);
}

unsigned long perf_misc_flags(struct pt_regs *regs)
{
        int misc = 0;

        /* However, NDS32 does not support virtualization */
        if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
                if (perf_guest_cbs->is_user_mode())
                        misc |= PERF_RECORD_MISC_GUEST_USER;
                else
                        misc |= PERF_RECORD_MISC_GUEST_KERNEL;
        } else {
                if (user_mode(regs))
                        misc |= PERF_RECORD_MISC_USER;
                else
                        misc |= PERF_RECORD_MISC_KERNEL;
        }

        return misc;
}