dmake: do not set MAKEFLAGS=k

[unleashed/tickless.git] / usr / src / uts / intel / pcbe / p123_pcbe.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * This file contains preset event names from the Performance Application
 * Programming Interface v3.5 which included the following notice:
 *
 *                             Copyright (c) 2005,6
 *                           Innovative Computing Labs
 *                         Computer Science Department,
 *                            University of Tennessee,
 *                                 Knoxville, TN.
 *                              All Rights Reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of the University of Tennessee nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * This open source software license conforms to the BSD License template.
 */

/*
 * Performance Counter Back-End for Pentiums I, II, and III.
 */

#include <sys/cpuvar.h>
#include <sys/param.h>
#include <sys/cpc_impl.h>
#include <sys/cpc_pcbe.h>
#include <sys/modctl.h>
#include <sys/inttypes.h>
#include <sys/systm.h>
#include <sys/cmn_err.h>
#include <sys/x86_archext.h>
#include <sys/sdt.h>
#include <sys/archsystm.h>
#include <sys/privregs.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

static int64_t diff3931(uint64_t sample, uint64_t old);
static uint64_t trunc3931(uint64_t value);

static int ptm_pcbe_init(void);
static uint_t ptm_pcbe_ncounters(void);
static const char *ptm_pcbe_impl_name(void);
static const char *ptm_pcbe_cpuref(void);
static char *ptm_pcbe_list_events(uint_t picnum);
static char *ptm_pcbe_list_attrs(void);
static uint64_t ptm_pcbe_event_coverage(char *event);
static int ptm_pcbe_pic_index(char *picname);
static uint64_t ptm_pcbe_overflow_bitmap(void);
static int ptm_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token);
static void ptm_pcbe_program(void *token);
static void ptm_pcbe_allstop(void);
static void ptm_pcbe_sample(void *token);
static void ptm_pcbe_free(void *config);

pcbe_ops_t ptm_pcbe_ops = {
        PCBE_VER_1,
        0,
        ptm_pcbe_ncounters,
        ptm_pcbe_impl_name,
        ptm_pcbe_cpuref,
        ptm_pcbe_list_events,
        ptm_pcbe_list_attrs,
        ptm_pcbe_event_coverage,
        ptm_pcbe_overflow_bitmap,
        ptm_pcbe_configure,
        ptm_pcbe_program,
        ptm_pcbe_allstop,
        ptm_pcbe_sample,
        ptm_pcbe_free
};

typedef enum _ptm_ver {
        PTM_VER_P5,
        PTM_VER_P6
} ptm_ver_t;

static ptm_ver_t ptm_ver;
static const char *ptm_impl_name;
static const char *ptm_cpuref;
static char *pic_events[2] = { NULL, NULL };

/*
 * Indicates whether the "rdpmc" instruction is available on this processor.
 */
static int ptm_rdpmc_avail = 0;

#define ALL_STOPPED     0ULL

typedef struct _ptm_pcbe_config {
        uint8_t         ptm_picno;      /* 0 for pic0 or 1 for pic1 */
        uint32_t        ptm_ctl;    /* P6: PerfEventSelect; P5: cesr, shifted */
        uint64_t        ptm_rawpic;
} ptm_pcbe_config_t;

struct nametable {
        uint8_t         bits;
        const char      *name;
};

typedef struct _ptm_generic_events {
        char *name;
        char *event;
        uint8_t umask;
} ptm_generic_event_t;

#define NT_END 0xFF
#define CPC_GEN_END { NULL, NULL }

/*
 * Basic Pentium events
 */
#define P5_EVENTS                               \
        {0x0,   "data_read"},                   \
        {0x1,   "data_write"},                  \
        {0x2,   "data_tlb_miss"},               \
        {0x3,   "data_read_miss"},              \
        {0x4,   "data_write_miss"},             \
        {0x5,   "write_hit_to_M_or_E"},         \
        {0x6,   "dcache_lines_wrback"},         \
        {0x7,   "external_snoops"},             \
        {0x8,   "external_dcache_snoop_hits"},  \
        {0x9,   "memory_access_in_both_pipes"}, \
        {0xa,   "bank_conflicts"},              \
        {0xb,   "misaligned_ref"},              \
        {0xc,   "code_read"},                   \
        {0xd,   "code_tlb_miss"},               \
        {0xe,   "code_cache_miss"},             \
        {0xf,   "any_segreg_loaded"},           \
        {0x12,  "branches"},                    \
        {0x13,  "btb_hits"},                    \
        {0x14,  "taken_or_btb_hit"},            \
        {0x15,  "pipeline_flushes"},            \
        {0x16,  "instr_exec"},                  \
        {0x17,  "instr_exec_V_pipe"},           \
        {0x18,  "clks_bus_cycle"},              \
        {0x19,  "clks_full_wbufs"},             \
        {0x1a,  "pipe_stall_read"},             \
        {0x1b,  "stall_on_write_ME"},           \
        {0x1c,  "locked_bus_cycle"},            \
        {0x1d,  "io_rw_cycles"},                \
        {0x1e,  "reads_noncache_mem"},          \
        {0x1f,  "pipeline_agi_stalls"},         \
        {0x22,  "flops"},                       \
        {0x23,  "bp_match_dr0"},                \
        {0x24,  "bp_match_dr1"},                \
        {0x25,  "bp_match_dr2"},                \
        {0x26,  "bp_match_dr3"},                \
        {0x27,  "hw_intrs"},                    \
        {0x28,  "data_rw"},                     \
        {0x29,  "data_rw_miss"}

static const struct nametable P5mmx_names0[] = {
        P5_EVENTS,
        {0x2a,  "bus_ownership_latency"},
        {0x2b,  "mmx_instr_upipe"},
        {0x2c,  "cache_M_line_sharing"},
        {0x2d,  "emms_instr"},
        {0x2e,  "bus_util_processor"},
        {0x2f,  "sat_mmx_instr"},
        {0x30,  "clks_not_HLT"},
        {0x31,  "mmx_data_read"},
        {0x32,  "clks_fp_stall"},
        {0x33,  "d1_starv_fifo_0"},
        {0x34,  "mmx_data_write"},
        {0x35,  "pipe_flush_wbp"},
        {0x36,  "mmx_misalign_data_refs"},
        {0x37,  "rets_pred_incorrect"},
        {0x38,  "mmx_multiply_unit_interlock"},
        {0x39,  "rets"},
        {0x3a,  "btb_false_entries"},
        {0x3b,  "clocks_stall_full_wb"},
        {NT_END, ""}
};

static const struct nametable P5mmx_names1[] = {
        P5_EVENTS,
        {0x2a,  "bus_ownership_transfers"},
        {0x2b,  "mmx_instr_vpipe"},
        {0x2c,  "cache_lint_sharing"},
        {0x2d,  "mmx_fp_transitions"},
        {0x2e,  "writes_noncache_mem"},
        {0x2f,  "sats_performed"},
        {0x30,  "clks_dcache_tlb_miss"},
        {0x31,  "mmx_data_read_miss"},
        {0x32,  "taken_br"},
        {0x33,  "d1_starv_fifo_1"},
        {0x34,  "mmx_data_write_miss"},
        {0x35,  "pipe_flush_wbp_wb"},
        {0x36,  "mmx_pipe_stall_data_read"},
        {0x37,  "rets_pred"},
        {0x38,  "movd_movq_stall"},
        {0x39,  "rsb_overflow"},
        {0x3a,  "btb_mispred_nt"},
        {0x3b,  "mmx_stall_write_ME"},
        {NT_END, ""}
};

static const struct nametable *P5mmx_names[2] = {
        P5mmx_names0,
        P5mmx_names1
};

/*
 * Pentium Pro and Pentium II events
 */
static const struct nametable _P6_names[] = {
        /*
         * Data cache unit
         */
        {0x43,  "data_mem_refs"},
        {0x45,  "dcu_lines_in"},
        {0x46,  "dcu_m_lines_in"},
        {0x47,  "dcu_m_lines_out"},
        {0x48,  "dcu_miss_outstanding"},

        /*
         * Instruction fetch unit
         */
        {0x80,  "ifu_ifetch"},
        {0x81,  "ifu_ifetch_miss"},
        {0x85,  "itlb_miss"},
        {0x86,  "ifu_mem_stall"},
        {0x87,  "ild_stall"},

        /*
         * L2 cache
         */
        {0x28,  "l2_ifetch"},
        {0x29,  "l2_ld"},
        {0x2a,  "l2_st"},
        {0x24,  "l2_lines_in"},
        {0x26,  "l2_lines_out"},
        {0x25,  "l2_m_lines_inm"},
        {0x27,  "l2_m_lines_outm"},
        {0x2e,  "l2_rqsts"},
        {0x21,  "l2_ads"},
        {0x22,  "l2_dbus_busy"},
        {0x23,  "l2_dbus_busy_rd"},

        /*
         * External bus logic
         */
        {0x62,  "bus_drdy_clocks"},
        {0x63,  "bus_lock_clocks"},
        {0x60,  "bus_req_outstanding"},
        {0x65,  "bus_tran_brd"},
        {0x66,  "bus_tran_rfo"},
        {0x67,  "bus_trans_wb"},
        {0x68,  "bus_tran_ifetch"},
        {0x69,  "bus_tran_inval"},
        {0x6a,  "bus_tran_pwr"},
        {0x6b,  "bus_trans_p"},
        {0x6c,  "bus_trans_io"},
        {0x6d,  "bus_tran_def"},
        {0x6e,  "bus_tran_burst"},
        {0x70,  "bus_tran_any"},
        {0x6f,  "bus_tran_mem"},
        {0x64,  "bus_data_rcv"},
        {0x61,  "bus_bnr_drv"},
        {0x7a,  "bus_hit_drv"},
        {0x7b,  "bus_hitm_drv"},
        {0x7e,  "bus_snoop_stall"},

        /*
         * Floating point unit
         */
        {0xc1,  "flops"},               /* 0 only */
        {0x10,  "fp_comp_ops_exe"},     /* 0 only */
        {0x11,  "fp_assist"},           /* 1 only */
        {0x12,  "mul"},                 /* 1 only */
        {0x13,  "div"},                 /* 1 only */
        {0x14,  "cycles_div_busy"},     /* 0 only */

        /*
         * Memory ordering
         */
        {0x3,   "ld_blocks"},
        {0x4,   "sb_drains"},
        {0x5,   "misalign_mem_ref"},

        /*
         * Instruction decoding and retirement
         */
        {0xc0,  "inst_retired"},
        {0xc2,  "uops_retired"},
        {0xd0,  "inst_decoder"},

        /*
         * Interrupts
         */
        {0xc8,  "hw_int_rx"},
        {0xc6,  "cycles_int_masked"},
        {0xc7,  "cycles_int_pending_and_masked"},

        /*
         * Branches
         */
        {0xc4,  "br_inst_retired"},
        {0xc5,  "br_miss_pred_retired"},
        {0xc9,  "br_taken_retired"},
        {0xca,  "br_miss_pred_taken_ret"},
        {0xe0,  "br_inst_decoded"},
        {0xe2,  "btb_misses"},
        {0xe4,  "br_bogus"},
        {0xe6,  "baclears"},

        /*
         * Stalls
         */
        {0xa2,  "resource_stalls"},
        {0xd2,  "partial_rat_stalls"},

        /*
         * Segment register loads
         */
        {0x6,   "segment_reg_loads"},

        /*
         * Clocks
         */
        {0x79,  "cpu_clk_unhalted"},

        /*
         * MMX
         */
        {0xb0,  "mmx_instr_exec"},
        {0xb1,  "mmx_sat_instr_exec"},
        {0xb2,  "mmx_uops_exec"},
        {0xb3,  "mmx_instr_type_exec"},
        {0xcc,  "fp_mmx_trans"},
        {0xcd,  "mmx_assists"},
        {0xce,  "mmx_instr_ret"},
        {0xd4,  "seg_rename_stalls"},
        {0xd5,  "seg_reg_renames"},
        {0xd6,  "ret_seg_renames"},

        {NT_END, ""}
};

static const struct nametable *P6_names[2] = {
        _P6_names,
        _P6_names
};

#define P5_GENERIC_EVENTS                                       \
        { "PAPI_tot_ins",       "instr_exec",    0x0 },         \
        { "PAPI_tlb_dm",        "data_tlb_miss", 0x0 },         \
        { "PAPI_tlb_im",        "code_tlb_miss", 0x0 },         \
        { "PAPI_fp_ops",        "flops" }

static const ptm_generic_event_t P5mmx_generic_names0[] = {
        P5_GENERIC_EVENTS,
        { "PAPI_tot_cyc",       "clks_not_HLT", 0x0 },
        CPC_GEN_END
};

static const ptm_generic_event_t P5mmx_generic_names1[] = {
        P5_GENERIC_EVENTS,
        { "PAPI_br_ins",        "taken_br",     0x0 },
        CPC_GEN_END
};

static const ptm_generic_event_t *P5mmx_generic_names[2] = {
        P5mmx_generic_names0,
        P5mmx_generic_names1
};

static const ptm_generic_event_t _P6_generic_names[] = {
        { "PAPI_ca_shr",        "l2_ifetch",            0xf },
        { "PAPI_ca_cln",        "bus_tran_rfo",         0x0 },
        { "PAPI_ca_itv",        "bus_tran_inval",       0x0 },
        { "PAPI_tlb_im",        "itlb_miss",            0x0 },
        { "PAPI_btac_m",        "btb_misses",           0x0 },
        { "PAPI_hw_int",        "hw_int_rx",            0x0 },
        { "PAPI_br_cn",         "br_inst_retired",      0x0 },
        { "PAPI_br_tkn",        "br_taken_retired",     0x0 },
        { "PAPI_br_msp",        "br_miss_pred_taken_ret", 0x0 },
        { "PAPI_br_ins",        "br_inst_retired",      0x0 },
        { "PAPI_res_stl",       "resource_stalls",      0x0 },
        { "PAPI_tot_iis",       "inst_decoder",         0x0 },
        { "PAPI_tot_ins",       "inst_retired",         0x0 },
        { "PAPI_tot_cyc",       "cpu_clk_unhalted",     0x0 },
        { "PAPI_l1_dcm",        "dcu_lines_in",         0x0 },
        { "PAPI_l1_icm",        "l2_ifetch",            0xf },
        { "PAPI_l1_tcm",        "l2_rqsts",             0xf },
        { "PAPI_l1_dca",        "data_mem_refs",        0x0 },
        { "PAPI_l1_stm",        "l2_st",                0xf },
        { "PAPI_l2_icm",        "bus_tran_ifetch",      0x0 },
        { "PAPI_l2_dcr",        "l2_ld",                0xf },
        { "PAPI_l2_dcw",        "l2_st",                0xf },
        { "PAPI_l2_tcm",        "l2_lines_in",          0x0 },
        { "PAPI_l2_tca",        "l2_rqsts",             0xf },
        { "PAPI_l2_tcw",        "l2_st",                0xf },
        { "PAPI_l2_stm",        "l2_m_lines_inm",       0x0 },
        { "PAPI_fp_ins",        "flops",                0x0 },
        { "PAPI_fp_ops",        "flops",                0x0 },
        { "PAPI_fml_ins",       "mul",                  0x0 },
        { "PAPI_fdv_ins",       "div",                  0x0 },
        CPC_GEN_END
};

static const ptm_generic_event_t *P6_generic_names[2] = {
        _P6_generic_names,
        _P6_generic_names
};

static const struct nametable **events;
static const ptm_generic_event_t **generic_events;

#define BITS(v, u, l)   \
        (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))

/*
 * "Well known" bit fields in the Pentium CES register
 * The interfaces in libcpc should make these #defines uninteresting.
 */
#define CPC_P5_CESR_ES0_SHIFT   0
#define CPC_P5_CESR_ES0_MASK    0x3f
#define CPC_P5_CESR_ES1_SHIFT   16
#define CPC_P5_CESR_ES1_MASK    0x3f

#define CPC_P5_CESR_OS0         6
#define CPC_P5_CESR_USR0        7
#define CPC_P5_CESR_CLK0        8
#define CPC_P5_CESR_PC0         9
#define CPC_P5_CESR_OS1         (CPC_P5_CESR_OS0 + 16)
#define CPC_P5_CESR_USR1        (CPC_P5_CESR_USR0 + 16)
#define CPC_P5_CESR_CLK1        (CPC_P5_CESR_CLK0 + 16)
#define CPC_P5_CESR_PC1         (CPC_P5_CESR_PC0 + 16)

/*
 * "Well known" bit fields in the Pentium Pro PerfEvtSel registers
 * The interfaces in libcpc should make these #defines uninteresting.
 */
#define CPC_P6_PES_INV          23
#define CPC_P6_PES_EN           22
#define CPC_P6_PES_INT          20
#define CPC_P6_PES_PC           19
#define CPC_P6_PES_E            18
#define CPC_P6_PES_OS           17
#define CPC_P6_PES_USR          16

#define CPC_P6_PES_UMASK_SHIFT  8
#define CPC_P6_PES_UMASK_MASK   (0xffu)

#define CPC_P6_PES_CMASK_SHIFT  24
#define CPC_P6_PES_CMASK_MASK   (0xffu)

#define CPC_P6_PES_PIC0_MASK    (0xffu)
#define CPC_P6_PES_PIC1_MASK    (0xffu)

#define P6_PES_EN       (UINT32_C(1) << CPC_P6_PES_EN)
#define P6_PES_INT      (UINT32_C(1) << CPC_P6_PES_INT)
#define P6_PES_OS       (UINT32_C(1) << CPC_P6_PES_OS)

/*
 * Pentium 5 attributes
 */
#define P5_NOEDGE       0x1     /* "noedge"     - no edge detection */
#define P5_PC           0x2     /* "pc"         - pin control */

/*
 * Pentium 6 attributes
 */
#define P6_NOEDGE       0x1
#define P6_PC           0x2
#define P6_INV          0x4     /* "inv" - count inverted transitions */
#define P6_INT          0x8     /* "int" - interrupt on overflow */

/*
 * CPU reference strings
 */

#define P5_CPUREF       "See Appendix A.4 of the \"IA-32 Intel Architecture "  \
                        "Software Developer's Manual Volume 3: System "        \
                        "Programming Guide,\" Order # 245472-012, 2003"

#define P6_CPUREF       "See Appendix A.3 of the \"IA-32 Intel Architecture "  \
                        "Software Developer's Manual Volume 3: System "        \
                        "Programming Guide,\" Order # 245472-012, 2003"

static int
ptm_pcbe_init(void)
{
        const struct nametable          *n;
        const ptm_generic_event_t       *gevp;
        int                             i;
        size_t                          size;

        if (is_x86_feature(x86_featureset, X86FSET_MMX))
                ptm_rdpmc_avail = 1;

        /*
         * Discover type of CPU and set events pointer appropriately.
         *
         * Map family and model into the performance
         * counter architectures we currently understand.
         *
         * See application note AP485 (from developer.intel.com)
         * for further explanation.
         */
        if (cpuid_getvendor(CPU) != X86_VENDOR_Intel)
                return (-1);
        switch (cpuid_getfamily(CPU)) {
        case 5:         /* Pentium and Pentium with MMX */
                events = P5mmx_names;
                generic_events = P5mmx_generic_names;
                ptm_ver = PTM_VER_P5;
                ptm_cpuref = P5_CPUREF;
                if (cpuid_getmodel(CPU) < 4)
                        ptm_impl_name = "Pentium";
                else
                        ptm_impl_name = "Pentium with MMX";
                break;
        case 6:         /* Pentium Pro and Pentium II and III */
                events = P6_names;
                generic_events = P6_generic_names;
                ptm_ver = PTM_VER_P6;
                ptm_cpuref = P6_CPUREF;
                ptm_pcbe_ops.pcbe_caps = CPC_CAP_OVERFLOW_INTERRUPT;
                if (is_x86_feature(x86_featureset, X86FSET_MMX))
                        ptm_impl_name = "Pentium Pro with MMX, Pentium II";
                else
                        ptm_impl_name = "Pentium Pro, Pentium II";
                break;
        default:
                return (-1);
        }

        /*
         * Initialize the list of events for each PIC.
         * Do two passes: one to compute the size necessary and another
         * to copy the strings. Need room for event, comma, and NULL terminator.
         */
        for (i = 0; i < 2; i++) {
                size = 0;
                for (n = events[i]; n->bits != NT_END; n++)
                        size += strlen(n->name) + 1;
                for (gevp = generic_events[i]; gevp->name != NULL; gevp++)
                        size += strlen(gevp->name) + 1;
                pic_events[i] = kmem_alloc(size + 1, KM_SLEEP);
                *pic_events[i] = '\0';
                for (n = events[i]; n->bits != NT_END; n++) {
                        (void) strcat(pic_events[i], n->name);
                        (void) strcat(pic_events[i], ",");
                }
                for (gevp = generic_events[i]; gevp->name != NULL; gevp++) {
                        (void) strcat(pic_events[i], gevp->name);
                        (void) strcat(pic_events[i], ",");
                }

                /*
                 * Remove trailing comma.
                 */
                pic_events[i][size - 1] = '\0';
        }

        return (0);
}

static uint_t
ptm_pcbe_ncounters(void)
{
        return (2);
}

static const char *
ptm_pcbe_impl_name(void)
{
        return (ptm_impl_name);
}

static const char *
ptm_pcbe_cpuref(void)
{
        return (ptm_cpuref);
}

static char *
ptm_pcbe_list_events(uint_t picnum)
{
        ASSERT(picnum >= 0 && picnum < cpc_ncounters);

        if (pic_events[0] == NULL) {
                ASSERT(pic_events[1] == NULL);
        }

        return (pic_events[picnum]);
}

static char *
ptm_pcbe_list_attrs(void)
{
        if (ptm_ver == PTM_VER_P5)
                return ("noedge,pc");
        else
                return ("noedge,pc,inv,int,umask,cmask");
}

static const ptm_generic_event_t *
find_generic_event(int regno, char *name)
{
        const ptm_generic_event_t       *gevp;

        for (gevp = generic_events[regno]; gevp->name != NULL; gevp++)
                if (strcmp(name, gevp->name) == 0)
                        return (gevp);

        return (NULL);
}

static const struct nametable *
find_event(int regno, char *name)
{
        const struct nametable *n;

        n = events[regno];

        for (; n->bits != NT_END; n++)
                if (strcmp(name, n->name) == 0)
                        return (n);

        return (NULL);
}

static uint64_t
ptm_pcbe_event_coverage(char *event)
{
        uint64_t bitmap = 0;

        if ((find_event(0, event) != NULL) ||
            (find_generic_event(0, event) != NULL))
                bitmap = 0x1;
        if ((find_event(1, event) != NULL) ||
            (find_generic_event(1, event) != NULL))
                bitmap |= 0x2;

        return (bitmap);
}

static uint64_t
ptm_pcbe_overflow_bitmap(void)
{
        uint64_t        ret = 0;
        uint64_t        pes[2];

        /*
         * P5 is not capable of generating interrupts.
         */
        ASSERT(ptm_ver == PTM_VER_P6);

        /*
         * CPC could have caused an interrupt provided that
         *
         * 1) Counters are enabled
         * 2) Either counter has requested an interrupt
         */

        pes[0] = rdmsr(REG_PERFEVNT0);
        if (((uint32_t)pes[0] & P6_PES_EN) != P6_PES_EN)
                return (0);

        /*
         * If a particular counter requested an interrupt, assume it caused
         * this interrupt. There is no way to determine which counter overflowed
         * on this hardware other than by using unreliable heuristics.
         */

        pes[1] = rdmsr(REG_PERFEVNT1);
        if ((uint32_t)pes[0] & P6_PES_INT)
                ret |= 0x1;
        if ((uint32_t)pes[1] & P6_PES_INT)
                ret |= 0x2;

        return (ret);
}

/*ARGSUSED*/
static int
ptm_pcbe_configure(uint_t picnum, char *eventname, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token)
{
        ptm_pcbe_config_t               *conf;
        const struct nametable          *n;
        const ptm_generic_event_t       *gevp;
        struct nametable                nt_raw = { 0, "raw" };
        int                             i;
        int                             ptm_flags = 0;

        /*
         * If we've been handed an existing configuration, we need only preset
         * the counter value.
         */
        if (*data != NULL) {
                conf = *data;
                conf->ptm_rawpic = trunc3931(preset);
                return (0);
        }

        if (picnum != 0 && picnum != 1)
                return (CPC_INVALID_PICNUM);

        conf = kmem_alloc(sizeof (ptm_pcbe_config_t), KM_SLEEP);

        conf->ptm_picno = picnum;
        conf->ptm_rawpic = trunc3931(preset);
        conf->ptm_ctl = 0;

        if ((n = find_event(picnum, eventname)) == NULL) {
                if ((gevp = find_generic_event(picnum, eventname)) != NULL) {
                        n = find_event(picnum, gevp->event);
                        ASSERT(n != NULL);

                        if (nattrs > 0) {
                                kmem_free(conf, sizeof (ptm_pcbe_config_t));
                                return (CPC_ATTRIBUTE_OUT_OF_RANGE);
                        }

                        if (ptm_ver == PTM_VER_P6)
                                conf->ptm_ctl |= gevp->umask <<
                                    CPC_P6_PES_UMASK_SHIFT;
                } else {
                        long tmp;

                        /*
                         * If ddi_strtol() likes this event, use it as a raw
                         * event code.
                         */
                        if (ddi_strtol(eventname, NULL, 0, &tmp) != 0) {
                                kmem_free(conf, sizeof (ptm_pcbe_config_t));
                                return (CPC_INVALID_EVENT);
                        }

                        nt_raw.bits = tmp;

                        if (ptm_ver == PTM_VER_P5)
                                nt_raw.bits &= CPC_P5_CESR_ES0_MASK;
                        else
                                nt_raw.bits &= CPC_P6_PES_PIC0_MASK;

                        n = &nt_raw;
                }
        }

        if (ptm_ver == PTM_VER_P5) {
                int picshift;
                picshift = (picnum == 0) ? 0 : 16;

                for (i = 0; i < nattrs; i++) {
                        /*
                         * Value of these attributes is ignored; their presence
                         * alone tells us to set the corresponding flag.
                         */
                        if (strncmp(attrs[i].ka_name, "noedge", 7) == 0) {
                                if (attrs[i].ka_val != 0)
                                        ptm_flags |= P5_NOEDGE;
                        } else if (strncmp(attrs[i].ka_name, "pc", 3) == 0) {
                                if (attrs[i].ka_val != 0)
                                        ptm_flags |= P5_PC;
                        } else {
                                kmem_free(conf, sizeof (ptm_pcbe_config_t));
                                return (CPC_INVALID_ATTRIBUTE);
                        }
                }

                if (flags & CPC_COUNT_USER)
                        conf->ptm_ctl |= (1 << (CPC_P5_CESR_USR0 + picshift));
                if (flags & CPC_COUNT_SYSTEM)
                        conf->ptm_ctl |= (1 << (CPC_P5_CESR_OS0 + picshift));
                if (ptm_flags & P5_NOEDGE)
                        conf->ptm_ctl |= (1 << (CPC_P5_CESR_CLK0 + picshift));
                if (ptm_flags & P5_PC)
                        conf->ptm_ctl |= (1 << (CPC_P5_CESR_PC0 + picshift));

                ASSERT((n->bits | CPC_P5_CESR_ES0_MASK) ==
                    CPC_P5_CESR_ES0_MASK);

                conf->ptm_ctl |= (n->bits << picshift);
        } else {
                for (i = 0; i < nattrs; i++) {
                        if (strncmp(attrs[i].ka_name, "noedge", 6) == 0) {
                                if (attrs[i].ka_val != 0)
                                        ptm_flags |= P6_NOEDGE;
                        } else if (strncmp(attrs[i].ka_name, "pc", 2) == 0) {
                                if (attrs[i].ka_val != 0)
                                        ptm_flags |= P6_PC;
                        } else if (strncmp(attrs[i].ka_name, "inv", 3) == 0) {
                                if (attrs[i].ka_val != 0)
                                        ptm_flags |= P6_INV;
                        } else if (strncmp(attrs[i].ka_name, "umask", 5) == 0) {
                                if ((attrs[i].ka_val | CPC_P6_PES_UMASK_MASK) !=
                                    CPC_P6_PES_UMASK_MASK) {
                                        kmem_free(conf,
                                            sizeof (ptm_pcbe_config_t));
                                        return (CPC_ATTRIBUTE_OUT_OF_RANGE);
                                }
                                conf->ptm_ctl |= (uint8_t)attrs[i].ka_val <<
                                    CPC_P6_PES_UMASK_SHIFT;
                        } else if (strncmp(attrs[i].ka_name, "cmask", 5) == 0) {
                                if ((attrs[i].ka_val | CPC_P6_PES_CMASK_MASK) !=
                                    CPC_P6_PES_CMASK_MASK) {
                                        kmem_free(conf,
                                            sizeof (ptm_pcbe_config_t));
                                        return (CPC_ATTRIBUTE_OUT_OF_RANGE);
                                }
                                conf->ptm_ctl |= (uint8_t)attrs[i].ka_val <<
                                    CPC_P6_PES_CMASK_SHIFT;
                        } else if (strncmp(attrs[i].ka_name, "int", 3) == 0) {
                                if (attrs[i].ka_val != 0)
                                        ptm_flags |= P6_INT;
                        } else {
                                kmem_free(conf, sizeof (ptm_pcbe_config_t));
                                return (CPC_INVALID_ATTRIBUTE);
                        }
                }

                if (flags & CPC_OVF_NOTIFY_EMT)
                        /*
                         * If the user has requested notification of overflows,
                         * we automatically program the hardware to generate
                         * overflow interrupts.
                         */
                        ptm_flags |= P6_INT;
                if (flags & CPC_COUNT_USER)
                        conf->ptm_ctl |= (1 << CPC_P6_PES_USR);
                if (flags & CPC_COUNT_SYSTEM)
                        conf->ptm_ctl |= (1 << CPC_P6_PES_OS);
                if ((ptm_flags & P6_NOEDGE) == 0)
                        conf->ptm_ctl |= (1 << CPC_P6_PES_E);
                if (ptm_flags & P6_PC)
                        conf->ptm_ctl |= (1 << CPC_P6_PES_PC);
                if (ptm_flags & P6_INV)
                        conf->ptm_ctl |= (1 << CPC_P6_PES_INV);
                if (ptm_flags & P6_INT)
                        conf->ptm_ctl |= (1 << CPC_P6_PES_INT);

                ASSERT((n->bits | CPC_P6_PES_PIC0_MASK) ==
                    CPC_P6_PES_PIC0_MASK);

                conf->ptm_ctl |= n->bits;
        }

        *data = conf;
        return (0);
}

static void
ptm_pcbe_program(void *token)
{
        ptm_pcbe_config_t       *pic0;
        ptm_pcbe_config_t       *pic1;
        ptm_pcbe_config_t       *tmp;
        ptm_pcbe_config_t       empty = { 1, 0, 0 }; /* assume pic1 to start */

        if ((pic0 = kcpc_next_config(token, NULL, NULL)) == NULL)
                panic("ptm_pcbe: token %p has no configs", token);

        if ((pic1 = kcpc_next_config(token, pic0, NULL)) == NULL)
                pic1 = &empty;

        if (pic0->ptm_picno != 0) {
                empty.ptm_picno = 0;
                tmp = pic1;
                pic1 = pic0;
                pic0 = tmp;
        }

        ASSERT(pic0->ptm_picno == 0 && pic1->ptm_picno == 1);

        if (ptm_rdpmc_avail) {
                ulong_t curcr4 = getcr4();
                if (kcpc_allow_nonpriv(token))
                        setcr4(curcr4 | CR4_PCE);
                else
                        setcr4(curcr4 & ~CR4_PCE);
        }

        if (ptm_ver == PTM_VER_P5) {
                wrmsr(P5_CESR, ALL_STOPPED);
                wrmsr(P5_CTR0, pic0->ptm_rawpic);
                wrmsr(P5_CTR1, pic1->ptm_rawpic);
                wrmsr(P5_CESR, pic0->ptm_ctl | pic1->ptm_ctl);
                pic0->ptm_rawpic = rdmsr(P5_CTR0);
                pic1->ptm_rawpic = rdmsr(P5_CTR1);
        } else {
                uint64_t        pes;
                wrmsr(REG_PERFEVNT0, ALL_STOPPED);
                wrmsr(REG_PERFCTR0, pic0->ptm_rawpic);
                wrmsr(REG_PERFCTR1, pic1->ptm_rawpic);
                pes = pic1->ptm_ctl;
                DTRACE_PROBE1(ptm__pes1, uint64_t, pes);
                wrmsr(REG_PERFEVNT1, pes);
                pes = pic0->ptm_ctl | (1 << CPC_P6_PES_EN);
                DTRACE_PROBE1(ptm__pes0, uint64_t, pes);
                wrmsr(REG_PERFEVNT0, pes);
        }
}

static void
ptm_pcbe_allstop(void)
{
        if (ptm_ver == PTM_VER_P5)
                wrmsr(P5_CESR, ALL_STOPPED);
        else {
                wrmsr(REG_PERFEVNT0, ALL_STOPPED);
                setcr4(getcr4() & ~CR4_PCE);
        }
}

static void
ptm_pcbe_sample(void *token)
{
        ptm_pcbe_config_t       *pic0;
        ptm_pcbe_config_t       *pic1;
        ptm_pcbe_config_t       *swap;
        ptm_pcbe_config_t       empty = { 1, 0, 0 }; /* assume pic1 to start */
        uint64_t                tmp;
        uint64_t                *pic0_data;
        uint64_t                *pic1_data;
        uint64_t                *dtmp;
        uint64_t                curpic[2];

        if ((pic0 = kcpc_next_config(token, NULL, &pic0_data)) == NULL)
                panic("ptm_pcbe: token %p has no configs", token);

        if ((pic1 = kcpc_next_config(token, pic0, &pic1_data)) == NULL) {
                pic1 = &empty;
                pic1_data = &tmp;
        }

        if (pic0->ptm_picno != 0) {
                empty.ptm_picno = 0;
                swap = pic0;
                pic0 = pic1;
                pic1 = swap;
                dtmp = pic0_data;
                pic0_data = pic1_data;
                pic1_data = dtmp;
        }

        ASSERT(pic0->ptm_picno == 0 && pic1->ptm_picno == 1);

        if (ptm_ver == PTM_VER_P5) {
                curpic[0] = rdmsr(P5_CTR0);
                curpic[1] = rdmsr(P5_CTR1);
        } else {
                curpic[0] = rdmsr(REG_PERFCTR0);
                curpic[1] = rdmsr(REG_PERFCTR1);
        }

        DTRACE_PROBE1(ptm__curpic0, uint64_t, curpic[0]);
        DTRACE_PROBE1(ptm__curpic1, uint64_t, curpic[1]);

        *pic0_data += diff3931(curpic[0], pic0->ptm_rawpic);
        pic0->ptm_rawpic = trunc3931(*pic0_data);

        *pic1_data += diff3931(curpic[1], pic1->ptm_rawpic);
        pic1->ptm_rawpic = trunc3931(*pic1_data);
}

static void
ptm_pcbe_free(void *config)
{
        kmem_free(config, sizeof (ptm_pcbe_config_t));
}

/*
 * Virtualizes the 40-bit field of the %pic
 * register into a 64-bit software register.
 *
 * We can retrieve 40 (signed) bits from the counters,
 * but we can set only 32 (signed) bits into the counters.
 * This makes virtualizing more than 31-bits of registers
 * quite tricky.
 *
 * If bits 39 to 31 are set in the virtualized pic register,
 * then we can preset the counter to this value using the fact
 * that wrmsr sign extends bit 31.   Though it might look easier
 * to only use the bottom 31-bits of the register, we have to allow
 * the full 40-bits to be used to perform overflow profiling.
 */

#define MASK40          UINT64_C(0xffffffffff)
#define MASK31          UINT64_C(0x7fffffff)
#define BITS_39_31      UINT64_C(0xff80000000)

static int64_t
diff3931(uint64_t sample, uint64_t old)
{
        int64_t diff;

        if ((old & BITS_39_31) == BITS_39_31) {
                diff = (MASK40 & sample) - old;
                if (diff < 0)
                        diff += (UINT64_C(1) << 40);
        } else {
                diff = (MASK31 & sample) - old;
                if (diff < 0)
                        diff += (UINT64_C(1) << 31);
        }
        return (diff);
}

static uint64_t
trunc3931(uint64_t value)
{
        if ((value & BITS_39_31) == BITS_39_31)
                return (MASK40 & value);
        return (MASK31 & value);
}

static struct modlpcbe modlpcbe = {
        &mod_pcbeops,
        "Pentium Performance Counters",
        &ptm_pcbe_ops
};

static struct modlinkage modl = {
        MODREV_1,
        &modlpcbe,
};

int
_init(void)
{
        if (ptm_pcbe_init() != 0)
                return (ENOTSUP);
        return (mod_install(&modl));
}

int
_fini(void)
{
        return (mod_remove(&modl));
}

int
_info(struct modinfo *mi)
{
        return (mod_info(&modl, mi));
}