arch/mips/kernel/perf_event_mipsxx.c

   1 #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \
   2     defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1)
   3
   4 #define M_CONFIG1_PC    (1 << 4)
   5
   6 #define M_PERFCTL_EXL                   (1UL      <<  0)
   7 #define M_PERFCTL_KERNEL                (1UL      <<  1)
   8 #define M_PERFCTL_SUPERVISOR            (1UL      <<  2)
   9 #define M_PERFCTL_USER                  (1UL      <<  3)
  10 #define M_PERFCTL_INTERRUPT_ENABLE      (1UL      <<  4)
  11 #define M_PERFCTL_EVENT(event)          (((event) & 0x3ff)  << 5)
  12 #define M_PERFCTL_VPEID(vpe)            ((vpe)    << 16)
  13 #define M_PERFCTL_MT_EN(filter)         ((filter) << 20)
  14 #define    M_TC_EN_ALL                  M_PERFCTL_MT_EN(0)
  15 #define    M_TC_EN_VPE                  M_PERFCTL_MT_EN(1)
  16 #define    M_TC_EN_TC                   M_PERFCTL_MT_EN(2)
  17 #define M_PERFCTL_TCID(tcid)            ((tcid)   << 22)
  18 #define M_PERFCTL_WIDE                  (1UL      << 30)
  19 #define M_PERFCTL_MORE                  (1UL      << 31)
  20
  21 #define M_PERFCTL_COUNT_EVENT_WHENEVER  (M_PERFCTL_EXL |                \
  22                                         M_PERFCTL_KERNEL |              \
  23                                         M_PERFCTL_USER |                \
  24                                         M_PERFCTL_SUPERVISOR |          \
  25                                         M_PERFCTL_INTERRUPT_ENABLE)
  26
  27 #ifdef CONFIG_MIPS_MT_SMP
  28 #define M_PERFCTL_CONFIG_MASK           0x3fff801f
  29 #else
  30 #define M_PERFCTL_CONFIG_MASK           0x1f
  31 #endif
  32 #define M_PERFCTL_EVENT_MASK            0xfe0
  33
  34 #define M_COUNTER_OVERFLOW              (1UL      << 31)
  35
  36 #ifdef CONFIG_MIPS_MT_SMP
  37 static int cpu_has_mipsmt_pertccounters;
  38
  39 /*
  40  * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because
  41  * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs.
  42  */
  43 #if defined(CONFIG_HW_PERF_EVENTS)
  44 #define vpe_id()        (cpu_has_mipsmt_pertccounters ? \
  45                         0 : smp_processor_id())
  46 #else
  47 #define vpe_id()        (cpu_has_mipsmt_pertccounters ? \
  48                         0 : cpu_data[smp_processor_id()].vpe_id)
  49 #endif
  50
  51 /* Copied from op_model_mipsxx.c */
  52 static inline unsigned int vpe_shift(void)
  53 {
  54         if (num_possible_cpus() > 1)
  55                 return 1;
  56
  57         return 0;
  58 }
  59 #else /* !CONFIG_MIPS_MT_SMP */
  60 #define vpe_id()        0
  61
  62 static inline unsigned int vpe_shift(void)
  63 {
  64         return 0;
  65 }
  66 #endif /* CONFIG_MIPS_MT_SMP */
  67
  68 static inline unsigned int
  69 counters_total_to_per_cpu(unsigned int counters)
  70 {
  71         return counters >> vpe_shift();
  72 }
  73
  74 static inline unsigned int
  75 counters_per_cpu_to_total(unsigned int counters)
  76 {
  77         return counters << vpe_shift();
  78 }
  79
  80 #define __define_perf_accessors(r, n, np)                               \
  81                                                                         \
  82 static inline unsigned int r_c0_ ## r ## n(void)                        \
  83 {                                                                       \
  84         unsigned int cpu = vpe_id();                                    \
  85                                                                         \
  86         switch (cpu) {                                                  \
  87         case 0:                                                         \
  88                 return read_c0_ ## r ## n();                            \
  89         case 1:                                                         \
  90                 return read_c0_ ## r ## np();                           \
  91         default:                                                        \
  92                 BUG();                                                  \
  93         }                                                               \
  94         return 0;                                                       \
  95 }                                                                       \
  96                                                                         \
  97 static inline void w_c0_ ## r ## n(unsigned int value)                  \
  98 {                                                                       \
  99         unsigned int cpu = vpe_id();                                    \
 100                                                                         \
 101         switch (cpu) {                                                  \
 102         case 0:                                                         \
 103                 write_c0_ ## r ## n(value);                             \
 104                 return;                                                 \
 105         case 1:                                                         \
 106                 write_c0_ ## r ## np(value);                            \
 107                 return;                                                 \
 108         default:                                                        \
 109                 BUG();                                                  \
 110         }                                                               \
 111         return;                                                         \
 112 }                                                                       \
 113
 114 __define_perf_accessors(perfcntr, 0, 2)
 115 __define_perf_accessors(perfcntr, 1, 3)
 116 __define_perf_accessors(perfcntr, 2, 0)
 117 __define_perf_accessors(perfcntr, 3, 1)
 118
 119 __define_perf_accessors(perfctrl, 0, 2)
 120 __define_perf_accessors(perfctrl, 1, 3)
 121 __define_perf_accessors(perfctrl, 2, 0)
 122 __define_perf_accessors(perfctrl, 3, 1)
 123
 124 static inline int __n_counters(void)
 125 {
 126         if (!(read_c0_config1() & M_CONFIG1_PC))
 127                 return 0;
 128         if (!(read_c0_perfctrl0() & M_PERFCTL_MORE))
 129                 return 1;
 130         if (!(read_c0_perfctrl1() & M_PERFCTL_MORE))
 131                 return 2;
 132         if (!(read_c0_perfctrl2() & M_PERFCTL_MORE))
 133                 return 3;
 134
 135         return 4;
 136 }
 137
 138 static inline int n_counters(void)
 139 {
 140         int counters;
 141
 142         switch (current_cpu_type()) {
 143         case CPU_R10000:
 144                 counters = 2;
 145                 break;
 146
 147         case CPU_R12000:
 148         case CPU_R14000:
 149                 counters = 4;
 150                 break;
 151
 152         default:
 153                 counters = __n_counters();
 154         }
 155
 156         return counters;
 157 }
 158
 159 static void reset_counters(void *arg)
 160 {
 161         int counters = (int)(long)arg;
 162         switch (counters) {
 163         case 4:
 164                 w_c0_perfctrl3(0);
 165                 w_c0_perfcntr3(0);
 166         case 3:
 167                 w_c0_perfctrl2(0);
 168                 w_c0_perfcntr2(0);
 169         case 2:
 170                 w_c0_perfctrl1(0);
 171                 w_c0_perfcntr1(0);
 172         case 1:
 173                 w_c0_perfctrl0(0);
 174                 w_c0_perfcntr0(0);
 175         }
 176 }
 177
 178 static inline u64
 179 mipsxx_pmu_read_counter(unsigned int idx)
 180 {
 181         switch (idx) {
 182         case 0:
 183                 return r_c0_perfcntr0();
 184         case 1:
 185                 return r_c0_perfcntr1();
 186         case 2:
 187                 return r_c0_perfcntr2();
 188         case 3:
 189                 return r_c0_perfcntr3();
 190         default:
 191                 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 192                 return 0;
 193         }
 194 }
 195
 196 static inline void
 197 mipsxx_pmu_write_counter(unsigned int idx, u64 val)
 198 {
 199         switch (idx) {
 200         case 0:
 201                 w_c0_perfcntr0(val);
 202                 return;
 203         case 1:
 204                 w_c0_perfcntr1(val);
 205                 return;
 206         case 2:
 207                 w_c0_perfcntr2(val);
 208                 return;
 209         case 3:
 210                 w_c0_perfcntr3(val);
 211                 return;
 212         }
 213 }
 214
 215 static inline unsigned int
 216 mipsxx_pmu_read_control(unsigned int idx)
 217 {
 218         switch (idx) {
 219         case 0:
 220                 return r_c0_perfctrl0();
 221         case 1:
 222                 return r_c0_perfctrl1();
 223         case 2:
 224                 return r_c0_perfctrl2();
 225         case 3:
 226                 return r_c0_perfctrl3();
 227         default:
 228                 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx);
 229                 return 0;
 230         }
 231 }
 232
 233 static inline void
 234 mipsxx_pmu_write_control(unsigned int idx, unsigned int val)
 235 {
 236         switch (idx) {
 237         case 0:
 238                 w_c0_perfctrl0(val);
 239                 return;
 240         case 1:
 241                 w_c0_perfctrl1(val);
 242                 return;
 243         case 2:
 244                 w_c0_perfctrl2(val);
 245                 return;
 246         case 3:
 247                 w_c0_perfctrl3(val);
 248                 return;
 249         }
 250 }
 251
 252 #ifdef CONFIG_MIPS_MT_SMP
 253 static DEFINE_RWLOCK(pmuint_rwlock);
 254 #endif
 255
 256 /* 24K/34K/1004K cores can share the same event map. */
 257 static const struct mips_perf_event mipsxxcore_event_map
 258                                 [PERF_COUNT_HW_MAX] = {
 259         [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
 260         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 261         [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID },
 262         [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID },
 263         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T },
 264         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T },
 265         [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID },
 266 };
 267
 268 /* 74K core has different branch event code. */
 269 static const struct mips_perf_event mipsxx74Kcore_event_map
 270                                 [PERF_COUNT_HW_MAX] = {
 271         [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P },
 272         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T },
 273         [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID },
 274         [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID },
 275         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T },
 276         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T },
 277         [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID },
 278 };
 279
 280 /* 24K/34K/1004K cores can share the same cache event map. */
 281 static const struct mips_perf_event mipsxxcore_cache_map
 282                                 [PERF_COUNT_HW_CACHE_MAX]
 283                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 284                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 285 [C(L1D)] = {
 286         /*
 287          * Like some other architectures (e.g. ARM), the performance
 288          * counters don't differentiate between read and write
 289          * accesses/misses, so this isn't strictly correct, but it's the
 290          * best we can do. Writes and reads get combined.
 291          */
 292         [C(OP_READ)] = {
 293                 [C(RESULT_ACCESS)]      = { 0x0a, CNTR_EVEN, T },
 294                 [C(RESULT_MISS)]        = { 0x0b, CNTR_EVEN | CNTR_ODD, T },
 295         },
 296         [C(OP_WRITE)] = {
 297                 [C(RESULT_ACCESS)]      = { 0x0a, CNTR_EVEN, T },
 298                 [C(RESULT_MISS)]        = { 0x0b, CNTR_EVEN | CNTR_ODD, T },
 299         },
 300         [C(OP_PREFETCH)] = {
 301                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 302                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 303         },
 304 },
 305 [C(L1I)] = {
 306         [C(OP_READ)] = {
 307                 [C(RESULT_ACCESS)]      = { 0x09, CNTR_EVEN, T },
 308                 [C(RESULT_MISS)]        = { 0x09, CNTR_ODD, T },
 309         },
 310         [C(OP_WRITE)] = {
 311                 [C(RESULT_ACCESS)]      = { 0x09, CNTR_EVEN, T },
 312                 [C(RESULT_MISS)]        = { 0x09, CNTR_ODD, T },
 313         },
 314         [C(OP_PREFETCH)] = {
 315                 [C(RESULT_ACCESS)]      = { 0x14, CNTR_EVEN, T },
 316                 /*
 317                  * Note that MIPS has only "hit" events countable for
 318                  * the prefetch operation.
 319                  */
 320                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 321         },
 322 },
 323 [C(LL)] = {
 324         [C(OP_READ)] = {
 325                 [C(RESULT_ACCESS)]      = { 0x15, CNTR_ODD, P },
 326                 [C(RESULT_MISS)]        = { 0x16, CNTR_EVEN, P },
 327         },
 328         [C(OP_WRITE)] = {
 329                 [C(RESULT_ACCESS)]      = { 0x15, CNTR_ODD, P },
 330                 [C(RESULT_MISS)]        = { 0x16, CNTR_EVEN, P },
 331         },
 332         [C(OP_PREFETCH)] = {
 333                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 334                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 335         },
 336 },
 337 [C(DTLB)] = {
 338         [C(OP_READ)] = {
 339                 [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 340                 [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 341         },
 342         [C(OP_WRITE)] = {
 343                 [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 344                 [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 345         },
 346         [C(OP_PREFETCH)] = {
 347                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 348                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 349         },
 350 },
 351 [C(ITLB)] = {
 352         [C(OP_READ)] = {
 353                 [C(RESULT_ACCESS)]      = { 0x05, CNTR_EVEN, T },
 354                 [C(RESULT_MISS)]        = { 0x05, CNTR_ODD, T },
 355         },
 356         [C(OP_WRITE)] = {
 357                 [C(RESULT_ACCESS)]      = { 0x05, CNTR_EVEN, T },
 358                 [C(RESULT_MISS)]        = { 0x05, CNTR_ODD, T },
 359         },
 360         [C(OP_PREFETCH)] = {
 361                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 362                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 363         },
 364 },
 365 [C(BPU)] = {
 366         /* Using the same code for *HW_BRANCH* */
 367         [C(OP_READ)] = {
 368                 [C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN, T },
 369                 [C(RESULT_MISS)]        = { 0x02, CNTR_ODD, T },
 370         },
 371         [C(OP_WRITE)] = {
 372                 [C(RESULT_ACCESS)]      = { 0x02, CNTR_EVEN, T },
 373                 [C(RESULT_MISS)]        = { 0x02, CNTR_ODD, T },
 374         },
 375         [C(OP_PREFETCH)] = {
 376                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 377                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 378         },
 379 },
 380 };
 381
 382 /* 74K core has completely different cache event map. */
 383 static const struct mips_perf_event mipsxx74Kcore_cache_map
 384                                 [PERF_COUNT_HW_CACHE_MAX]
 385                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 386                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 387 [C(L1D)] = {
 388         /*
 389          * Like some other architectures (e.g. ARM), the performance
 390          * counters don't differentiate between read and write
 391          * accesses/misses, so this isn't strictly correct, but it's the
 392          * best we can do. Writes and reads get combined.
 393          */
 394         [C(OP_READ)] = {
 395                 [C(RESULT_ACCESS)]      = { 0x17, CNTR_ODD, T },
 396                 [C(RESULT_MISS)]        = { 0x18, CNTR_ODD, T },
 397         },
 398         [C(OP_WRITE)] = {
 399                 [C(RESULT_ACCESS)]      = { 0x17, CNTR_ODD, T },
 400                 [C(RESULT_MISS)]        = { 0x18, CNTR_ODD, T },
 401         },
 402         [C(OP_PREFETCH)] = {
 403                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 404                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 405         },
 406 },
 407 [C(L1I)] = {
 408         [C(OP_READ)] = {
 409                 [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 410                 [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 411         },
 412         [C(OP_WRITE)] = {
 413                 [C(RESULT_ACCESS)]      = { 0x06, CNTR_EVEN, T },
 414                 [C(RESULT_MISS)]        = { 0x06, CNTR_ODD, T },
 415         },
 416         [C(OP_PREFETCH)] = {
 417                 [C(RESULT_ACCESS)]      = { 0x34, CNTR_EVEN, T },
 418                 /*
 419                  * Note that MIPS has only "hit" events countable for
 420                  * the prefetch operation.
 421                  */
 422                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 423         },
 424 },
 425 [C(LL)] = {
 426         [C(OP_READ)] = {
 427                 [C(RESULT_ACCESS)]      = { 0x1c, CNTR_ODD, P },
 428                 [C(RESULT_MISS)]        = { 0x1d, CNTR_EVEN | CNTR_ODD, P },
 429         },
 430         [C(OP_WRITE)] = {
 431                 [C(RESULT_ACCESS)]      = { 0x1c, CNTR_ODD, P },
 432                 [C(RESULT_MISS)]        = { 0x1d, CNTR_EVEN | CNTR_ODD, P },
 433         },
 434         [C(OP_PREFETCH)] = {
 435                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 436                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 437         },
 438 },
 439 [C(DTLB)] = {
 440         /* 74K core does not have specific DTLB events. */
 441         [C(OP_READ)] = {
 442                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 443                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 444         },
 445         [C(OP_WRITE)] = {
 446                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 447                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 448         },
 449         [C(OP_PREFETCH)] = {
 450                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 451                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 452         },
 453 },
 454 [C(ITLB)] = {
 455         [C(OP_READ)] = {
 456                 [C(RESULT_ACCESS)]      = { 0x04, CNTR_EVEN, T },
 457                 [C(RESULT_MISS)]        = { 0x04, CNTR_ODD, T },
 458         },
 459         [C(OP_WRITE)] = {
 460                 [C(RESULT_ACCESS)]      = { 0x04, CNTR_EVEN, T },
 461                 [C(RESULT_MISS)]        = { 0x04, CNTR_ODD, T },
 462         },
 463         [C(OP_PREFETCH)] = {
 464                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 465                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 466         },
 467 },
 468 [C(BPU)] = {
 469         /* Using the same code for *HW_BRANCH* */
 470         [C(OP_READ)] = {
 471                 [C(RESULT_ACCESS)]      = { 0x27, CNTR_EVEN, T },
 472                 [C(RESULT_MISS)]        = { 0x27, CNTR_ODD, T },
 473         },
 474         [C(OP_WRITE)] = {
 475                 [C(RESULT_ACCESS)]      = { 0x27, CNTR_EVEN, T },
 476                 [C(RESULT_MISS)]        = { 0x27, CNTR_ODD, T },
 477         },
 478         [C(OP_PREFETCH)] = {
 479                 [C(RESULT_ACCESS)]      = { UNSUPPORTED_PERF_EVENT_ID },
 480                 [C(RESULT_MISS)]        = { UNSUPPORTED_PERF_EVENT_ID },
 481         },
 482 },
 483 };
 484
 485 #ifdef CONFIG_MIPS_MT_SMP
 486 static void
 487 check_and_calc_range(struct perf_event *event,
 488                         const struct mips_perf_event *pev)
 489 {
 490         struct hw_perf_event *hwc = &event->hw;
 491
 492         if (event->cpu >= 0) {
 493                 if (pev->range > V) {
 494                         /*
 495                          * The user selected an event that is processor
 496                          * wide, while expecting it to be VPE wide.
 497                          */
 498                         hwc->config_base |= M_TC_EN_ALL;
 499                 } else {
 500                         /*
 501                          * FIXME: cpu_data[event->cpu].vpe_id reports 0
 502                          * for both CPUs.
 503                          */
 504                         hwc->config_base |= M_PERFCTL_VPEID(event->cpu);
 505                         hwc->config_base |= M_TC_EN_VPE;
 506                 }
 507         } else
 508                 hwc->config_base |= M_TC_EN_ALL;
 509 }
 510 #else
 511 static void
 512 check_and_calc_range(struct perf_event *event,
 513                         const struct mips_perf_event *pev)
 514 {
 515 }
 516 #endif
 517
 518 static int __hw_perf_event_init(struct perf_event *event)
 519 {
 520         struct perf_event_attr *attr = &event->attr;
 521         struct hw_perf_event *hwc = &event->hw;
 522         const struct mips_perf_event *pev;
 523         int err;
 524
 525         /* Returning MIPS event descriptor for generic perf event. */
 526         if (PERF_TYPE_HARDWARE == event->attr.type) {
 527                 if (event->attr.config >= PERF_COUNT_HW_MAX)
 528                         return -EINVAL;
 529                 pev = mipspmu_map_general_event(event->attr.config);
 530         } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
 531                 pev = mipspmu_map_cache_event(event->attr.config);
 532         } else if (PERF_TYPE_RAW == event->attr.type) {
 533                 /* We are working on the global raw event. */
 534                 mutex_lock(&raw_event_mutex);
 535                 pev = mipspmu->map_raw_event(event->attr.config);
 536         } else {
 537                 /* The event type is not (yet) supported. */
 538                 return -EOPNOTSUPP;
 539         }
 540
 541         if (IS_ERR(pev)) {
 542                 if (PERF_TYPE_RAW == event->attr.type)
 543                         mutex_unlock(&raw_event_mutex);
 544                 return PTR_ERR(pev);
 545         }
 546
 547         /*
 548          * We allow max flexibility on how each individual counter shared
 549          * by the single CPU operates (the mode exclusion and the range).
 550          */
 551         hwc->config_base = M_PERFCTL_INTERRUPT_ENABLE;
 552
 553         /* Calculate range bits and validate it. */
 554         if (num_possible_cpus() > 1)
 555                 check_and_calc_range(event, pev);
 556
 557         hwc->event_base = mipspmu_perf_event_encode(pev);
 558         if (PERF_TYPE_RAW == event->attr.type)
 559                 mutex_unlock(&raw_event_mutex);
 560
 561         if (!attr->exclude_user)
 562                 hwc->config_base |= M_PERFCTL_USER;
 563         if (!attr->exclude_kernel) {
 564                 hwc->config_base |= M_PERFCTL_KERNEL;
 565                 /* MIPS kernel mode: KSU == 00b || EXL == 1 || ERL == 1 */
 566                 hwc->config_base |= M_PERFCTL_EXL;
 567         }
 568         if (!attr->exclude_hv)
 569                 hwc->config_base |= M_PERFCTL_SUPERVISOR;
 570
 571         hwc->config_base &= M_PERFCTL_CONFIG_MASK;
 572         /*
 573          * The event can belong to another cpu. We do not assign a local
 574          * counter for it for now.
 575          */
 576         hwc->idx = -1;
 577         hwc->config = 0;
 578
 579         if (!hwc->sample_period) {
 580                 hwc->sample_period  = MAX_PERIOD;
 581                 hwc->last_period    = hwc->sample_period;
 582                 local64_set(&hwc->period_left, hwc->sample_period);
 583         }
 584
 585         err = 0;
 586         if (event->group_leader != event) {
 587                 err = validate_group(event);
 588                 if (err)
 589                         return -EINVAL;
 590         }
 591
 592         event->destroy = hw_perf_event_destroy;
 593
 594         return err;
 595 }
 596
 597 static void pause_local_counters(void)
 598 {
 599         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 600         int counters = mipspmu->num_counters;
 601         unsigned long flags;
 602
 603         local_irq_save(flags);
 604         switch (counters) {
 605         case 4:
 606                 cpuc->saved_ctrl[3] = r_c0_perfctrl3();
 607                 w_c0_perfctrl3(cpuc->saved_ctrl[3] &
 608                         ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 609         case 3:
 610                 cpuc->saved_ctrl[2] = r_c0_perfctrl2();
 611                 w_c0_perfctrl2(cpuc->saved_ctrl[2] &
 612                         ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 613         case 2:
 614                 cpuc->saved_ctrl[1] = r_c0_perfctrl1();
 615                 w_c0_perfctrl1(cpuc->saved_ctrl[1] &
 616                         ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 617         case 1:
 618                 cpuc->saved_ctrl[0] = r_c0_perfctrl0();
 619                 w_c0_perfctrl0(cpuc->saved_ctrl[0] &
 620                         ~M_PERFCTL_COUNT_EVENT_WHENEVER);
 621         }
 622         local_irq_restore(flags);
 623 }
 624
 625 static void resume_local_counters(void)
 626 {
 627         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 628         int counters = mipspmu->num_counters;
 629         unsigned long flags;
 630
 631         local_irq_save(flags);
 632         switch (counters) {
 633         case 4:
 634                 w_c0_perfctrl3(cpuc->saved_ctrl[3]);
 635         case 3:
 636                 w_c0_perfctrl2(cpuc->saved_ctrl[2]);
 637         case 2:
 638                 w_c0_perfctrl1(cpuc->saved_ctrl[1]);
 639         case 1:
 640                 w_c0_perfctrl0(cpuc->saved_ctrl[0]);
 641         }
 642         local_irq_restore(flags);
 643 }
 644
 645 static int mipsxx_pmu_handle_shared_irq(void)
 646 {
 647         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 648         struct perf_sample_data data;
 649         unsigned int counters = mipspmu->num_counters;
 650         unsigned int counter;
 651         int handled = IRQ_NONE;
 652         struct pt_regs *regs;
 653
 654         if (cpu_has_mips_r2 && !(read_c0_cause() & (1 << 26)))
 655                 return handled;
 656
 657         /*
 658          * First we pause the local counters, so that when we are locked
 659          * here, the counters are all paused. When it gets locked due to
 660          * perf_disable(), the timer interrupt handler will be delayed.
 661          *
 662          * See also mipsxx_pmu_start().
 663          */
 664         pause_local_counters();
 665 #ifdef CONFIG_MIPS_MT_SMP
 666         read_lock(&pmuint_rwlock);
 667 #endif
 668
 669         regs = get_irq_regs();
 670
 671         perf_sample_data_init(&data, 0);
 672
 673         switch (counters) {
 674 #define HANDLE_COUNTER(n)                                               \
 675         case n + 1:                                                     \
 676                 if (test_bit(n, cpuc->used_mask)) {                     \
 677                         counter = r_c0_perfcntr ## n();                 \
 678                         if (counter & M_COUNTER_OVERFLOW) {             \
 679                                 w_c0_perfcntr ## n(counter &            \
 680                                                 VALID_COUNT);           \
 681                                 if (test_and_change_bit(n, cpuc->msbs)) \
 682                                         handle_associated_event(cpuc,   \
 683                                                 n, &data, regs);        \
 684                                 handled = IRQ_HANDLED;                  \
 685                         }                                               \
 686                 }
 687         HANDLE_COUNTER(3)
 688         HANDLE_COUNTER(2)
 689         HANDLE_COUNTER(1)
 690         HANDLE_COUNTER(0)
 691         }
 692
 693         /*
 694          * Do all the work for the pending perf events. We can do this
 695          * in here because the performance counter interrupt is a regular
 696          * interrupt, not NMI.
 697          */
 698         if (handled == IRQ_HANDLED)
 699                 irq_work_run();
 700
 701 #ifdef CONFIG_MIPS_MT_SMP
 702         read_unlock(&pmuint_rwlock);
 703 #endif
 704         resume_local_counters();
 705         return handled;
 706 }
 707
 708 static irqreturn_t
 709 mipsxx_pmu_handle_irq(int irq, void *dev)
 710 {
 711         return mipsxx_pmu_handle_shared_irq();
 712 }
 713
 714 static void mipsxx_pmu_start(void)
 715 {
 716 #ifdef CONFIG_MIPS_MT_SMP
 717         write_unlock(&pmuint_rwlock);
 718 #endif
 719         resume_local_counters();
 720 }
 721
 722 /*
 723  * MIPS performance counters can be per-TC. The control registers can
 724  * not be directly accessed across CPUs. Hence if we want to do global
 725  * control, we need cross CPU calls. on_each_cpu() can help us, but we
 726  * can not make sure this function is called with interrupts enabled. So
 727  * here we pause local counters and then grab a rwlock and leave the
 728  * counters on other CPUs alone. If any counter interrupt raises while
 729  * we own the write lock, simply pause local counters on that CPU and
 730  * spin in the handler. Also we know we won't be switched to another
 731  * CPU after pausing local counters and before grabbing the lock.
 732  */
 733 static void mipsxx_pmu_stop(void)
 734 {
 735         pause_local_counters();
 736 #ifdef CONFIG_MIPS_MT_SMP
 737         write_lock(&pmuint_rwlock);
 738 #endif
 739 }
 740
 741 static int
 742 mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc,
 743                         struct hw_perf_event *hwc)
 744 {
 745         int i;
 746
 747         /*
 748          * We only need to care the counter mask. The range has been
 749          * checked definitely.
 750          */
 751         unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff;
 752
 753         for (i = mipspmu->num_counters - 1; i >= 0; i--) {
 754                 /*
 755                  * Note that some MIPS perf events can be counted by both
 756                  * even and odd counters, wheresas many other are only by
 757                  * even _or_ odd counters. This introduces an issue that
 758                  * when the former kind of event takes the counter the
 759                  * latter kind of event wants to use, then the "counter
 760                  * allocation" for the latter event will fail. In fact if
 761                  * they can be dynamically swapped, they both feel happy.
 762                  * But here we leave this issue alone for now.
 763                  */
 764                 if (test_bit(i, &cntr_mask) &&
 765                         !test_and_set_bit(i, cpuc->used_mask))
 766                         return i;
 767         }
 768
 769         return -EAGAIN;
 770 }
 771
 772 static void
 773 mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx)
 774 {
 775         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 776         unsigned long flags;
 777
 778         WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
 779
 780         local_irq_save(flags);
 781         cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) |
 782                 (evt->config_base & M_PERFCTL_CONFIG_MASK) |
 783                 /* Make sure interrupt enabled. */
 784                 M_PERFCTL_INTERRUPT_ENABLE;
 785         /*
 786          * We do not actually let the counter run. Leave it until start().
 787          */
 788         local_irq_restore(flags);
 789 }
 790
 791 static void
 792 mipsxx_pmu_disable_event(int idx)
 793 {
 794         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 795         unsigned long flags;
 796
 797         WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
 798
 799         local_irq_save(flags);
 800         cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) &
 801                 ~M_PERFCTL_COUNT_EVENT_WHENEVER;
 802         mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]);
 803         local_irq_restore(flags);
 804 }
 805
 806 /* 24K */
 807 #define IS_UNSUPPORTED_24K_EVENT(r, b)                                  \
 808         ((b) == 12 || (r) == 151 || (r) == 152 || (b) == 26 ||          \
 809          (b) == 27 || (r) == 28 || (r) == 158 || (b) == 31 ||           \
 810          (b) == 32 || (b) == 34 || (b) == 36 || (r) == 168 ||           \
 811          (r) == 172 || (b) == 47 || ((b) >= 56 && (b) <= 63) ||         \
 812          ((b) >= 68 && (b) <= 127))
 813 #define IS_BOTH_COUNTERS_24K_EVENT(b)                                   \
 814         ((b) == 0 || (b) == 1 || (b) == 11)
 815
 816 /* 34K */
 817 #define IS_UNSUPPORTED_34K_EVENT(r, b)                                  \
 818         ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 36 ||           \
 819          (b) == 38 || (r) == 175 || ((b) >= 56 && (b) <= 63) ||         \
 820          ((b) >= 68 && (b) <= 127))
 821 #define IS_BOTH_COUNTERS_34K_EVENT(b)                                   \
 822         ((b) == 0 || (b) == 1 || (b) == 11)
 823 #ifdef CONFIG_MIPS_MT_SMP
 824 #define IS_RANGE_P_34K_EVENT(r, b)                                      \
 825         ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||             \
 826          (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 ||           \
 827          (r) == 176 || ((b) >= 50 && (b) <= 55) ||                      \
 828          ((b) >= 64 && (b) <= 67))
 829 #define IS_RANGE_V_34K_EVENT(r) ((r) == 47)
 830 #endif
 831
 832 /* 74K */
 833 #define IS_UNSUPPORTED_74K_EVENT(r, b)                                  \
 834         ((r) == 5 || ((r) >= 135 && (r) <= 137) ||                      \
 835          ((b) >= 10 && (b) <= 12) || (b) == 22 || (b) == 27 ||          \
 836          (b) == 33 || (b) == 34 || ((b) >= 47 && (b) <= 49) ||          \
 837          (r) == 178 || (b) == 55 || (b) == 57 || (b) == 60 ||           \
 838          (b) == 61 || (r) == 62 || (r) == 191 ||                        \
 839          ((b) >= 64 && (b) <= 127))
 840 #define IS_BOTH_COUNTERS_74K_EVENT(b)                                   \
 841         ((b) == 0 || (b) == 1)
 842
 843 /* 1004K */
 844 #define IS_UNSUPPORTED_1004K_EVENT(r, b)                                \
 845         ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 38 ||           \
 846          (r) == 175 || (b) == 63 || ((b) >= 68 && (b) <= 127))
 847 #define IS_BOTH_COUNTERS_1004K_EVENT(b)                                 \
 848         ((b) == 0 || (b) == 1 || (b) == 11)
 849 #ifdef CONFIG_MIPS_MT_SMP
 850 #define IS_RANGE_P_1004K_EVENT(r, b)                                    \
 851         ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 ||             \
 852          (b) == 25 || (b) == 36 || (b) == 39 || (r) == 44 ||            \
 853          (r) == 174 || (r) == 176 || ((b) >= 50 && (b) <= 59) ||        \
 854          (r) == 188 || (b) == 61 || (b) == 62 ||                        \
 855          ((b) >= 64 && (b) <= 67))
 856 #define IS_RANGE_V_1004K_EVENT(r)       ((r) == 47)
 857 #endif
 858
 859 /*
 860  * User can use 0-255 raw events, where 0-127 for the events of even
 861  * counters, and 128-255 for odd counters. Note that bit 7 is used to
 862  * indicate the parity. So, for example, when user wants to take the
 863  * Event Num of 15 for odd counters (by referring to the user manual),
 864  * then 128 needs to be added to 15 as the input for the event config,
 865  * i.e., 143 (0x8F) to be used.
 866  */
 867 static const struct mips_perf_event *
 868 mipsxx_pmu_map_raw_event(u64 config)
 869 {
 870         unsigned int raw_id = config & 0xff;
 871         unsigned int base_id = raw_id & 0x7f;
 872
 873         switch (current_cpu_type()) {
 874         case CPU_24K:
 875                 if (IS_UNSUPPORTED_24K_EVENT(raw_id, base_id))
 876                         return ERR_PTR(-EOPNOTSUPP);
 877                 raw_event.event_id = base_id;
 878                 if (IS_BOTH_COUNTERS_24K_EVENT(base_id))
 879                         raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 880                 else
 881                         raw_event.cntr_mask =
 882                                 raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 883 #ifdef CONFIG_MIPS_MT_SMP
 884                 /*
 885                  * This is actually doing nothing. Non-multithreading
 886                  * CPUs will not check and calculate the range.
 887                  */
 888                 raw_event.range = P;
 889 #endif
 890                 break;
 891         case CPU_34K:
 892                 if (IS_UNSUPPORTED_34K_EVENT(raw_id, base_id))
 893                         return ERR_PTR(-EOPNOTSUPP);
 894                 raw_event.event_id = base_id;
 895                 if (IS_BOTH_COUNTERS_34K_EVENT(base_id))
 896                         raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 897                 else
 898                         raw_event.cntr_mask =
 899                                 raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 900 #ifdef CONFIG_MIPS_MT_SMP
 901                 if (IS_RANGE_P_34K_EVENT(raw_id, base_id))
 902                         raw_event.range = P;
 903                 else if (unlikely(IS_RANGE_V_34K_EVENT(raw_id)))
 904                         raw_event.range = V;
 905                 else
 906                         raw_event.range = T;
 907 #endif
 908                 break;
 909         case CPU_74K:
 910                 if (IS_UNSUPPORTED_74K_EVENT(raw_id, base_id))
 911                         return ERR_PTR(-EOPNOTSUPP);
 912                 raw_event.event_id = base_id;
 913                 if (IS_BOTH_COUNTERS_74K_EVENT(base_id))
 914                         raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 915                 else
 916                         raw_event.cntr_mask =
 917                                 raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 918 #ifdef CONFIG_MIPS_MT_SMP
 919                 raw_event.range = P;
 920 #endif
 921                 break;
 922         case CPU_1004K:
 923                 if (IS_UNSUPPORTED_1004K_EVENT(raw_id, base_id))
 924                         return ERR_PTR(-EOPNOTSUPP);
 925                 raw_event.event_id = base_id;
 926                 if (IS_BOTH_COUNTERS_1004K_EVENT(base_id))
 927                         raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD;
 928                 else
 929                         raw_event.cntr_mask =
 930                                 raw_id > 127 ? CNTR_ODD : CNTR_EVEN;
 931 #ifdef CONFIG_MIPS_MT_SMP
 932                 if (IS_RANGE_P_1004K_EVENT(raw_id, base_id))
 933                         raw_event.range = P;
 934                 else if (unlikely(IS_RANGE_V_1004K_EVENT(raw_id)))
 935                         raw_event.range = V;
 936                 else
 937                         raw_event.range = T;
 938 #endif
 939                 break;
 940         }
 941
 942         return &raw_event;
 943 }
 944
 945 static struct mips_pmu mipsxxcore_pmu = {
 946         .handle_irq = mipsxx_pmu_handle_irq,
 947         .handle_shared_irq = mipsxx_pmu_handle_shared_irq,
 948         .start = mipsxx_pmu_start,
 949         .stop = mipsxx_pmu_stop,
 950         .alloc_counter = mipsxx_pmu_alloc_counter,
 951         .read_counter = mipsxx_pmu_read_counter,
 952         .write_counter = mipsxx_pmu_write_counter,
 953         .enable_event = mipsxx_pmu_enable_event,
 954         .disable_event = mipsxx_pmu_disable_event,
 955         .map_raw_event = mipsxx_pmu_map_raw_event,
 956         .general_event_map = &mipsxxcore_event_map,
 957         .cache_event_map = &mipsxxcore_cache_map,
 958 };
 959
 960 static struct mips_pmu mipsxx74Kcore_pmu = {
 961         .handle_irq = mipsxx_pmu_handle_irq,
 962         .handle_shared_irq = mipsxx_pmu_handle_shared_irq,
 963         .start = mipsxx_pmu_start,
 964         .stop = mipsxx_pmu_stop,
 965         .alloc_counter = mipsxx_pmu_alloc_counter,
 966         .read_counter = mipsxx_pmu_read_counter,
 967         .write_counter = mipsxx_pmu_write_counter,
 968         .enable_event = mipsxx_pmu_enable_event,
 969         .disable_event = mipsxx_pmu_disable_event,
 970         .map_raw_event = mipsxx_pmu_map_raw_event,
 971         .general_event_map = &mipsxx74Kcore_event_map,
 972         .cache_event_map = &mipsxx74Kcore_cache_map,
 973 };
 974
 975 static int __init
 976 init_hw_perf_events(void)
 977 {
 978         int counters, irq;
 979
 980         pr_info("Performance counters: ");
 981
 982         counters = n_counters();
 983         if (counters == 0) {
 984                 pr_cont("No available PMU.\n");
 985                 return -ENODEV;
 986         }
 987
 988 #ifdef CONFIG_MIPS_MT_SMP
 989         cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19);
 990         if (!cpu_has_mipsmt_pertccounters)
 991                 counters = counters_total_to_per_cpu(counters);
 992 #endif
 993
 994 #ifdef MSC01E_INT_BASE
 995         if (cpu_has_veic) {
 996                 /*
 997                  * Using platform specific interrupt controller defines.
 998                  */
 999                 irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR;
1000         } else {
1001 #endif
1002                 if (cp0_perfcount_irq >= 0)
1003                         irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq;
1004                 else
1005                         irq = -1;
1006 #ifdef MSC01E_INT_BASE
1007         }
1008 #endif
1009
1010         on_each_cpu(reset_counters, (void *)(long)counters, 1);
1011
1012         switch (current_cpu_type()) {
1013         case CPU_24K:
1014                 mipsxxcore_pmu.name = "mips/24K";
1015                 mipsxxcore_pmu.num_counters = counters;
1016                 mipsxxcore_pmu.irq = irq;
1017                 mipspmu = &mipsxxcore_pmu;
1018                 break;
1019         case CPU_34K:
1020                 mipsxxcore_pmu.name = "mips/34K";
1021                 mipsxxcore_pmu.num_counters = counters;
1022                 mipsxxcore_pmu.irq = irq;
1023                 mipspmu = &mipsxxcore_pmu;
1024                 break;
1025         case CPU_74K:
1026                 mipsxx74Kcore_pmu.name = "mips/74K";
1027                 mipsxx74Kcore_pmu.num_counters = counters;
1028                 mipsxx74Kcore_pmu.irq = irq;
1029                 mipspmu = &mipsxx74Kcore_pmu;
1030                 break;
1031         case CPU_1004K:
1032                 mipsxxcore_pmu.name = "mips/1004K";
1033                 mipsxxcore_pmu.num_counters = counters;
1034                 mipsxxcore_pmu.irq = irq;
1035                 mipspmu = &mipsxxcore_pmu;
1036                 break;
1037         default:
1038                 pr_cont("Either hardware does not support performance "
1039                         "counters, or not yet implemented.\n");
1040                 return -ENODEV;
1041         }
1042
1043         if (mipspmu)
1044                 pr_cont("%s PMU enabled, %d counters available to each "
1045                         "CPU, irq %d%s\n", mipspmu->name, counters, irq,
1046                         irq < 0 ? " (share with timer interrupt)" : "");
1047
1048         perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1049
1050         return 0;
1051 }
1052 early_initcall(init_hw_perf_events);
1053
1054 #endif /* defined(CONFIG_CPU_MIPS32)... */