arch/s390/oprofile/hwsampler.c

   1 /*
   2  * Copyright IBM Corp. 2010
   3  * Author: Heinz Graalfs <graalfs@de.ibm.com>
   4  */
   5
   6 #include <linux/kernel_stat.h>
   7 #include <linux/kernel.h>
   8 #include <linux/module.h>
   9 #include <linux/smp.h>
  10 #include <linux/errno.h>
  11 #include <linux/workqueue.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/notifier.h>
  14 #include <linux/cpu.h>
  15 #include <linux/semaphore.h>
  16 #include <linux/oom.h>
  17 #include <linux/oprofile.h>
  18
  19 #include <asm/facility.h>
  20 #include <asm/cpu_mf.h>
  21 #include <asm/irq.h>
  22
  23 #include "hwsampler.h"
  24 #include "op_counter.h"
  25
  26 #define MAX_NUM_SDB 511
  27 #define MIN_NUM_SDB 1
  28
  29 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  30
  31 struct hws_execute_parms {
  32         void *buffer;
  33         signed int rc;
  34 };
  35
  36 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
  37 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
  38
  39 static DEFINE_MUTEX(hws_sem);
  40 static DEFINE_MUTEX(hws_sem_oom);
  41
  42 static unsigned char hws_flush_all;
  43 static unsigned int hws_oom;
  44 static unsigned int hws_alert;
  45 static struct workqueue_struct *hws_wq;
  46
  47 static unsigned int hws_state;
  48 enum {
  49         HWS_INIT = 1,
  50         HWS_DEALLOCATED,
  51         HWS_STOPPED,
  52         HWS_STARTED,
  53         HWS_STOPPING };
  54
  55 /* set to 1 if called by kernel during memory allocation */
  56 static unsigned char oom_killer_was_active;
  57 /* size of SDBT and SDB as of allocate API */
  58 static unsigned long num_sdbt = 100;
  59 static unsigned long num_sdb = 511;
  60 /* sampling interval (machine cycles) */
  61 static unsigned long interval;
  62
  63 static unsigned long min_sampler_rate;
  64 static unsigned long max_sampler_rate;
  65
  66 static void execute_qsi(void *parms)
  67 {
  68         struct hws_execute_parms *ep = parms;
  69
  70         ep->rc = qsi(ep->buffer);
  71 }
  72
  73 static void execute_ssctl(void *parms)
  74 {
  75         struct hws_execute_parms *ep = parms;
  76
  77         ep->rc = lsctl(ep->buffer);
  78 }
  79
  80 static int smp_ctl_ssctl_stop(int cpu)
  81 {
  82         int rc;
  83         struct hws_execute_parms ep;
  84         struct hws_cpu_buffer *cb;
  85
  86         cb = &per_cpu(sampler_cpu_buffer, cpu);
  87
  88         cb->ssctl.es = 0;
  89         cb->ssctl.cs = 0;
  90
  91         ep.buffer = &cb->ssctl;
  92         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
  93         rc = ep.rc;
  94         if (rc) {
  95                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
  96                 dump_stack();
  97         }
  98
  99         ep.buffer = &cb->qsi;
 100         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 101
 102         if (cb->qsi.es || cb->qsi.cs) {
 103                 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
 104                 dump_stack();
 105         }
 106
 107         return rc;
 108 }
 109
 110 static int smp_ctl_ssctl_deactivate(int cpu)
 111 {
 112         int rc;
 113         struct hws_execute_parms ep;
 114         struct hws_cpu_buffer *cb;
 115
 116         cb = &per_cpu(sampler_cpu_buffer, cpu);
 117
 118         cb->ssctl.es = 1;
 119         cb->ssctl.cs = 0;
 120
 121         ep.buffer = &cb->ssctl;
 122         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 123         rc = ep.rc;
 124         if (rc)
 125                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 126
 127         ep.buffer = &cb->qsi;
 128         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 129
 130         if (cb->qsi.cs)
 131                 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
 132
 133         return rc;
 134 }
 135
 136 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
 137 {
 138         int rc;
 139         struct hws_execute_parms ep;
 140         struct hws_cpu_buffer *cb;
 141
 142         cb = &per_cpu(sampler_cpu_buffer, cpu);
 143
 144         cb->ssctl.h = 1;
 145         cb->ssctl.tear = cb->first_sdbt;
 146         cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
 147         cb->ssctl.interval = interval;
 148         cb->ssctl.es = 1;
 149         cb->ssctl.cs = 1;
 150
 151         ep.buffer = &cb->ssctl;
 152         smp_call_function_single(cpu, execute_ssctl, &ep, 1);
 153         rc = ep.rc;
 154         if (rc)
 155                 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
 156
 157         ep.buffer = &cb->qsi;
 158         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 159         if (ep.rc)
 160                 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
 161
 162         return rc;
 163 }
 164
 165 static int smp_ctl_qsi(int cpu)
 166 {
 167         struct hws_execute_parms ep;
 168         struct hws_cpu_buffer *cb;
 169
 170         cb = &per_cpu(sampler_cpu_buffer, cpu);
 171
 172         ep.buffer = &cb->qsi;
 173         smp_call_function_single(cpu, execute_qsi, &ep, 1);
 174
 175         return ep.rc;
 176 }
 177
 178 static void hws_ext_handler(struct ext_code ext_code,
 179                             unsigned int param32, unsigned long param64)
 180 {
 181         struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
 182
 183         if (!(param32 & CPU_MF_INT_SF_MASK))
 184                 return;
 185
 186         if (!hws_alert)
 187                 return;
 188
 189         inc_irq_stat(IRQEXT_CMS);
 190         atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
 191
 192         if (hws_wq)
 193                 queue_work(hws_wq, &cb->worker);
 194 }
 195
 196 static void worker(struct work_struct *work);
 197
 198 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
 199                                 unsigned long *dear);
 200
 201 static void init_all_cpu_buffers(void)
 202 {
 203         int cpu;
 204         struct hws_cpu_buffer *cb;
 205
 206         for_each_online_cpu(cpu) {
 207                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 208                 memset(cb, 0, sizeof(struct hws_cpu_buffer));
 209         }
 210 }
 211
 212 static int prepare_cpu_buffers(void)
 213 {
 214         int cpu;
 215         int rc;
 216         struct hws_cpu_buffer *cb;
 217
 218         rc = 0;
 219         for_each_online_cpu(cpu) {
 220                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 221                 atomic_set(&cb->ext_params, 0);
 222                 cb->worker_entry = 0;
 223                 cb->sample_overflow = 0;
 224                 cb->req_alert = 0;
 225                 cb->incorrect_sdbt_entry = 0;
 226                 cb->invalid_entry_address = 0;
 227                 cb->loss_of_sample_data = 0;
 228                 cb->sample_auth_change_alert = 0;
 229                 cb->finish = 0;
 230                 cb->oom = 0;
 231                 cb->stop_mode = 0;
 232         }
 233
 234         return rc;
 235 }
 236
 237 /*
 238  * allocate_sdbt() - allocate sampler memory
 239  * @cpu: the cpu for which sampler memory is allocated
 240  *
 241  * A 4K page is allocated for each requested SDBT.
 242  * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
 243  * Set ALERT_REQ mask in each SDBs trailer.
 244  * Returns zero if successful, <0 otherwise.
 245  */
 246 static int allocate_sdbt(int cpu)
 247 {
 248         int j, k, rc;
 249         unsigned long *sdbt;
 250         unsigned long  sdb;
 251         unsigned long *tail;
 252         unsigned long *trailer;
 253         struct hws_cpu_buffer *cb;
 254
 255         cb = &per_cpu(sampler_cpu_buffer, cpu);
 256
 257         if (cb->first_sdbt)
 258                 return -EINVAL;
 259
 260         sdbt = NULL;
 261         tail = sdbt;
 262
 263         for (j = 0; j < num_sdbt; j++) {
 264                 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 265
 266                 mutex_lock(&hws_sem_oom);
 267                 /* OOM killer might have been activated */
 268                 barrier();
 269                 if (oom_killer_was_active || !sdbt) {
 270                         if (sdbt)
 271                                 free_page((unsigned long)sdbt);
 272
 273                         goto allocate_sdbt_error;
 274                 }
 275                 if (cb->first_sdbt == 0)
 276                         cb->first_sdbt = (unsigned long)sdbt;
 277
 278                 /* link current page to tail of chain */
 279                 if (tail)
 280                         *tail = (unsigned long)(void *)sdbt + 1;
 281
 282                 mutex_unlock(&hws_sem_oom);
 283
 284                 for (k = 0; k < num_sdb; k++) {
 285                         /* get and set SDB page */
 286                         sdb = get_zeroed_page(GFP_KERNEL);
 287
 288                         mutex_lock(&hws_sem_oom);
 289                         /* OOM killer might have been activated */
 290                         barrier();
 291                         if (oom_killer_was_active || !sdb) {
 292                                 if (sdb)
 293                                         free_page(sdb);
 294
 295                                 goto allocate_sdbt_error;
 296                         }
 297                         *sdbt = sdb;
 298                         trailer = trailer_entry_ptr(*sdbt);
 299                         *trailer = SDB_TE_ALERT_REQ_MASK;
 300                         sdbt++;
 301                         mutex_unlock(&hws_sem_oom);
 302                 }
 303                 tail = sdbt;
 304         }
 305         mutex_lock(&hws_sem_oom);
 306         if (oom_killer_was_active)
 307                 goto allocate_sdbt_error;
 308
 309         rc = 0;
 310         if (tail)
 311                 *tail = (unsigned long)
 312                         ((void *)cb->first_sdbt) + 1;
 313
 314 allocate_sdbt_exit:
 315         mutex_unlock(&hws_sem_oom);
 316         return rc;
 317
 318 allocate_sdbt_error:
 319         rc = -ENOMEM;
 320         goto allocate_sdbt_exit;
 321 }
 322
 323 /*
 324  * deallocate_sdbt() - deallocate all sampler memory
 325  *
 326  * For each online CPU all SDBT trees are deallocated.
 327  * Returns the number of freed pages.
 328  */
 329 static int deallocate_sdbt(void)
 330 {
 331         int cpu;
 332         int counter;
 333
 334         counter = 0;
 335
 336         for_each_online_cpu(cpu) {
 337                 unsigned long start;
 338                 unsigned long sdbt;
 339                 unsigned long *curr;
 340                 struct hws_cpu_buffer *cb;
 341
 342                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 343
 344                 if (!cb->first_sdbt)
 345                         continue;
 346
 347                 sdbt = cb->first_sdbt;
 348                 curr = (unsigned long *) sdbt;
 349                 start = sdbt;
 350
 351                 /* we'll free the SDBT after all SDBs are processed... */
 352                 while (1) {
 353                         if (!*curr || !sdbt)
 354                                 break;
 355
 356                         /* watch for link entry reset if found */
 357                         if (is_link_entry(curr)) {
 358                                 curr = get_next_sdbt(curr);
 359                                 if (sdbt)
 360                                         free_page(sdbt);
 361
 362                                 /* we are done if we reach the start */
 363                                 if ((unsigned long) curr == start)
 364                                         break;
 365                                 else
 366                                         sdbt = (unsigned long) curr;
 367                         } else {
 368                                 /* process SDB pointer */
 369                                 if (*curr) {
 370                                         free_page(*curr);
 371                                         curr++;
 372                                 }
 373                         }
 374                         counter++;
 375                 }
 376                 cb->first_sdbt = 0;
 377         }
 378         return counter;
 379 }
 380
 381 static int start_sampling(int cpu)
 382 {
 383         int rc;
 384         struct hws_cpu_buffer *cb;
 385
 386         cb = &per_cpu(sampler_cpu_buffer, cpu);
 387         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 388         if (rc) {
 389                 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
 390                 goto start_exit;
 391         }
 392
 393         rc = -EINVAL;
 394         if (!cb->qsi.es) {
 395                 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
 396                 goto start_exit;
 397         }
 398
 399         if (!cb->qsi.cs) {
 400                 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
 401                 goto start_exit;
 402         }
 403
 404         printk(KERN_INFO
 405                 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
 406                 cpu, interval);
 407
 408         rc = 0;
 409
 410 start_exit:
 411         return rc;
 412 }
 413
 414 static int stop_sampling(int cpu)
 415 {
 416         unsigned long v;
 417         int rc;
 418         struct hws_cpu_buffer *cb;
 419
 420         rc = smp_ctl_qsi(cpu);
 421         WARN_ON(rc);
 422
 423         cb = &per_cpu(sampler_cpu_buffer, cpu);
 424         if (!rc && !cb->qsi.es)
 425                 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
 426
 427         rc = smp_ctl_ssctl_stop(cpu);
 428         if (rc) {
 429                 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
 430                                 cpu, rc);
 431                 goto stop_exit;
 432         }
 433
 434         printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
 435
 436 stop_exit:
 437         v = cb->req_alert;
 438         if (v)
 439                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
 440                                 " count=%lu.\n", cpu, v);
 441
 442         v = cb->loss_of_sample_data;
 443         if (v)
 444                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
 445                                 " count=%lu.\n", cpu, v);
 446
 447         v = cb->invalid_entry_address;
 448         if (v)
 449                 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
 450                                 " count=%lu.\n", cpu, v);
 451
 452         v = cb->incorrect_sdbt_entry;
 453         if (v)
 454                 printk(KERN_ERR
 455                                 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
 456                                 " count=%lu.\n", cpu, v);
 457
 458         v = cb->sample_auth_change_alert;
 459         if (v)
 460                 printk(KERN_ERR
 461                                 "hwsampler: CPU %d CPUMF Sample authorization change,"
 462                                 " count=%lu.\n", cpu, v);
 463
 464         return rc;
 465 }
 466
 467 static int check_hardware_prerequisites(void)
 468 {
 469         if (!test_facility(68))
 470                 return -EOPNOTSUPP;
 471         return 0;
 472 }
 473 /*
 474  * hws_oom_callback() - the OOM callback function
 475  *
 476  * In case the callback is invoked during memory allocation for the
 477  *  hw sampler, all obtained memory is deallocated and a flag is set
 478  *  so main sampler memory allocation can exit with a failure code.
 479  * In case the callback is invoked during sampling the hw sampler
 480  *  is deactivated for all CPUs.
 481  */
 482 static int hws_oom_callback(struct notifier_block *nfb,
 483         unsigned long dummy, void *parm)
 484 {
 485         unsigned long *freed;
 486         int cpu;
 487         struct hws_cpu_buffer *cb;
 488
 489         freed = parm;
 490
 491         mutex_lock(&hws_sem_oom);
 492
 493         if (hws_state == HWS_DEALLOCATED) {
 494                 /* during memory allocation */
 495                 if (oom_killer_was_active == 0) {
 496                         oom_killer_was_active = 1;
 497                         *freed += deallocate_sdbt();
 498                 }
 499         } else {
 500                 int i;
 501                 cpu = get_cpu();
 502                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 503
 504                 if (!cb->oom) {
 505                         for_each_online_cpu(i) {
 506                                 smp_ctl_ssctl_deactivate(i);
 507                                 cb->oom = 1;
 508                         }
 509                         cb->finish = 1;
 510
 511                         printk(KERN_INFO
 512                                 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
 513                                 cpu);
 514                 }
 515         }
 516
 517         mutex_unlock(&hws_sem_oom);
 518
 519         return NOTIFY_OK;
 520 }
 521
 522 static struct notifier_block hws_oom_notifier = {
 523         .notifier_call = hws_oom_callback
 524 };
 525
 526 static int hws_cpu_callback(struct notifier_block *nfb,
 527         unsigned long action, void *hcpu)
 528 {
 529         /* We do not have sampler space available for all possible CPUs.
 530            All CPUs should be online when hw sampling is activated. */
 531         return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
 532 }
 533
 534 static struct notifier_block hws_cpu_notifier = {
 535         .notifier_call = hws_cpu_callback
 536 };
 537
 538 /**
 539  * hwsampler_deactivate() - set hardware sampling temporarily inactive
 540  * @cpu:  specifies the CPU to be set inactive.
 541  *
 542  * Returns 0 on success, !0 on failure.
 543  */
 544 int hwsampler_deactivate(unsigned int cpu)
 545 {
 546         /*
 547          * Deactivate hw sampling temporarily and flush the buffer
 548          * by pushing all the pending samples to oprofile buffer.
 549          *
 550          * This function can be called under one of the following conditions:
 551          *     Memory unmap, task is exiting.
 552          */
 553         int rc;
 554         struct hws_cpu_buffer *cb;
 555
 556         rc = 0;
 557         mutex_lock(&hws_sem);
 558
 559         cb = &per_cpu(sampler_cpu_buffer, cpu);
 560         if (hws_state == HWS_STARTED) {
 561                 rc = smp_ctl_qsi(cpu);
 562                 WARN_ON(rc);
 563                 if (cb->qsi.cs) {
 564                         rc = smp_ctl_ssctl_deactivate(cpu);
 565                         if (rc) {
 566                                 printk(KERN_INFO
 567                                 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
 568                                 cb->finish = 1;
 569                                 hws_state = HWS_STOPPING;
 570                         } else  {
 571                                 hws_flush_all = 1;
 572                                 /* Add work to queue to read pending samples.*/
 573                                 queue_work_on(cpu, hws_wq, &cb->worker);
 574                         }
 575                 }
 576         }
 577         mutex_unlock(&hws_sem);
 578
 579         if (hws_wq)
 580                 flush_workqueue(hws_wq);
 581
 582         return rc;
 583 }
 584
 585 /**
 586  * hwsampler_activate() - activate/resume hardware sampling which was deactivated
 587  * @cpu:  specifies the CPU to be set active.
 588  *
 589  * Returns 0 on success, !0 on failure.
 590  */
 591 int hwsampler_activate(unsigned int cpu)
 592 {
 593         /*
 594          * Re-activate hw sampling. This should be called in pair with
 595          * hwsampler_deactivate().
 596          */
 597         int rc;
 598         struct hws_cpu_buffer *cb;
 599
 600         rc = 0;
 601         mutex_lock(&hws_sem);
 602
 603         cb = &per_cpu(sampler_cpu_buffer, cpu);
 604         if (hws_state == HWS_STARTED) {
 605                 rc = smp_ctl_qsi(cpu);
 606                 WARN_ON(rc);
 607                 if (!cb->qsi.cs) {
 608                         hws_flush_all = 0;
 609                         rc = smp_ctl_ssctl_enable_activate(cpu, interval);
 610                         if (rc) {
 611                                 printk(KERN_ERR
 612                                 "CPU %d, CPUMF activate sampling failed.\n",
 613                                          cpu);
 614                         }
 615                 }
 616         }
 617
 618         mutex_unlock(&hws_sem);
 619
 620         return rc;
 621 }
 622
 623 static int check_qsi_on_setup(void)
 624 {
 625         int rc;
 626         unsigned int cpu;
 627         struct hws_cpu_buffer *cb;
 628
 629         for_each_online_cpu(cpu) {
 630                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 631                 rc = smp_ctl_qsi(cpu);
 632                 WARN_ON(rc);
 633                 if (rc)
 634                         return -EOPNOTSUPP;
 635
 636                 if (!cb->qsi.as) {
 637                         printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
 638                         return -EINVAL;
 639                 }
 640
 641                 if (cb->qsi.es) {
 642                         printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
 643                         rc = smp_ctl_ssctl_stop(cpu);
 644                         if (rc)
 645                                 return -EINVAL;
 646
 647                         printk(KERN_INFO
 648                                 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
 649                 }
 650         }
 651         return 0;
 652 }
 653
 654 static int check_qsi_on_start(void)
 655 {
 656         unsigned int cpu;
 657         int rc;
 658         struct hws_cpu_buffer *cb;
 659
 660         for_each_online_cpu(cpu) {
 661                 cb = &per_cpu(sampler_cpu_buffer, cpu);
 662                 rc = smp_ctl_qsi(cpu);
 663                 WARN_ON(rc);
 664
 665                 if (!cb->qsi.as)
 666                         return -EINVAL;
 667
 668                 if (cb->qsi.es)
 669                         return -EINVAL;
 670
 671                 if (cb->qsi.cs)
 672                         return -EINVAL;
 673         }
 674         return 0;
 675 }
 676
 677 static void worker_on_start(unsigned int cpu)
 678 {
 679         struct hws_cpu_buffer *cb;
 680
 681         cb = &per_cpu(sampler_cpu_buffer, cpu);
 682         cb->worker_entry = cb->first_sdbt;
 683 }
 684
 685 static int worker_check_error(unsigned int cpu, int ext_params)
 686 {
 687         int rc;
 688         unsigned long *sdbt;
 689         struct hws_cpu_buffer *cb;
 690
 691         rc = 0;
 692         cb = &per_cpu(sampler_cpu_buffer, cpu);
 693         sdbt = (unsigned long *) cb->worker_entry;
 694
 695         if (!sdbt || !*sdbt)
 696                 return -EINVAL;
 697
 698         if (ext_params & CPU_MF_INT_SF_PRA)
 699                 cb->req_alert++;
 700
 701         if (ext_params & CPU_MF_INT_SF_LSDA)
 702                 cb->loss_of_sample_data++;
 703
 704         if (ext_params & CPU_MF_INT_SF_IAE) {
 705                 cb->invalid_entry_address++;
 706                 rc = -EINVAL;
 707         }
 708
 709         if (ext_params & CPU_MF_INT_SF_ISE) {
 710                 cb->incorrect_sdbt_entry++;
 711                 rc = -EINVAL;
 712         }
 713
 714         if (ext_params & CPU_MF_INT_SF_SACA) {
 715                 cb->sample_auth_change_alert++;
 716                 rc = -EINVAL;
 717         }
 718
 719         return rc;
 720 }
 721
 722 static void worker_on_finish(unsigned int cpu)
 723 {
 724         int rc, i;
 725         struct hws_cpu_buffer *cb;
 726
 727         cb = &per_cpu(sampler_cpu_buffer, cpu);
 728
 729         if (cb->finish) {
 730                 rc = smp_ctl_qsi(cpu);
 731                 WARN_ON(rc);
 732                 if (cb->qsi.es) {
 733                         printk(KERN_INFO
 734                                 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
 735                                 cpu);
 736                         rc = smp_ctl_ssctl_stop(cpu);
 737                         if (rc)
 738                                 printk(KERN_INFO
 739                                         "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
 740                                         cpu);
 741
 742                         for_each_online_cpu(i) {
 743                                 if (i == cpu)
 744                                         continue;
 745                                 if (!cb->finish) {
 746                                         cb->finish = 1;
 747                                         queue_work_on(i, hws_wq,
 748                                                 &cb->worker);
 749                                 }
 750                         }
 751                 }
 752         }
 753 }
 754
 755 static void worker_on_interrupt(unsigned int cpu)
 756 {
 757         unsigned long *sdbt;
 758         unsigned char done;
 759         struct hws_cpu_buffer *cb;
 760
 761         cb = &per_cpu(sampler_cpu_buffer, cpu);
 762
 763         sdbt = (unsigned long *) cb->worker_entry;
 764
 765         done = 0;
 766         /* do not proceed if stop was entered,
 767          * forget the buffers not yet processed */
 768         while (!done && !cb->stop_mode) {
 769                 unsigned long *trailer;
 770                 struct hws_trailer_entry *te;
 771                 unsigned long *dear = 0;
 772
 773                 trailer = trailer_entry_ptr(*sdbt);
 774                 /* leave loop if no more work to do */
 775                 if (!(*trailer & SDB_TE_BUFFER_FULL_MASK)) {
 776                         done = 1;
 777                         if (!hws_flush_all)
 778                                 continue;
 779                 }
 780
 781                 te = (struct hws_trailer_entry *)trailer;
 782                 cb->sample_overflow += te->overflow;
 783
 784                 add_samples_to_oprofile(cpu, sdbt, dear);
 785
 786                 /* reset trailer */
 787                 xchg((unsigned char *) te, 0x40);
 788
 789                 /* advance to next sdb slot in current sdbt */
 790                 sdbt++;
 791                 /* in case link bit is set use address w/o link bit */
 792                 if (is_link_entry(sdbt))
 793                         sdbt = get_next_sdbt(sdbt);
 794
 795                 cb->worker_entry = (unsigned long)sdbt;
 796         }
 797 }
 798
 799 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
 800                 unsigned long *dear)
 801 {
 802         struct hws_basic_entry *sample_data_ptr;
 803         unsigned long *trailer;
 804
 805         trailer = trailer_entry_ptr(*sdbt);
 806         if (dear) {
 807                 if (dear > trailer)
 808                         return;
 809                 trailer = dear;
 810         }
 811
 812         sample_data_ptr = (struct hws_basic_entry *)(*sdbt);
 813
 814         while ((unsigned long *)sample_data_ptr < trailer) {
 815                 struct pt_regs *regs = NULL;
 816                 struct task_struct *tsk = NULL;
 817
 818                 /*
 819                  * Check sampling mode, 1 indicates basic (=customer) sampling
 820                  * mode.
 821                  */
 822                 if (sample_data_ptr->def != 1) {
 823                         /* sample slot is not yet written */
 824                         break;
 825                 } else {
 826                         /* make sure we don't use it twice,
 827                          * the next time the sampler will set it again */
 828                         sample_data_ptr->def = 0;
 829                 }
 830
 831                 /* Get pt_regs. */
 832                 if (sample_data_ptr->P == 1) {
 833                         /* userspace sample */
 834                         unsigned int pid = sample_data_ptr->prim_asn;
 835                         if (!counter_config.user)
 836                                 goto skip_sample;
 837                         rcu_read_lock();
 838                         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 839                         if (tsk)
 840                                 regs = task_pt_regs(tsk);
 841                         rcu_read_unlock();
 842                 } else {
 843                         /* kernelspace sample */
 844                         if (!counter_config.kernel)
 845                                 goto skip_sample;
 846                         regs = task_pt_regs(current);
 847                 }
 848
 849                 mutex_lock(&hws_sem);
 850                 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
 851                                 !sample_data_ptr->P, tsk);
 852                 mutex_unlock(&hws_sem);
 853         skip_sample:
 854                 sample_data_ptr++;
 855         }
 856 }
 857
 858 static void worker(struct work_struct *work)
 859 {
 860         unsigned int cpu;
 861         int ext_params;
 862         struct hws_cpu_buffer *cb;
 863
 864         cb = container_of(work, struct hws_cpu_buffer, worker);
 865         cpu = smp_processor_id();
 866         ext_params = atomic_xchg(&cb->ext_params, 0);
 867
 868         if (!cb->worker_entry)
 869                 worker_on_start(cpu);
 870
 871         if (worker_check_error(cpu, ext_params))
 872                 return;
 873
 874         if (!cb->finish)
 875                 worker_on_interrupt(cpu);
 876
 877         if (cb->finish)
 878                 worker_on_finish(cpu);
 879 }
 880
 881 /**
 882  * hwsampler_allocate() - allocate memory for the hardware sampler
 883  * @sdbt:  number of SDBTs per online CPU (must be > 0)
 884  * @sdb:   number of SDBs per SDBT (minimum 1, maximum 511)
 885  *
 886  * Returns 0 on success, !0 on failure.
 887  */
 888 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
 889 {
 890         int cpu, rc;
 891         mutex_lock(&hws_sem);
 892
 893         rc = -EINVAL;
 894         if (hws_state != HWS_DEALLOCATED)
 895                 goto allocate_exit;
 896
 897         if (sdbt < 1)
 898                 goto allocate_exit;
 899
 900         if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
 901                 goto allocate_exit;
 902
 903         num_sdbt = sdbt;
 904         num_sdb = sdb;
 905
 906         oom_killer_was_active = 0;
 907         register_oom_notifier(&hws_oom_notifier);
 908
 909         for_each_online_cpu(cpu) {
 910                 if (allocate_sdbt(cpu)) {
 911                         unregister_oom_notifier(&hws_oom_notifier);
 912                         goto allocate_error;
 913                 }
 914         }
 915         unregister_oom_notifier(&hws_oom_notifier);
 916         if (oom_killer_was_active)
 917                 goto allocate_error;
 918
 919         hws_state = HWS_STOPPED;
 920         rc = 0;
 921
 922 allocate_exit:
 923         mutex_unlock(&hws_sem);
 924         return rc;
 925
 926 allocate_error:
 927         rc = -ENOMEM;
 928         printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
 929         goto allocate_exit;
 930 }
 931
 932 /**
 933  * hwsampler_deallocate() - deallocate hardware sampler memory
 934  *
 935  * Returns 0 on success, !0 on failure.
 936  */
 937 int hwsampler_deallocate(void)
 938 {
 939         int rc;
 940
 941         mutex_lock(&hws_sem);
 942
 943         rc = -EINVAL;
 944         if (hws_state != HWS_STOPPED)
 945                 goto deallocate_exit;
 946
 947         irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
 948         hws_alert = 0;
 949         deallocate_sdbt();
 950
 951         hws_state = HWS_DEALLOCATED;
 952         rc = 0;
 953
 954 deallocate_exit:
 955         mutex_unlock(&hws_sem);
 956
 957         return rc;
 958 }
 959
 960 unsigned long hwsampler_query_min_interval(void)
 961 {
 962         return min_sampler_rate;
 963 }
 964
 965 unsigned long hwsampler_query_max_interval(void)
 966 {
 967         return max_sampler_rate;
 968 }
 969
 970 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
 971 {
 972         struct hws_cpu_buffer *cb;
 973
 974         cb = &per_cpu(sampler_cpu_buffer, cpu);
 975
 976         return cb->sample_overflow;
 977 }
 978
 979 int hwsampler_setup(void)
 980 {
 981         int rc;
 982         int cpu;
 983         struct hws_cpu_buffer *cb;
 984
 985         mutex_lock(&hws_sem);
 986
 987         rc = -EINVAL;
 988         if (hws_state)
 989                 goto setup_exit;
 990
 991         hws_state = HWS_INIT;
 992
 993         init_all_cpu_buffers();
 994
 995         rc = check_hardware_prerequisites();
 996         if (rc)
 997                 goto setup_exit;
 998
 999         rc = check_qsi_on_setup();
1000         if (rc)
1001                 goto setup_exit;
1002
1003         rc = -EINVAL;
1004         hws_wq = create_workqueue("hwsampler");
1005         if (!hws_wq)
1006                 goto setup_exit;
1007
1008         register_cpu_notifier(&hws_cpu_notifier);
1009
1010         for_each_online_cpu(cpu) {
1011                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1012                 INIT_WORK(&cb->worker, worker);
1013                 rc = smp_ctl_qsi(cpu);
1014                 WARN_ON(rc);
1015                 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1016                         if (min_sampler_rate) {
1017                                 printk(KERN_WARNING
1018                                         "hwsampler: different min sampler rate values.\n");
1019                                 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1020                                         min_sampler_rate =
1021                                                 cb->qsi.min_sampl_rate;
1022                         } else
1023                                 min_sampler_rate = cb->qsi.min_sampl_rate;
1024                 }
1025                 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1026                         if (max_sampler_rate) {
1027                                 printk(KERN_WARNING
1028                                         "hwsampler: different max sampler rate values.\n");
1029                                 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1030                                         max_sampler_rate =
1031                                                 cb->qsi.max_sampl_rate;
1032                         } else
1033                                 max_sampler_rate = cb->qsi.max_sampl_rate;
1034                 }
1035         }
1036         register_external_interrupt(0x1407, hws_ext_handler);
1037
1038         hws_state = HWS_DEALLOCATED;
1039         rc = 0;
1040
1041 setup_exit:
1042         mutex_unlock(&hws_sem);
1043         return rc;
1044 }
1045
1046 int hwsampler_shutdown(void)
1047 {
1048         int rc;
1049
1050         mutex_lock(&hws_sem);
1051
1052         rc = -EINVAL;
1053         if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1054                 mutex_unlock(&hws_sem);
1055
1056                 if (hws_wq)
1057                         flush_workqueue(hws_wq);
1058
1059                 mutex_lock(&hws_sem);
1060
1061                 if (hws_state == HWS_STOPPED) {
1062                         irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
1063                         hws_alert = 0;
1064                         deallocate_sdbt();
1065                 }
1066                 if (hws_wq) {
1067                         destroy_workqueue(hws_wq);
1068                         hws_wq = NULL;
1069                 }
1070
1071                 unregister_external_interrupt(0x1407, hws_ext_handler);
1072                 hws_state = HWS_INIT;
1073                 rc = 0;
1074         }
1075         mutex_unlock(&hws_sem);
1076
1077         unregister_cpu_notifier(&hws_cpu_notifier);
1078
1079         return rc;
1080 }
1081
1082 /**
1083  * hwsampler_start_all() - start hardware sampling on all online CPUs
1084  * @rate:  specifies the used interval when samples are taken
1085  *
1086  * Returns 0 on success, !0 on failure.
1087  */
1088 int hwsampler_start_all(unsigned long rate)
1089 {
1090         int rc, cpu;
1091
1092         mutex_lock(&hws_sem);
1093
1094         hws_oom = 0;
1095
1096         rc = -EINVAL;
1097         if (hws_state != HWS_STOPPED)
1098                 goto start_all_exit;
1099
1100         interval = rate;
1101
1102         /* fail if rate is not valid */
1103         if (interval < min_sampler_rate || interval > max_sampler_rate)
1104                 goto start_all_exit;
1105
1106         rc = check_qsi_on_start();
1107         if (rc)
1108                 goto start_all_exit;
1109
1110         rc = prepare_cpu_buffers();
1111         if (rc)
1112                 goto start_all_exit;
1113
1114         for_each_online_cpu(cpu) {
1115                 rc = start_sampling(cpu);
1116                 if (rc)
1117                         break;
1118         }
1119         if (rc) {
1120                 for_each_online_cpu(cpu) {
1121                         stop_sampling(cpu);
1122                 }
1123                 goto start_all_exit;
1124         }
1125         hws_state = HWS_STARTED;
1126         rc = 0;
1127
1128 start_all_exit:
1129         mutex_unlock(&hws_sem);
1130
1131         if (rc)
1132                 return rc;
1133
1134         register_oom_notifier(&hws_oom_notifier);
1135         hws_oom = 1;
1136         hws_flush_all = 0;
1137         /* now let them in, 1407 CPUMF external interrupts */
1138         hws_alert = 1;
1139         irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
1140
1141         return 0;
1142 }
1143
1144 /**
1145  * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1146  *
1147  * Returns 0 on success, !0 on failure.
1148  */
1149 int hwsampler_stop_all(void)
1150 {
1151         int tmp_rc, rc, cpu;
1152         struct hws_cpu_buffer *cb;
1153
1154         mutex_lock(&hws_sem);
1155
1156         rc = 0;
1157         if (hws_state == HWS_INIT) {
1158                 mutex_unlock(&hws_sem);
1159                 return rc;
1160         }
1161         hws_state = HWS_STOPPING;
1162         mutex_unlock(&hws_sem);
1163
1164         for_each_online_cpu(cpu) {
1165                 cb = &per_cpu(sampler_cpu_buffer, cpu);
1166                 cb->stop_mode = 1;
1167                 tmp_rc = stop_sampling(cpu);
1168                 if (tmp_rc)
1169                         rc = tmp_rc;
1170         }
1171
1172         if (hws_wq)
1173                 flush_workqueue(hws_wq);
1174
1175         mutex_lock(&hws_sem);
1176         if (hws_oom) {
1177                 unregister_oom_notifier(&hws_oom_notifier);
1178                 hws_oom = 0;
1179         }
1180         hws_state = HWS_STOPPED;
1181         mutex_unlock(&hws_sem);
1182
1183         return rc;
1184 }