?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 3.11-rc3
[cris-mirror.git] / arch / s390 / oprofile / hwsampler.c
blobb5b2916895e08dd8b562fb7784628c4a0294f55b
1 /*
2 * Copyright IBM Corp. 2010
3 * Author: Heinz Graalfs <graalfs@de.ibm.com>
4 */
5
6 #include <linux/kernel_stat.h>
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/smp.h>
10 #include <linux/errno.h>
11 #include <linux/workqueue.h>
12 #include <linux/interrupt.h>
13 #include <linux/notifier.h>
14 #include <linux/cpu.h>
15 #include <linux/semaphore.h>
16 #include <linux/oom.h>
17 #include <linux/oprofile.h>
18
19 #include <asm/facility.h>
20 #include <asm/cpu_mf.h>
21 #include <asm/irq.h>
22
23 #include "hwsampler.h"
24 #include "op_counter.h"
25
26 #define MAX_NUM_SDB 511
27 #define MIN_NUM_SDB 1
28
29 #define ALERT_REQ_MASK 0x4000000000000000ul
30 #define BUFFER_FULL_MASK 0x8000000000000000ul
31
32 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
33
34 struct hws_execute_parms {
35 void *buffer;
36 signed int rc;
37 };
38
39 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
40 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
41
42 static DEFINE_MUTEX(hws_sem);
43 static DEFINE_MUTEX(hws_sem_oom);
44
45 static unsigned char hws_flush_all;
46 static unsigned int hws_oom;
47 static struct workqueue_struct *hws_wq;
48
49 static unsigned int hws_state;
50 enum {
51 HWS_INIT = 1,
52 HWS_DEALLOCATED,
53 HWS_STOPPED,
54 HWS_STARTED,
55 HWS_STOPPING };
56
57 /* set to 1 if called by kernel during memory allocation */
58 static unsigned char oom_killer_was_active;
59 /* size of SDBT and SDB as of allocate API */
60 static unsigned long num_sdbt = 100;
61 static unsigned long num_sdb = 511;
62 /* sampling interval (machine cycles) */
63 static unsigned long interval;
64
65 static unsigned long min_sampler_rate;
66 static unsigned long max_sampler_rate;
67
68 static int ssctl(void *buffer)
69 {
70 int cc;
71
72 /* set in order to detect a program check */
73 cc = 1;
74
75 asm volatile(
76 "0: .insn s,0xB2870000,0(%1)\n"
77 "1: ipm %0\n"
78 " srl %0,28\n"
79 "2:\n"
80 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
81 : "+d" (cc), "+a" (buffer)
82 : "m" (*((struct hws_ssctl_request_block *)buffer))
83 : "cc", "memory");
84
85 return cc ? -EINVAL : 0 ;
86 }
87
88 static int qsi(void *buffer)
89 {
90 int cc;
91 cc = 1;
92
93 asm volatile(
94 "0: .insn s,0xB2860000,0(%1)\n"
95 "1: lhi %0,0\n"
96 "2:\n"
97 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
98 : "=d" (cc), "+a" (buffer)
99 : "m" (*((struct hws_qsi_info_block *)buffer))
100 : "cc", "memory");
101
102 return cc ? -EINVAL : 0;
103 }
104
105 static void execute_qsi(void *parms)
106 {
107 struct hws_execute_parms *ep = parms;
108
109 ep->rc = qsi(ep->buffer);
110 }
111
112 static void execute_ssctl(void *parms)
113 {
114 struct hws_execute_parms *ep = parms;
115
116 ep->rc = ssctl(ep->buffer);
117 }
118
119 static int smp_ctl_ssctl_stop(int cpu)
120 {
121 int rc;
122 struct hws_execute_parms ep;
123 struct hws_cpu_buffer *cb;
124
125 cb = &per_cpu(sampler_cpu_buffer, cpu);
126
127 cb->ssctl.es = 0;
128 cb->ssctl.cs = 0;
129
130 ep.buffer = &cb->ssctl;
131 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
132 rc = ep.rc;
133 if (rc) {
134 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
135 dump_stack();
136 }
137
138 ep.buffer = &cb->qsi;
139 smp_call_function_single(cpu, execute_qsi, &ep, 1);
140
141 if (cb->qsi.es || cb->qsi.cs) {
142 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
143 dump_stack();
144 }
145
146 return rc;
147 }
148
149 static int smp_ctl_ssctl_deactivate(int cpu)
150 {
151 int rc;
152 struct hws_execute_parms ep;
153 struct hws_cpu_buffer *cb;
154
155 cb = &per_cpu(sampler_cpu_buffer, cpu);
156
157 cb->ssctl.es = 1;
158 cb->ssctl.cs = 0;
159
160 ep.buffer = &cb->ssctl;
161 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
162 rc = ep.rc;
163 if (rc)
164 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
165
166 ep.buffer = &cb->qsi;
167 smp_call_function_single(cpu, execute_qsi, &ep, 1);
168
169 if (cb->qsi.cs)
170 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
171
172 return rc;
173 }
174
175 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
176 {
177 int rc;
178 struct hws_execute_parms ep;
179 struct hws_cpu_buffer *cb;
180
181 cb = &per_cpu(sampler_cpu_buffer, cpu);
182
183 cb->ssctl.h = 1;
184 cb->ssctl.tear = cb->first_sdbt;
185 cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
186 cb->ssctl.interval = interval;
187 cb->ssctl.es = 1;
188 cb->ssctl.cs = 1;
189
190 ep.buffer = &cb->ssctl;
191 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
192 rc = ep.rc;
193 if (rc)
194 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
195
196 ep.buffer = &cb->qsi;
197 smp_call_function_single(cpu, execute_qsi, &ep, 1);
198 if (ep.rc)
199 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
200
201 return rc;
202 }
203
204 static int smp_ctl_qsi(int cpu)
205 {
206 struct hws_execute_parms ep;
207 struct hws_cpu_buffer *cb;
208
209 cb = &per_cpu(sampler_cpu_buffer, cpu);
210
211 ep.buffer = &cb->qsi;
212 smp_call_function_single(cpu, execute_qsi, &ep, 1);
213
214 return ep.rc;
215 }
216
217 static inline unsigned long *trailer_entry_ptr(unsigned long v)
218 {
219 void *ret;
220
221 ret = (void *)v;
222 ret += PAGE_SIZE;
223 ret -= sizeof(struct hws_trailer_entry);
224
225 return (unsigned long *) ret;
226 }
227
228 static void hws_ext_handler(struct ext_code ext_code,
229 unsigned int param32, unsigned long param64)
230 {
231 struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
232
233 if (!(param32 & CPU_MF_INT_SF_MASK))
234 return;
235
236 inc_irq_stat(IRQEXT_CMS);
237 atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
238
239 if (hws_wq)
240 queue_work(hws_wq, &cb->worker);
241 }
242
243 static void worker(struct work_struct *work);
244
245 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
246 unsigned long *dear);
247
248 static void init_all_cpu_buffers(void)
249 {
250 int cpu;
251 struct hws_cpu_buffer *cb;
252
253 for_each_online_cpu(cpu) {
254 cb = &per_cpu(sampler_cpu_buffer, cpu);
255 memset(cb, 0, sizeof(struct hws_cpu_buffer));
256 }
257 }
258
259 static int is_link_entry(unsigned long *s)
260 {
261 return *s & 0x1ul ? 1 : 0;
262 }
263
264 static unsigned long *get_next_sdbt(unsigned long *s)
265 {
266 return (unsigned long *) (*s & ~0x1ul);
267 }
268
269 static int prepare_cpu_buffers(void)
270 {
271 int cpu;
272 int rc;
273 struct hws_cpu_buffer *cb;
274
275 rc = 0;
276 for_each_online_cpu(cpu) {
277 cb = &per_cpu(sampler_cpu_buffer, cpu);
278 atomic_set(&cb->ext_params, 0);
279 cb->worker_entry = 0;
280 cb->sample_overflow = 0;
281 cb->req_alert = 0;
282 cb->incorrect_sdbt_entry = 0;
283 cb->invalid_entry_address = 0;
284 cb->loss_of_sample_data = 0;
285 cb->sample_auth_change_alert = 0;
286 cb->finish = 0;
287 cb->oom = 0;
288 cb->stop_mode = 0;
289 }
290
291 return rc;
292 }
293
294 /*
295 * allocate_sdbt() - allocate sampler memory
296 * @cpu: the cpu for which sampler memory is allocated
297 *
298 * A 4K page is allocated for each requested SDBT.
299 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
300 * Set ALERT_REQ mask in each SDBs trailer.
301 * Returns zero if successful, <0 otherwise.
302 */
303 static int allocate_sdbt(int cpu)
304 {
305 int j, k, rc;
306 unsigned long *sdbt;
307 unsigned long sdb;
308 unsigned long *tail;
309 unsigned long *trailer;
310 struct hws_cpu_buffer *cb;
311
312 cb = &per_cpu(sampler_cpu_buffer, cpu);
313
314 if (cb->first_sdbt)
315 return -EINVAL;
316
317 sdbt = NULL;
318 tail = sdbt;
319
320 for (j = 0; j < num_sdbt; j++) {
321 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
322
323 mutex_lock(&hws_sem_oom);
324 /* OOM killer might have been activated */
325 barrier();
326 if (oom_killer_was_active || !sdbt) {
327 if (sdbt)
328 free_page((unsigned long)sdbt);
329
330 goto allocate_sdbt_error;
331 }
332 if (cb->first_sdbt == 0)
333 cb->first_sdbt = (unsigned long)sdbt;
334
335 /* link current page to tail of chain */
336 if (tail)
337 *tail = (unsigned long)(void *)sdbt + 1;
338
339 mutex_unlock(&hws_sem_oom);
340
341 for (k = 0; k < num_sdb; k++) {
342 /* get and set SDB page */
343 sdb = get_zeroed_page(GFP_KERNEL);
344
345 mutex_lock(&hws_sem_oom);
346 /* OOM killer might have been activated */
347 barrier();
348 if (oom_killer_was_active || !sdb) {
349 if (sdb)
350 free_page(sdb);
351
352 goto allocate_sdbt_error;
353 }
354 *sdbt = sdb;
355 trailer = trailer_entry_ptr(*sdbt);
356 *trailer = ALERT_REQ_MASK;
357 sdbt++;
358 mutex_unlock(&hws_sem_oom);
359 }
360 tail = sdbt;
361 }
362 mutex_lock(&hws_sem_oom);
363 if (oom_killer_was_active)
364 goto allocate_sdbt_error;
365
366 rc = 0;
367 if (tail)
368 *tail = (unsigned long)
369 ((void *)cb->first_sdbt) + 1;
370
371 allocate_sdbt_exit:
372 mutex_unlock(&hws_sem_oom);
373 return rc;
374
375 allocate_sdbt_error:
376 rc = -ENOMEM;
377 goto allocate_sdbt_exit;
378 }
379
380 /*
381 * deallocate_sdbt() - deallocate all sampler memory
382 *
383 * For each online CPU all SDBT trees are deallocated.
384 * Returns the number of freed pages.
385 */
386 static int deallocate_sdbt(void)
387 {
388 int cpu;
389 int counter;
390
391 counter = 0;
392
393 for_each_online_cpu(cpu) {
394 unsigned long start;
395 unsigned long sdbt;
396 unsigned long *curr;
397 struct hws_cpu_buffer *cb;
398
399 cb = &per_cpu(sampler_cpu_buffer, cpu);
400
401 if (!cb->first_sdbt)
402 continue;
403
404 sdbt = cb->first_sdbt;
405 curr = (unsigned long *) sdbt;
406 start = sdbt;
407
408 /* we'll free the SDBT after all SDBs are processed... */
409 while (1) {
410 if (!*curr || !sdbt)
411 break;
412
413 /* watch for link entry reset if found */
414 if (is_link_entry(curr)) {
415 curr = get_next_sdbt(curr);
416 if (sdbt)
417 free_page(sdbt);
418
419 /* we are done if we reach the start */
420 if ((unsigned long) curr == start)
421 break;
422 else
423 sdbt = (unsigned long) curr;
424 } else {
425 /* process SDB pointer */
426 if (*curr) {
427 free_page(*curr);
428 curr++;
429 }
430 }
431 counter++;
432 }
433 cb->first_sdbt = 0;
434 }
435 return counter;
436 }
437
438 static int start_sampling(int cpu)
439 {
440 int rc;
441 struct hws_cpu_buffer *cb;
442
443 cb = &per_cpu(sampler_cpu_buffer, cpu);
444 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
445 if (rc) {
446 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
447 goto start_exit;
448 }
449
450 rc = -EINVAL;
451 if (!cb->qsi.es) {
452 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
453 goto start_exit;
454 }
455
456 if (!cb->qsi.cs) {
457 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
458 goto start_exit;
459 }
460
461 printk(KERN_INFO
462 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
463 cpu, interval);
464
465 rc = 0;
466
467 start_exit:
468 return rc;
469 }
470
471 static int stop_sampling(int cpu)
472 {
473 unsigned long v;
474 int rc;
475 struct hws_cpu_buffer *cb;
476
477 rc = smp_ctl_qsi(cpu);
478 WARN_ON(rc);
479
480 cb = &per_cpu(sampler_cpu_buffer, cpu);
481 if (!rc && !cb->qsi.es)
482 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
483
484 rc = smp_ctl_ssctl_stop(cpu);
485 if (rc) {
486 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
487 cpu, rc);
488 goto stop_exit;
489 }
490
491 printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
492
493 stop_exit:
494 v = cb->req_alert;
495 if (v)
496 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
497 " count=%lu.\n", cpu, v);
498
499 v = cb->loss_of_sample_data;
500 if (v)
501 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
502 " count=%lu.\n", cpu, v);
503
504 v = cb->invalid_entry_address;
505 if (v)
506 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
507 " count=%lu.\n", cpu, v);
508
509 v = cb->incorrect_sdbt_entry;
510 if (v)
511 printk(KERN_ERR
512 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
513 " count=%lu.\n", cpu, v);
514
515 v = cb->sample_auth_change_alert;
516 if (v)
517 printk(KERN_ERR
518 "hwsampler: CPU %d CPUMF Sample authorization change,"
519 " count=%lu.\n", cpu, v);
520
521 return rc;
522 }
523
524 static int check_hardware_prerequisites(void)
525 {
526 if (!test_facility(68))
527 return -EOPNOTSUPP;
528 return 0;
529 }
530 /*
531 * hws_oom_callback() - the OOM callback function
532 *
533 * In case the callback is invoked during memory allocation for the
534 * hw sampler, all obtained memory is deallocated and a flag is set
535 * so main sampler memory allocation can exit with a failure code.
536 * In case the callback is invoked during sampling the hw sampler
537 * is deactivated for all CPUs.
538 */
539 static int hws_oom_callback(struct notifier_block *nfb,
540 unsigned long dummy, void *parm)
541 {
542 unsigned long *freed;
543 int cpu;
544 struct hws_cpu_buffer *cb;
545
546 freed = parm;
547
548 mutex_lock(&hws_sem_oom);
549
550 if (hws_state == HWS_DEALLOCATED) {
551 /* during memory allocation */
552 if (oom_killer_was_active == 0) {
553 oom_killer_was_active = 1;
554 *freed += deallocate_sdbt();
555 }
556 } else {
557 int i;
558 cpu = get_cpu();
559 cb = &per_cpu(sampler_cpu_buffer, cpu);
560
561 if (!cb->oom) {
562 for_each_online_cpu(i) {
563 smp_ctl_ssctl_deactivate(i);
564 cb->oom = 1;
565 }
566 cb->finish = 1;
567
568 printk(KERN_INFO
569 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
570 cpu);
571 }
572 }
573
574 mutex_unlock(&hws_sem_oom);
575
576 return NOTIFY_OK;
577 }
578
579 static struct notifier_block hws_oom_notifier = {
580 .notifier_call = hws_oom_callback
581 };
582
583 static int hws_cpu_callback(struct notifier_block *nfb,
584 unsigned long action, void *hcpu)
585 {
586 /* We do not have sampler space available for all possible CPUs.
587 All CPUs should be online when hw sampling is activated. */
588 return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
589 }
590
591 static struct notifier_block hws_cpu_notifier = {
592 .notifier_call = hws_cpu_callback
593 };
594
595 /**
596 * hwsampler_deactivate() - set hardware sampling temporarily inactive
597 * @cpu: specifies the CPU to be set inactive.
598 *
599 * Returns 0 on success, !0 on failure.
600 */
601 int hwsampler_deactivate(unsigned int cpu)
602 {
603 /*
604 * Deactivate hw sampling temporarily and flush the buffer
605 * by pushing all the pending samples to oprofile buffer.
606 *
607 * This function can be called under one of the following conditions:
608 * Memory unmap, task is exiting.
609 */
610 int rc;
611 struct hws_cpu_buffer *cb;
612
613 rc = 0;
614 mutex_lock(&hws_sem);
615
616 cb = &per_cpu(sampler_cpu_buffer, cpu);
617 if (hws_state == HWS_STARTED) {
618 rc = smp_ctl_qsi(cpu);
619 WARN_ON(rc);
620 if (cb->qsi.cs) {
621 rc = smp_ctl_ssctl_deactivate(cpu);
622 if (rc) {
623 printk(KERN_INFO
624 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
625 cb->finish = 1;
626 hws_state = HWS_STOPPING;
627 } else {
628 hws_flush_all = 1;
629 /* Add work to queue to read pending samples.*/
630 queue_work_on(cpu, hws_wq, &cb->worker);
631 }
632 }
633 }
634 mutex_unlock(&hws_sem);
635
636 if (hws_wq)
637 flush_workqueue(hws_wq);
638
639 return rc;
640 }
641
642 /**
643 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
644 * @cpu: specifies the CPU to be set active.
645 *
646 * Returns 0 on success, !0 on failure.
647 */
648 int hwsampler_activate(unsigned int cpu)
649 {
650 /*
651 * Re-activate hw sampling. This should be called in pair with
652 * hwsampler_deactivate().
653 */
654 int rc;
655 struct hws_cpu_buffer *cb;
656
657 rc = 0;
658 mutex_lock(&hws_sem);
659
660 cb = &per_cpu(sampler_cpu_buffer, cpu);
661 if (hws_state == HWS_STARTED) {
662 rc = smp_ctl_qsi(cpu);
663 WARN_ON(rc);
664 if (!cb->qsi.cs) {
665 hws_flush_all = 0;
666 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
667 if (rc) {
668 printk(KERN_ERR
669 "CPU %d, CPUMF activate sampling failed.\n",
670 cpu);
671 }
672 }
673 }
674
675 mutex_unlock(&hws_sem);
676
677 return rc;
678 }
679
680 static int check_qsi_on_setup(void)
681 {
682 int rc;
683 unsigned int cpu;
684 struct hws_cpu_buffer *cb;
685
686 for_each_online_cpu(cpu) {
687 cb = &per_cpu(sampler_cpu_buffer, cpu);
688 rc = smp_ctl_qsi(cpu);
689 WARN_ON(rc);
690 if (rc)
691 return -EOPNOTSUPP;
692
693 if (!cb->qsi.as) {
694 printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
695 return -EINVAL;
696 }
697
698 if (cb->qsi.es) {
699 printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
700 rc = smp_ctl_ssctl_stop(cpu);
701 if (rc)
702 return -EINVAL;
703
704 printk(KERN_INFO
705 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
706 }
707 }
708 return 0;
709 }
710
711 static int check_qsi_on_start(void)
712 {
713 unsigned int cpu;
714 int rc;
715 struct hws_cpu_buffer *cb;
716
717 for_each_online_cpu(cpu) {
718 cb = &per_cpu(sampler_cpu_buffer, cpu);
719 rc = smp_ctl_qsi(cpu);
720 WARN_ON(rc);
721
722 if (!cb->qsi.as)
723 return -EINVAL;
724
725 if (cb->qsi.es)
726 return -EINVAL;
727
728 if (cb->qsi.cs)
729 return -EINVAL;
730 }
731 return 0;
732 }
733
734 static void worker_on_start(unsigned int cpu)
735 {
736 struct hws_cpu_buffer *cb;
737
738 cb = &per_cpu(sampler_cpu_buffer, cpu);
739 cb->worker_entry = cb->first_sdbt;
740 }
741
742 static int worker_check_error(unsigned int cpu, int ext_params)
743 {
744 int rc;
745 unsigned long *sdbt;
746 struct hws_cpu_buffer *cb;
747
748 rc = 0;
749 cb = &per_cpu(sampler_cpu_buffer, cpu);
750 sdbt = (unsigned long *) cb->worker_entry;
751
752 if (!sdbt || !*sdbt)
753 return -EINVAL;
754
755 if (ext_params & CPU_MF_INT_SF_PRA)
756 cb->req_alert++;
757
758 if (ext_params & CPU_MF_INT_SF_LSDA)
759 cb->loss_of_sample_data++;
760
761 if (ext_params & CPU_MF_INT_SF_IAE) {
762 cb->invalid_entry_address++;
763 rc = -EINVAL;
764 }
765
766 if (ext_params & CPU_MF_INT_SF_ISE) {
767 cb->incorrect_sdbt_entry++;
768 rc = -EINVAL;
769 }
770
771 if (ext_params & CPU_MF_INT_SF_SACA) {
772 cb->sample_auth_change_alert++;
773 rc = -EINVAL;
774 }
775
776 return rc;
777 }
778
779 static void worker_on_finish(unsigned int cpu)
780 {
781 int rc, i;
782 struct hws_cpu_buffer *cb;
783
784 cb = &per_cpu(sampler_cpu_buffer, cpu);
785
786 if (cb->finish) {
787 rc = smp_ctl_qsi(cpu);
788 WARN_ON(rc);
789 if (cb->qsi.es) {
790 printk(KERN_INFO
791 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
792 cpu);
793 rc = smp_ctl_ssctl_stop(cpu);
794 if (rc)
795 printk(KERN_INFO
796 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
797 cpu);
798
799 for_each_online_cpu(i) {
800 if (i == cpu)
801 continue;
802 if (!cb->finish) {
803 cb->finish = 1;
804 queue_work_on(i, hws_wq,
805 &cb->worker);
806 }
807 }
808 }
809 }
810 }
811
812 static void worker_on_interrupt(unsigned int cpu)
813 {
814 unsigned long *sdbt;
815 unsigned char done;
816 struct hws_cpu_buffer *cb;
817
818 cb = &per_cpu(sampler_cpu_buffer, cpu);
819
820 sdbt = (unsigned long *) cb->worker_entry;
821
822 done = 0;
823 /* do not proceed if stop was entered,
824 * forget the buffers not yet processed */
825 while (!done && !cb->stop_mode) {
826 unsigned long *trailer;
827 struct hws_trailer_entry *te;
828 unsigned long *dear = 0;
829
830 trailer = trailer_entry_ptr(*sdbt);
831 /* leave loop if no more work to do */
832 if (!(*trailer & BUFFER_FULL_MASK)) {
833 done = 1;
834 if (!hws_flush_all)
835 continue;
836 }
837
838 te = (struct hws_trailer_entry *)trailer;
839 cb->sample_overflow += te->overflow;
840
841 add_samples_to_oprofile(cpu, sdbt, dear);
842
843 /* reset trailer */
844 xchg((unsigned char *) te, 0x40);
845
846 /* advance to next sdb slot in current sdbt */
847 sdbt++;
848 /* in case link bit is set use address w/o link bit */
849 if (is_link_entry(sdbt))
850 sdbt = get_next_sdbt(sdbt);
851
852 cb->worker_entry = (unsigned long)sdbt;
853 }
854 }
855
856 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
857 unsigned long *dear)
858 {
859 struct hws_data_entry *sample_data_ptr;
860 unsigned long *trailer;
861
862 trailer = trailer_entry_ptr(*sdbt);
863 if (dear) {
864 if (dear > trailer)
865 return;
866 trailer = dear;
867 }
868
869 sample_data_ptr = (struct hws_data_entry *)(*sdbt);
870
871 while ((unsigned long *)sample_data_ptr < trailer) {
872 struct pt_regs *regs = NULL;
873 struct task_struct *tsk = NULL;
874
875 /*
876 * Check sampling mode, 1 indicates basic (=customer) sampling
877 * mode.
878 */
879 if (sample_data_ptr->def != 1) {
880 /* sample slot is not yet written */
881 break;
882 } else {
883 /* make sure we don't use it twice,
884 * the next time the sampler will set it again */
885 sample_data_ptr->def = 0;
886 }
887
888 /* Get pt_regs. */
889 if (sample_data_ptr->P == 1) {
890 /* userspace sample */
891 unsigned int pid = sample_data_ptr->prim_asn;
892 if (!counter_config.user)
893 goto skip_sample;
894 rcu_read_lock();
895 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
896 if (tsk)
897 regs = task_pt_regs(tsk);
898 rcu_read_unlock();
899 } else {
900 /* kernelspace sample */
901 if (!counter_config.kernel)
902 goto skip_sample;
903 regs = task_pt_regs(current);
904 }
905
906 mutex_lock(&hws_sem);
907 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
908 !sample_data_ptr->P, tsk);
909 mutex_unlock(&hws_sem);
910 skip_sample:
911 sample_data_ptr++;
912 }
913 }
914
915 static void worker(struct work_struct *work)
916 {
917 unsigned int cpu;
918 int ext_params;
919 struct hws_cpu_buffer *cb;
920
921 cb = container_of(work, struct hws_cpu_buffer, worker);
922 cpu = smp_processor_id();
923 ext_params = atomic_xchg(&cb->ext_params, 0);
924
925 if (!cb->worker_entry)
926 worker_on_start(cpu);
927
928 if (worker_check_error(cpu, ext_params))
929 return;
930
931 if (!cb->finish)
932 worker_on_interrupt(cpu);
933
934 if (cb->finish)
935 worker_on_finish(cpu);
936 }
937
938 /**
939 * hwsampler_allocate() - allocate memory for the hardware sampler
940 * @sdbt: number of SDBTs per online CPU (must be > 0)
941 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
942 *
943 * Returns 0 on success, !0 on failure.
944 */
945 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
946 {
947 int cpu, rc;
948 mutex_lock(&hws_sem);
949
950 rc = -EINVAL;
951 if (hws_state != HWS_DEALLOCATED)
952 goto allocate_exit;
953
954 if (sdbt < 1)
955 goto allocate_exit;
956
957 if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
958 goto allocate_exit;
959
960 num_sdbt = sdbt;
961 num_sdb = sdb;
962
963 oom_killer_was_active = 0;
964 register_oom_notifier(&hws_oom_notifier);
965
966 for_each_online_cpu(cpu) {
967 if (allocate_sdbt(cpu)) {
968 unregister_oom_notifier(&hws_oom_notifier);
969 goto allocate_error;
970 }
971 }
972 unregister_oom_notifier(&hws_oom_notifier);
973 if (oom_killer_was_active)
974 goto allocate_error;
975
976 hws_state = HWS_STOPPED;
977 rc = 0;
978
979 allocate_exit:
980 mutex_unlock(&hws_sem);
981 return rc;
982
983 allocate_error:
984 rc = -ENOMEM;
985 printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
986 goto allocate_exit;
987 }
988
989 /**
990 * hwsampler_deallocate() - deallocate hardware sampler memory
991 *
992 * Returns 0 on success, !0 on failure.
993 */
994 int hwsampler_deallocate(void)
995 {
996 int rc;
997
998 mutex_lock(&hws_sem);
999
1000 rc = -EINVAL;
1001 if (hws_state != HWS_STOPPED)
1002 goto deallocate_exit;
1003
1004 measurement_alert_subclass_unregister();
1005 deallocate_sdbt();
1006
1007 hws_state = HWS_DEALLOCATED;
1008 rc = 0;
1009
1010 deallocate_exit:
1011 mutex_unlock(&hws_sem);
1012
1013 return rc;
1014 }
1015
1016 unsigned long hwsampler_query_min_interval(void)
1017 {
1018 return min_sampler_rate;
1019 }
1020
1021 unsigned long hwsampler_query_max_interval(void)
1022 {
1023 return max_sampler_rate;
1024 }
1025
1026 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1027 {
1028 struct hws_cpu_buffer *cb;
1029
1030 cb = &per_cpu(sampler_cpu_buffer, cpu);
1031
1032 return cb->sample_overflow;
1033 }
1034
1035 int hwsampler_setup(void)
1036 {
1037 int rc;
1038 int cpu;
1039 struct hws_cpu_buffer *cb;
1040
1041 mutex_lock(&hws_sem);
1042
1043 rc = -EINVAL;
1044 if (hws_state)
1045 goto setup_exit;
1046
1047 hws_state = HWS_INIT;
1048
1049 init_all_cpu_buffers();
1050
1051 rc = check_hardware_prerequisites();
1052 if (rc)
1053 goto setup_exit;
1054
1055 rc = check_qsi_on_setup();
1056 if (rc)
1057 goto setup_exit;
1058
1059 rc = -EINVAL;
1060 hws_wq = create_workqueue("hwsampler");
1061 if (!hws_wq)
1062 goto setup_exit;
1063
1064 register_cpu_notifier(&hws_cpu_notifier);
1065
1066 for_each_online_cpu(cpu) {
1067 cb = &per_cpu(sampler_cpu_buffer, cpu);
1068 INIT_WORK(&cb->worker, worker);
1069 rc = smp_ctl_qsi(cpu);
1070 WARN_ON(rc);
1071 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1072 if (min_sampler_rate) {
1073 printk(KERN_WARNING
1074 "hwsampler: different min sampler rate values.\n");
1075 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1076 min_sampler_rate =
1077 cb->qsi.min_sampl_rate;
1078 } else
1079 min_sampler_rate = cb->qsi.min_sampl_rate;
1080 }
1081 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1082 if (max_sampler_rate) {
1083 printk(KERN_WARNING
1084 "hwsampler: different max sampler rate values.\n");
1085 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1086 max_sampler_rate =
1087 cb->qsi.max_sampl_rate;
1088 } else
1089 max_sampler_rate = cb->qsi.max_sampl_rate;
1090 }
1091 }
1092 register_external_interrupt(0x1407, hws_ext_handler);
1093
1094 hws_state = HWS_DEALLOCATED;
1095 rc = 0;
1096
1097 setup_exit:
1098 mutex_unlock(&hws_sem);
1099 return rc;
1100 }
1101
1102 int hwsampler_shutdown(void)
1103 {
1104 int rc;
1105
1106 mutex_lock(&hws_sem);
1107
1108 rc = -EINVAL;
1109 if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1110 mutex_unlock(&hws_sem);
1111
1112 if (hws_wq)
1113 flush_workqueue(hws_wq);
1114
1115 mutex_lock(&hws_sem);
1116
1117 if (hws_state == HWS_STOPPED) {
1118 measurement_alert_subclass_unregister();
1119 deallocate_sdbt();
1120 }
1121 if (hws_wq) {
1122 destroy_workqueue(hws_wq);
1123 hws_wq = NULL;
1124 }
1125
1126 unregister_external_interrupt(0x1407, hws_ext_handler);
1127 hws_state = HWS_INIT;
1128 rc = 0;
1129 }
1130 mutex_unlock(&hws_sem);
1131
1132 unregister_cpu_notifier(&hws_cpu_notifier);
1133
1134 return rc;
1135 }
1136
1137 /**
1138 * hwsampler_start_all() - start hardware sampling on all online CPUs
1139 * @rate: specifies the used interval when samples are taken
1140 *
1141 * Returns 0 on success, !0 on failure.
1142 */
1143 int hwsampler_start_all(unsigned long rate)
1144 {
1145 int rc, cpu;
1146
1147 mutex_lock(&hws_sem);
1148
1149 hws_oom = 0;
1150
1151 rc = -EINVAL;
1152 if (hws_state != HWS_STOPPED)
1153 goto start_all_exit;
1154
1155 interval = rate;
1156
1157 /* fail if rate is not valid */
1158 if (interval < min_sampler_rate || interval > max_sampler_rate)
1159 goto start_all_exit;
1160
1161 rc = check_qsi_on_start();
1162 if (rc)
1163 goto start_all_exit;
1164
1165 rc = prepare_cpu_buffers();
1166 if (rc)
1167 goto start_all_exit;
1168
1169 for_each_online_cpu(cpu) {
1170 rc = start_sampling(cpu);
1171 if (rc)
1172 break;
1173 }
1174 if (rc) {
1175 for_each_online_cpu(cpu) {
1176 stop_sampling(cpu);
1177 }
1178 goto start_all_exit;
1179 }
1180 hws_state = HWS_STARTED;
1181 rc = 0;
1182
1183 start_all_exit:
1184 mutex_unlock(&hws_sem);
1185
1186 if (rc)
1187 return rc;
1188
1189 register_oom_notifier(&hws_oom_notifier);
1190 hws_oom = 1;
1191 hws_flush_all = 0;
1192 /* now let them in, 1407 CPUMF external interrupts */
1193 measurement_alert_subclass_register();
1194
1195 return 0;
1196 }
1197
1198 /**
1199 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1200 *
1201 * Returns 0 on success, !0 on failure.
1202 */
1203 int hwsampler_stop_all(void)
1204 {
1205 int tmp_rc, rc, cpu;
1206 struct hws_cpu_buffer *cb;
1207
1208 mutex_lock(&hws_sem);
1209
1210 rc = 0;
1211 if (hws_state == HWS_INIT) {
1212 mutex_unlock(&hws_sem);
1213 return rc;
1214 }
1215 hws_state = HWS_STOPPING;
1216 mutex_unlock(&hws_sem);
1217
1218 for_each_online_cpu(cpu) {
1219 cb = &per_cpu(sampler_cpu_buffer, cpu);
1220 cb->stop_mode = 1;
1221 tmp_rc = stop_sampling(cpu);
1222 if (tmp_rc)
1223 rc = tmp_rc;
1224 }
1225
1226 if (hws_wq)
1227 flush_workqueue(hws_wq);
1228
1229 mutex_lock(&hws_sem);
1230 if (hws_oom) {
1231 unregister_oom_notifier(&hws_oom_notifier);
1232 hws_oom = 0;
1233 }
1234 hws_state = HWS_STOPPED;
1235 mutex_unlock(&hws_sem);
1236
1237 return rc;
1238 }
CRIS linux kernel tree