?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Avoid beyond bounds copy while caching ACL
[zen-stable.git] / arch / s390 / oprofile / hwsampler.c
blob9daee91e6c3fc20f8668a1797a8d388c102b62aa
1 /**
2 * arch/s390/oprofile/hwsampler.c
3 *
4 * Copyright IBM Corp. 2010
5 * Author: Heinz Graalfs <graalfs@de.ibm.com>
6 */
7
8 #include <linux/kernel_stat.h>
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/smp.h>
12 #include <linux/errno.h>
13 #include <linux/workqueue.h>
14 #include <linux/interrupt.h>
15 #include <linux/notifier.h>
16 #include <linux/cpu.h>
17 #include <linux/semaphore.h>
18 #include <linux/oom.h>
19 #include <linux/oprofile.h>
20
21 #include <asm/lowcore.h>
22 #include <asm/irq.h>
23
24 #include "hwsampler.h"
25 #include "op_counter.h"
26
27 #define MAX_NUM_SDB 511
28 #define MIN_NUM_SDB 1
29
30 #define ALERT_REQ_MASK 0x4000000000000000ul
31 #define BUFFER_FULL_MASK 0x8000000000000000ul
32
33 #define EI_IEA (1 << 31) /* invalid entry address */
34 #define EI_ISE (1 << 30) /* incorrect SDBT entry */
35 #define EI_PRA (1 << 29) /* program request alert */
36 #define EI_SACA (1 << 23) /* sampler authorization change alert */
37 #define EI_LSDA (1 << 22) /* loss of sample data alert */
38
39 DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
40
41 struct hws_execute_parms {
42 void *buffer;
43 signed int rc;
44 };
45
46 DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
47 EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
48
49 static DEFINE_MUTEX(hws_sem);
50 static DEFINE_MUTEX(hws_sem_oom);
51
52 static unsigned char hws_flush_all;
53 static unsigned int hws_oom;
54 static struct workqueue_struct *hws_wq;
55
56 static unsigned int hws_state;
57 enum {
58 HWS_INIT = 1,
59 HWS_DEALLOCATED,
60 HWS_STOPPED,
61 HWS_STARTED,
62 HWS_STOPPING };
63
64 /* set to 1 if called by kernel during memory allocation */
65 static unsigned char oom_killer_was_active;
66 /* size of SDBT and SDB as of allocate API */
67 static unsigned long num_sdbt = 100;
68 static unsigned long num_sdb = 511;
69 /* sampling interval (machine cycles) */
70 static unsigned long interval;
71
72 static unsigned long min_sampler_rate;
73 static unsigned long max_sampler_rate;
74
75 static int ssctl(void *buffer)
76 {
77 int cc;
78
79 /* set in order to detect a program check */
80 cc = 1;
81
82 asm volatile(
83 "0: .insn s,0xB2870000,0(%1)\n"
84 "1: ipm %0\n"
85 " srl %0,28\n"
86 "2:\n"
87 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
88 : "+d" (cc), "+a" (buffer)
89 : "m" (*((struct hws_ssctl_request_block *)buffer))
90 : "cc", "memory");
91
92 return cc ? -EINVAL : 0 ;
93 }
94
95 static int qsi(void *buffer)
96 {
97 int cc;
98 cc = 1;
99
100 asm volatile(
101 "0: .insn s,0xB2860000,0(%1)\n"
102 "1: lhi %0,0\n"
103 "2:\n"
104 EX_TABLE(0b, 2b) EX_TABLE(1b, 2b)
105 : "=d" (cc), "+a" (buffer)
106 : "m" (*((struct hws_qsi_info_block *)buffer))
107 : "cc", "memory");
108
109 return cc ? -EINVAL : 0;
110 }
111
112 static void execute_qsi(void *parms)
113 {
114 struct hws_execute_parms *ep = parms;
115
116 ep->rc = qsi(ep->buffer);
117 }
118
119 static void execute_ssctl(void *parms)
120 {
121 struct hws_execute_parms *ep = parms;
122
123 ep->rc = ssctl(ep->buffer);
124 }
125
126 static int smp_ctl_ssctl_stop(int cpu)
127 {
128 int rc;
129 struct hws_execute_parms ep;
130 struct hws_cpu_buffer *cb;
131
132 cb = &per_cpu(sampler_cpu_buffer, cpu);
133
134 cb->ssctl.es = 0;
135 cb->ssctl.cs = 0;
136
137 ep.buffer = &cb->ssctl;
138 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
139 rc = ep.rc;
140 if (rc) {
141 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
142 dump_stack();
143 }
144
145 ep.buffer = &cb->qsi;
146 smp_call_function_single(cpu, execute_qsi, &ep, 1);
147
148 if (cb->qsi.es || cb->qsi.cs) {
149 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
150 dump_stack();
151 }
152
153 return rc;
154 }
155
156 static int smp_ctl_ssctl_deactivate(int cpu)
157 {
158 int rc;
159 struct hws_execute_parms ep;
160 struct hws_cpu_buffer *cb;
161
162 cb = &per_cpu(sampler_cpu_buffer, cpu);
163
164 cb->ssctl.es = 1;
165 cb->ssctl.cs = 0;
166
167 ep.buffer = &cb->ssctl;
168 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
169 rc = ep.rc;
170 if (rc)
171 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
172
173 ep.buffer = &cb->qsi;
174 smp_call_function_single(cpu, execute_qsi, &ep, 1);
175
176 if (cb->qsi.cs)
177 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
178
179 return rc;
180 }
181
182 static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
183 {
184 int rc;
185 struct hws_execute_parms ep;
186 struct hws_cpu_buffer *cb;
187
188 cb = &per_cpu(sampler_cpu_buffer, cpu);
189
190 cb->ssctl.h = 1;
191 cb->ssctl.tear = cb->first_sdbt;
192 cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
193 cb->ssctl.interval = interval;
194 cb->ssctl.es = 1;
195 cb->ssctl.cs = 1;
196
197 ep.buffer = &cb->ssctl;
198 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
199 rc = ep.rc;
200 if (rc)
201 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
202
203 ep.buffer = &cb->qsi;
204 smp_call_function_single(cpu, execute_qsi, &ep, 1);
205 if (ep.rc)
206 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
207
208 return rc;
209 }
210
211 static int smp_ctl_qsi(int cpu)
212 {
213 struct hws_execute_parms ep;
214 struct hws_cpu_buffer *cb;
215
216 cb = &per_cpu(sampler_cpu_buffer, cpu);
217
218 ep.buffer = &cb->qsi;
219 smp_call_function_single(cpu, execute_qsi, &ep, 1);
220
221 return ep.rc;
222 }
223
224 static inline unsigned long *trailer_entry_ptr(unsigned long v)
225 {
226 void *ret;
227
228 ret = (void *)v;
229 ret += PAGE_SIZE;
230 ret -= sizeof(struct hws_trailer_entry);
231
232 return (unsigned long *) ret;
233 }
234
235 /* prototypes for external interrupt handler and worker */
236 static void hws_ext_handler(unsigned int ext_int_code,
237 unsigned int param32, unsigned long param64);
238
239 static void worker(struct work_struct *work);
240
241 static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
242 unsigned long *dear);
243
244 static void init_all_cpu_buffers(void)
245 {
246 int cpu;
247 struct hws_cpu_buffer *cb;
248
249 for_each_online_cpu(cpu) {
250 cb = &per_cpu(sampler_cpu_buffer, cpu);
251 memset(cb, 0, sizeof(struct hws_cpu_buffer));
252 }
253 }
254
255 static int is_link_entry(unsigned long *s)
256 {
257 return *s & 0x1ul ? 1 : 0;
258 }
259
260 static unsigned long *get_next_sdbt(unsigned long *s)
261 {
262 return (unsigned long *) (*s & ~0x1ul);
263 }
264
265 static int prepare_cpu_buffers(void)
266 {
267 int cpu;
268 int rc;
269 struct hws_cpu_buffer *cb;
270
271 rc = 0;
272 for_each_online_cpu(cpu) {
273 cb = &per_cpu(sampler_cpu_buffer, cpu);
274 atomic_set(&cb->ext_params, 0);
275 cb->worker_entry = 0;
276 cb->sample_overflow = 0;
277 cb->req_alert = 0;
278 cb->incorrect_sdbt_entry = 0;
279 cb->invalid_entry_address = 0;
280 cb->loss_of_sample_data = 0;
281 cb->sample_auth_change_alert = 0;
282 cb->finish = 0;
283 cb->oom = 0;
284 cb->stop_mode = 0;
285 }
286
287 return rc;
288 }
289
290 /*
291 * allocate_sdbt() - allocate sampler memory
292 * @cpu: the cpu for which sampler memory is allocated
293 *
294 * A 4K page is allocated for each requested SDBT.
295 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
296 * Set ALERT_REQ mask in each SDBs trailer.
297 * Returns zero if successful, <0 otherwise.
298 */
299 static int allocate_sdbt(int cpu)
300 {
301 int j, k, rc;
302 unsigned long *sdbt;
303 unsigned long sdb;
304 unsigned long *tail;
305 unsigned long *trailer;
306 struct hws_cpu_buffer *cb;
307
308 cb = &per_cpu(sampler_cpu_buffer, cpu);
309
310 if (cb->first_sdbt)
311 return -EINVAL;
312
313 sdbt = NULL;
314 tail = sdbt;
315
316 for (j = 0; j < num_sdbt; j++) {
317 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
318
319 mutex_lock(&hws_sem_oom);
320 /* OOM killer might have been activated */
321 barrier();
322 if (oom_killer_was_active || !sdbt) {
323 if (sdbt)
324 free_page((unsigned long)sdbt);
325
326 goto allocate_sdbt_error;
327 }
328 if (cb->first_sdbt == 0)
329 cb->first_sdbt = (unsigned long)sdbt;
330
331 /* link current page to tail of chain */
332 if (tail)
333 *tail = (unsigned long)(void *)sdbt + 1;
334
335 mutex_unlock(&hws_sem_oom);
336
337 for (k = 0; k < num_sdb; k++) {
338 /* get and set SDB page */
339 sdb = get_zeroed_page(GFP_KERNEL);
340
341 mutex_lock(&hws_sem_oom);
342 /* OOM killer might have been activated */
343 barrier();
344 if (oom_killer_was_active || !sdb) {
345 if (sdb)
346 free_page(sdb);
347
348 goto allocate_sdbt_error;
349 }
350 *sdbt = sdb;
351 trailer = trailer_entry_ptr(*sdbt);
352 *trailer = ALERT_REQ_MASK;
353 sdbt++;
354 mutex_unlock(&hws_sem_oom);
355 }
356 tail = sdbt;
357 }
358 mutex_lock(&hws_sem_oom);
359 if (oom_killer_was_active)
360 goto allocate_sdbt_error;
361
362 rc = 0;
363 if (tail)
364 *tail = (unsigned long)
365 ((void *)cb->first_sdbt) + 1;
366
367 allocate_sdbt_exit:
368 mutex_unlock(&hws_sem_oom);
369 return rc;
370
371 allocate_sdbt_error:
372 rc = -ENOMEM;
373 goto allocate_sdbt_exit;
374 }
375
376 /*
377 * deallocate_sdbt() - deallocate all sampler memory
378 *
379 * For each online CPU all SDBT trees are deallocated.
380 * Returns the number of freed pages.
381 */
382 static int deallocate_sdbt(void)
383 {
384 int cpu;
385 int counter;
386
387 counter = 0;
388
389 for_each_online_cpu(cpu) {
390 unsigned long start;
391 unsigned long sdbt;
392 unsigned long *curr;
393 struct hws_cpu_buffer *cb;
394
395 cb = &per_cpu(sampler_cpu_buffer, cpu);
396
397 if (!cb->first_sdbt)
398 continue;
399
400 sdbt = cb->first_sdbt;
401 curr = (unsigned long *) sdbt;
402 start = sdbt;
403
404 /* we'll free the SDBT after all SDBs are processed... */
405 while (1) {
406 if (!*curr || !sdbt)
407 break;
408
409 /* watch for link entry reset if found */
410 if (is_link_entry(curr)) {
411 curr = get_next_sdbt(curr);
412 if (sdbt)
413 free_page(sdbt);
414
415 /* we are done if we reach the start */
416 if ((unsigned long) curr == start)
417 break;
418 else
419 sdbt = (unsigned long) curr;
420 } else {
421 /* process SDB pointer */
422 if (*curr) {
423 free_page(*curr);
424 curr++;
425 }
426 }
427 counter++;
428 }
429 cb->first_sdbt = 0;
430 }
431 return counter;
432 }
433
434 static int start_sampling(int cpu)
435 {
436 int rc;
437 struct hws_cpu_buffer *cb;
438
439 cb = &per_cpu(sampler_cpu_buffer, cpu);
440 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
441 if (rc) {
442 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
443 goto start_exit;
444 }
445
446 rc = -EINVAL;
447 if (!cb->qsi.es) {
448 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
449 goto start_exit;
450 }
451
452 if (!cb->qsi.cs) {
453 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
454 goto start_exit;
455 }
456
457 printk(KERN_INFO
458 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
459 cpu, interval);
460
461 rc = 0;
462
463 start_exit:
464 return rc;
465 }
466
467 static int stop_sampling(int cpu)
468 {
469 unsigned long v;
470 int rc;
471 struct hws_cpu_buffer *cb;
472
473 rc = smp_ctl_qsi(cpu);
474 WARN_ON(rc);
475
476 cb = &per_cpu(sampler_cpu_buffer, cpu);
477 if (!rc && !cb->qsi.es)
478 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
479
480 rc = smp_ctl_ssctl_stop(cpu);
481 if (rc) {
482 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
483 cpu, rc);
484 goto stop_exit;
485 }
486
487 printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
488
489 stop_exit:
490 v = cb->req_alert;
491 if (v)
492 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
493 " count=%lu.\n", cpu, v);
494
495 v = cb->loss_of_sample_data;
496 if (v)
497 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
498 " count=%lu.\n", cpu, v);
499
500 v = cb->invalid_entry_address;
501 if (v)
502 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
503 " count=%lu.\n", cpu, v);
504
505 v = cb->incorrect_sdbt_entry;
506 if (v)
507 printk(KERN_ERR
508 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
509 " count=%lu.\n", cpu, v);
510
511 v = cb->sample_auth_change_alert;
512 if (v)
513 printk(KERN_ERR
514 "hwsampler: CPU %d CPUMF Sample authorization change,"
515 " count=%lu.\n", cpu, v);
516
517 return rc;
518 }
519
520 static int check_hardware_prerequisites(void)
521 {
522 if (!test_facility(68))
523 return -EOPNOTSUPP;
524 return 0;
525 }
526 /*
527 * hws_oom_callback() - the OOM callback function
528 *
529 * In case the callback is invoked during memory allocation for the
530 * hw sampler, all obtained memory is deallocated and a flag is set
531 * so main sampler memory allocation can exit with a failure code.
532 * In case the callback is invoked during sampling the hw sampler
533 * is deactivated for all CPUs.
534 */
535 static int hws_oom_callback(struct notifier_block *nfb,
536 unsigned long dummy, void *parm)
537 {
538 unsigned long *freed;
539 int cpu;
540 struct hws_cpu_buffer *cb;
541
542 freed = parm;
543
544 mutex_lock(&hws_sem_oom);
545
546 if (hws_state == HWS_DEALLOCATED) {
547 /* during memory allocation */
548 if (oom_killer_was_active == 0) {
549 oom_killer_was_active = 1;
550 *freed += deallocate_sdbt();
551 }
552 } else {
553 int i;
554 cpu = get_cpu();
555 cb = &per_cpu(sampler_cpu_buffer, cpu);
556
557 if (!cb->oom) {
558 for_each_online_cpu(i) {
559 smp_ctl_ssctl_deactivate(i);
560 cb->oom = 1;
561 }
562 cb->finish = 1;
563
564 printk(KERN_INFO
565 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
566 cpu);
567 }
568 }
569
570 mutex_unlock(&hws_sem_oom);
571
572 return NOTIFY_OK;
573 }
574
575 static struct notifier_block hws_oom_notifier = {
576 .notifier_call = hws_oom_callback
577 };
578
579 static int hws_cpu_callback(struct notifier_block *nfb,
580 unsigned long action, void *hcpu)
581 {
582 /* We do not have sampler space available for all possible CPUs.
583 All CPUs should be online when hw sampling is activated. */
584 return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
585 }
586
587 static struct notifier_block hws_cpu_notifier = {
588 .notifier_call = hws_cpu_callback
589 };
590
591 /**
592 * hwsampler_deactivate() - set hardware sampling temporarily inactive
593 * @cpu: specifies the CPU to be set inactive.
594 *
595 * Returns 0 on success, !0 on failure.
596 */
597 int hwsampler_deactivate(unsigned int cpu)
598 {
599 /*
600 * Deactivate hw sampling temporarily and flush the buffer
601 * by pushing all the pending samples to oprofile buffer.
602 *
603 * This function can be called under one of the following conditions:
604 * Memory unmap, task is exiting.
605 */
606 int rc;
607 struct hws_cpu_buffer *cb;
608
609 rc = 0;
610 mutex_lock(&hws_sem);
611
612 cb = &per_cpu(sampler_cpu_buffer, cpu);
613 if (hws_state == HWS_STARTED) {
614 rc = smp_ctl_qsi(cpu);
615 WARN_ON(rc);
616 if (cb->qsi.cs) {
617 rc = smp_ctl_ssctl_deactivate(cpu);
618 if (rc) {
619 printk(KERN_INFO
620 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
621 cb->finish = 1;
622 hws_state = HWS_STOPPING;
623 } else {
624 hws_flush_all = 1;
625 /* Add work to queue to read pending samples.*/
626 queue_work_on(cpu, hws_wq, &cb->worker);
627 }
628 }
629 }
630 mutex_unlock(&hws_sem);
631
632 if (hws_wq)
633 flush_workqueue(hws_wq);
634
635 return rc;
636 }
637
638 /**
639 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
640 * @cpu: specifies the CPU to be set active.
641 *
642 * Returns 0 on success, !0 on failure.
643 */
644 int hwsampler_activate(unsigned int cpu)
645 {
646 /*
647 * Re-activate hw sampling. This should be called in pair with
648 * hwsampler_deactivate().
649 */
650 int rc;
651 struct hws_cpu_buffer *cb;
652
653 rc = 0;
654 mutex_lock(&hws_sem);
655
656 cb = &per_cpu(sampler_cpu_buffer, cpu);
657 if (hws_state == HWS_STARTED) {
658 rc = smp_ctl_qsi(cpu);
659 WARN_ON(rc);
660 if (!cb->qsi.cs) {
661 hws_flush_all = 0;
662 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
663 if (rc) {
664 printk(KERN_ERR
665 "CPU %d, CPUMF activate sampling failed.\n",
666 cpu);
667 }
668 }
669 }
670
671 mutex_unlock(&hws_sem);
672
673 return rc;
674 }
675
676 static void hws_ext_handler(unsigned int ext_int_code,
677 unsigned int param32, unsigned long param64)
678 {
679 struct hws_cpu_buffer *cb;
680
681 kstat_cpu(smp_processor_id()).irqs[EXTINT_CPM]++;
682 cb = &__get_cpu_var(sampler_cpu_buffer);
683 atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
684 if (hws_wq)
685 queue_work(hws_wq, &cb->worker);
686 }
687
688 static int check_qsi_on_setup(void)
689 {
690 int rc;
691 unsigned int cpu;
692 struct hws_cpu_buffer *cb;
693
694 for_each_online_cpu(cpu) {
695 cb = &per_cpu(sampler_cpu_buffer, cpu);
696 rc = smp_ctl_qsi(cpu);
697 WARN_ON(rc);
698 if (rc)
699 return -EOPNOTSUPP;
700
701 if (!cb->qsi.as) {
702 printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
703 return -EINVAL;
704 }
705
706 if (cb->qsi.es) {
707 printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
708 rc = smp_ctl_ssctl_stop(cpu);
709 if (rc)
710 return -EINVAL;
711
712 printk(KERN_INFO
713 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
714 }
715 }
716 return 0;
717 }
718
719 static int check_qsi_on_start(void)
720 {
721 unsigned int cpu;
722 int rc;
723 struct hws_cpu_buffer *cb;
724
725 for_each_online_cpu(cpu) {
726 cb = &per_cpu(sampler_cpu_buffer, cpu);
727 rc = smp_ctl_qsi(cpu);
728 WARN_ON(rc);
729
730 if (!cb->qsi.as)
731 return -EINVAL;
732
733 if (cb->qsi.es)
734 return -EINVAL;
735
736 if (cb->qsi.cs)
737 return -EINVAL;
738 }
739 return 0;
740 }
741
742 static void worker_on_start(unsigned int cpu)
743 {
744 struct hws_cpu_buffer *cb;
745
746 cb = &per_cpu(sampler_cpu_buffer, cpu);
747 cb->worker_entry = cb->first_sdbt;
748 }
749
750 static int worker_check_error(unsigned int cpu, int ext_params)
751 {
752 int rc;
753 unsigned long *sdbt;
754 struct hws_cpu_buffer *cb;
755
756 rc = 0;
757 cb = &per_cpu(sampler_cpu_buffer, cpu);
758 sdbt = (unsigned long *) cb->worker_entry;
759
760 if (!sdbt || !*sdbt)
761 return -EINVAL;
762
763 if (ext_params & EI_PRA)
764 cb->req_alert++;
765
766 if (ext_params & EI_LSDA)
767 cb->loss_of_sample_data++;
768
769 if (ext_params & EI_IEA) {
770 cb->invalid_entry_address++;
771 rc = -EINVAL;
772 }
773
774 if (ext_params & EI_ISE) {
775 cb->incorrect_sdbt_entry++;
776 rc = -EINVAL;
777 }
778
779 if (ext_params & EI_SACA) {
780 cb->sample_auth_change_alert++;
781 rc = -EINVAL;
782 }
783
784 return rc;
785 }
786
787 static void worker_on_finish(unsigned int cpu)
788 {
789 int rc, i;
790 struct hws_cpu_buffer *cb;
791
792 cb = &per_cpu(sampler_cpu_buffer, cpu);
793
794 if (cb->finish) {
795 rc = smp_ctl_qsi(cpu);
796 WARN_ON(rc);
797 if (cb->qsi.es) {
798 printk(KERN_INFO
799 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
800 cpu);
801 rc = smp_ctl_ssctl_stop(cpu);
802 if (rc)
803 printk(KERN_INFO
804 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
805 cpu);
806
807 for_each_online_cpu(i) {
808 if (i == cpu)
809 continue;
810 if (!cb->finish) {
811 cb->finish = 1;
812 queue_work_on(i, hws_wq,
813 &cb->worker);
814 }
815 }
816 }
817 }
818 }
819
820 static void worker_on_interrupt(unsigned int cpu)
821 {
822 unsigned long *sdbt;
823 unsigned char done;
824 struct hws_cpu_buffer *cb;
825
826 cb = &per_cpu(sampler_cpu_buffer, cpu);
827
828 sdbt = (unsigned long *) cb->worker_entry;
829
830 done = 0;
831 /* do not proceed if stop was entered,
832 * forget the buffers not yet processed */
833 while (!done && !cb->stop_mode) {
834 unsigned long *trailer;
835 struct hws_trailer_entry *te;
836 unsigned long *dear = 0;
837
838 trailer = trailer_entry_ptr(*sdbt);
839 /* leave loop if no more work to do */
840 if (!(*trailer & BUFFER_FULL_MASK)) {
841 done = 1;
842 if (!hws_flush_all)
843 continue;
844 }
845
846 te = (struct hws_trailer_entry *)trailer;
847 cb->sample_overflow += te->overflow;
848
849 add_samples_to_oprofile(cpu, sdbt, dear);
850
851 /* reset trailer */
852 xchg((unsigned char *) te, 0x40);
853
854 /* advance to next sdb slot in current sdbt */
855 sdbt++;
856 /* in case link bit is set use address w/o link bit */
857 if (is_link_entry(sdbt))
858 sdbt = get_next_sdbt(sdbt);
859
860 cb->worker_entry = (unsigned long)sdbt;
861 }
862 }
863
864 static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
865 unsigned long *dear)
866 {
867 struct hws_data_entry *sample_data_ptr;
868 unsigned long *trailer;
869
870 trailer = trailer_entry_ptr(*sdbt);
871 if (dear) {
872 if (dear > trailer)
873 return;
874 trailer = dear;
875 }
876
877 sample_data_ptr = (struct hws_data_entry *)(*sdbt);
878
879 while ((unsigned long *)sample_data_ptr < trailer) {
880 struct pt_regs *regs = NULL;
881 struct task_struct *tsk = NULL;
882
883 /*
884 * Check sampling mode, 1 indicates basic (=customer) sampling
885 * mode.
886 */
887 if (sample_data_ptr->def != 1) {
888 /* sample slot is not yet written */
889 break;
890 } else {
891 /* make sure we don't use it twice,
892 * the next time the sampler will set it again */
893 sample_data_ptr->def = 0;
894 }
895
896 /* Get pt_regs. */
897 if (sample_data_ptr->P == 1) {
898 /* userspace sample */
899 unsigned int pid = sample_data_ptr->prim_asn;
900 if (!counter_config.user)
901 goto skip_sample;
902 rcu_read_lock();
903 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
904 if (tsk)
905 regs = task_pt_regs(tsk);
906 rcu_read_unlock();
907 } else {
908 /* kernelspace sample */
909 if (!counter_config.kernel)
910 goto skip_sample;
911 regs = task_pt_regs(current);
912 }
913
914 mutex_lock(&hws_sem);
915 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
916 !sample_data_ptr->P, tsk);
917 mutex_unlock(&hws_sem);
918 skip_sample:
919 sample_data_ptr++;
920 }
921 }
922
923 static void worker(struct work_struct *work)
924 {
925 unsigned int cpu;
926 int ext_params;
927 struct hws_cpu_buffer *cb;
928
929 cb = container_of(work, struct hws_cpu_buffer, worker);
930 cpu = smp_processor_id();
931 ext_params = atomic_xchg(&cb->ext_params, 0);
932
933 if (!cb->worker_entry)
934 worker_on_start(cpu);
935
936 if (worker_check_error(cpu, ext_params))
937 return;
938
939 if (!cb->finish)
940 worker_on_interrupt(cpu);
941
942 if (cb->finish)
943 worker_on_finish(cpu);
944 }
945
946 /**
947 * hwsampler_allocate() - allocate memory for the hardware sampler
948 * @sdbt: number of SDBTs per online CPU (must be > 0)
949 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
950 *
951 * Returns 0 on success, !0 on failure.
952 */
953 int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
954 {
955 int cpu, rc;
956 mutex_lock(&hws_sem);
957
958 rc = -EINVAL;
959 if (hws_state != HWS_DEALLOCATED)
960 goto allocate_exit;
961
962 if (sdbt < 1)
963 goto allocate_exit;
964
965 if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
966 goto allocate_exit;
967
968 num_sdbt = sdbt;
969 num_sdb = sdb;
970
971 oom_killer_was_active = 0;
972 register_oom_notifier(&hws_oom_notifier);
973
974 for_each_online_cpu(cpu) {
975 if (allocate_sdbt(cpu)) {
976 unregister_oom_notifier(&hws_oom_notifier);
977 goto allocate_error;
978 }
979 }
980 unregister_oom_notifier(&hws_oom_notifier);
981 if (oom_killer_was_active)
982 goto allocate_error;
983
984 hws_state = HWS_STOPPED;
985 rc = 0;
986
987 allocate_exit:
988 mutex_unlock(&hws_sem);
989 return rc;
990
991 allocate_error:
992 rc = -ENOMEM;
993 printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
994 goto allocate_exit;
995 }
996
997 /**
998 * hwsampler_deallocate() - deallocate hardware sampler memory
999 *
1000 * Returns 0 on success, !0 on failure.
1001 */
1002 int hwsampler_deallocate(void)
1003 {
1004 int rc;
1005
1006 mutex_lock(&hws_sem);
1007
1008 rc = -EINVAL;
1009 if (hws_state != HWS_STOPPED)
1010 goto deallocate_exit;
1011
1012 ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1013 deallocate_sdbt();
1014
1015 hws_state = HWS_DEALLOCATED;
1016 rc = 0;
1017
1018 deallocate_exit:
1019 mutex_unlock(&hws_sem);
1020
1021 return rc;
1022 }
1023
1024 unsigned long hwsampler_query_min_interval(void)
1025 {
1026 return min_sampler_rate;
1027 }
1028
1029 unsigned long hwsampler_query_max_interval(void)
1030 {
1031 return max_sampler_rate;
1032 }
1033
1034 unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
1035 {
1036 struct hws_cpu_buffer *cb;
1037
1038 cb = &per_cpu(sampler_cpu_buffer, cpu);
1039
1040 return cb->sample_overflow;
1041 }
1042
1043 int hwsampler_setup(void)
1044 {
1045 int rc;
1046 int cpu;
1047 struct hws_cpu_buffer *cb;
1048
1049 mutex_lock(&hws_sem);
1050
1051 rc = -EINVAL;
1052 if (hws_state)
1053 goto setup_exit;
1054
1055 hws_state = HWS_INIT;
1056
1057 init_all_cpu_buffers();
1058
1059 rc = check_hardware_prerequisites();
1060 if (rc)
1061 goto setup_exit;
1062
1063 rc = check_qsi_on_setup();
1064 if (rc)
1065 goto setup_exit;
1066
1067 rc = -EINVAL;
1068 hws_wq = create_workqueue("hwsampler");
1069 if (!hws_wq)
1070 goto setup_exit;
1071
1072 register_cpu_notifier(&hws_cpu_notifier);
1073
1074 for_each_online_cpu(cpu) {
1075 cb = &per_cpu(sampler_cpu_buffer, cpu);
1076 INIT_WORK(&cb->worker, worker);
1077 rc = smp_ctl_qsi(cpu);
1078 WARN_ON(rc);
1079 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1080 if (min_sampler_rate) {
1081 printk(KERN_WARNING
1082 "hwsampler: different min sampler rate values.\n");
1083 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1084 min_sampler_rate =
1085 cb->qsi.min_sampl_rate;
1086 } else
1087 min_sampler_rate = cb->qsi.min_sampl_rate;
1088 }
1089 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1090 if (max_sampler_rate) {
1091 printk(KERN_WARNING
1092 "hwsampler: different max sampler rate values.\n");
1093 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1094 max_sampler_rate =
1095 cb->qsi.max_sampl_rate;
1096 } else
1097 max_sampler_rate = cb->qsi.max_sampl_rate;
1098 }
1099 }
1100 register_external_interrupt(0x1407, hws_ext_handler);
1101
1102 hws_state = HWS_DEALLOCATED;
1103 rc = 0;
1104
1105 setup_exit:
1106 mutex_unlock(&hws_sem);
1107 return rc;
1108 }
1109
1110 int hwsampler_shutdown(void)
1111 {
1112 int rc;
1113
1114 mutex_lock(&hws_sem);
1115
1116 rc = -EINVAL;
1117 if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1118 mutex_unlock(&hws_sem);
1119
1120 if (hws_wq)
1121 flush_workqueue(hws_wq);
1122
1123 mutex_lock(&hws_sem);
1124
1125 if (hws_state == HWS_STOPPED) {
1126 ctl_clear_bit(0, 5); /* set bit 58 CR0 off */
1127 deallocate_sdbt();
1128 }
1129 if (hws_wq) {
1130 destroy_workqueue(hws_wq);
1131 hws_wq = NULL;
1132 }
1133
1134 unregister_external_interrupt(0x1407, hws_ext_handler);
1135 hws_state = HWS_INIT;
1136 rc = 0;
1137 }
1138 mutex_unlock(&hws_sem);
1139
1140 unregister_cpu_notifier(&hws_cpu_notifier);
1141
1142 return rc;
1143 }
1144
1145 /**
1146 * hwsampler_start_all() - start hardware sampling on all online CPUs
1147 * @rate: specifies the used interval when samples are taken
1148 *
1149 * Returns 0 on success, !0 on failure.
1150 */
1151 int hwsampler_start_all(unsigned long rate)
1152 {
1153 int rc, cpu;
1154
1155 mutex_lock(&hws_sem);
1156
1157 hws_oom = 0;
1158
1159 rc = -EINVAL;
1160 if (hws_state != HWS_STOPPED)
1161 goto start_all_exit;
1162
1163 interval = rate;
1164
1165 /* fail if rate is not valid */
1166 if (interval < min_sampler_rate || interval > max_sampler_rate)
1167 goto start_all_exit;
1168
1169 rc = check_qsi_on_start();
1170 if (rc)
1171 goto start_all_exit;
1172
1173 rc = prepare_cpu_buffers();
1174 if (rc)
1175 goto start_all_exit;
1176
1177 for_each_online_cpu(cpu) {
1178 rc = start_sampling(cpu);
1179 if (rc)
1180 break;
1181 }
1182 if (rc) {
1183 for_each_online_cpu(cpu) {
1184 stop_sampling(cpu);
1185 }
1186 goto start_all_exit;
1187 }
1188 hws_state = HWS_STARTED;
1189 rc = 0;
1190
1191 start_all_exit:
1192 mutex_unlock(&hws_sem);
1193
1194 if (rc)
1195 return rc;
1196
1197 register_oom_notifier(&hws_oom_notifier);
1198 hws_oom = 1;
1199 hws_flush_all = 0;
1200 /* now let them in, 1407 CPUMF external interrupts */
1201 ctl_set_bit(0, 5); /* set CR0 bit 58 */
1202
1203 return 0;
1204 }
1205
1206 /**
1207 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1208 *
1209 * Returns 0 on success, !0 on failure.
1210 */
1211 int hwsampler_stop_all(void)
1212 {
1213 int tmp_rc, rc, cpu;
1214 struct hws_cpu_buffer *cb;
1215
1216 mutex_lock(&hws_sem);
1217
1218 rc = 0;
1219 if (hws_state == HWS_INIT) {
1220 mutex_unlock(&hws_sem);
1221 return rc;
1222 }
1223 hws_state = HWS_STOPPING;
1224 mutex_unlock(&hws_sem);
1225
1226 for_each_online_cpu(cpu) {
1227 cb = &per_cpu(sampler_cpu_buffer, cpu);
1228 cb->stop_mode = 1;
1229 tmp_rc = stop_sampling(cpu);
1230 if (tmp_rc)
1231 rc = tmp_rc;
1232 }
1233
1234 if (hws_wq)
1235 flush_workqueue(hws_wq);
1236
1237 mutex_lock(&hws_sem);
1238 if (hws_oom) {
1239 unregister_oom_notifier(&hws_oom_notifier);
1240 hws_oom = 0;
1241 }
1242 hws_state = HWS_STOPPED;
1243 mutex_unlock(&hws_sem);
1244
1245 return rc;
1246 }