?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cris-mirror.git] / tools / perf / util / intel-pt.c
blob3773d9c54f45e60cc9e293f0a1622f660e416561
1 /*
2 * intel_pt.c: Intel Processor Trace support
3 * Copyright (c) 2013-2015, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16 #include <inttypes.h>
17 #include <stdio.h>
18 #include <stdbool.h>
19 #include <errno.h>
20 #include <linux/kernel.h>
21 #include <linux/types.h>
22
23 #include "../perf.h"
24 #include "session.h"
25 #include "machine.h"
26 #include "memswap.h"
27 #include "sort.h"
28 #include "tool.h"
29 #include "event.h"
30 #include "evlist.h"
31 #include "evsel.h"
32 #include "map.h"
33 #include "color.h"
34 #include "util.h"
35 #include "thread.h"
36 #include "thread-stack.h"
37 #include "symbol.h"
38 #include "callchain.h"
39 #include "dso.h"
40 #include "debug.h"
41 #include "auxtrace.h"
42 #include "tsc.h"
43 #include "intel-pt.h"
44 #include "config.h"
45
46 #include "intel-pt-decoder/intel-pt-log.h"
47 #include "intel-pt-decoder/intel-pt-decoder.h"
48 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
49 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
50
51 #define MAX_TIMESTAMP (~0ULL)
52
53 struct intel_pt {
54 struct auxtrace auxtrace;
55 struct auxtrace_queues queues;
56 struct auxtrace_heap heap;
57 u32 auxtrace_type;
58 struct perf_session *session;
59 struct machine *machine;
60 struct perf_evsel *switch_evsel;
61 struct thread *unknown_thread;
62 bool timeless_decoding;
63 bool sampling_mode;
64 bool snapshot_mode;
65 bool per_cpu_mmaps;
66 bool have_tsc;
67 bool data_queued;
68 bool est_tsc;
69 bool sync_switch;
70 bool mispred_all;
71 int have_sched_switch;
72 u32 pmu_type;
73 u64 kernel_start;
74 u64 switch_ip;
75 u64 ptss_ip;
76
77 struct perf_tsc_conversion tc;
78 bool cap_user_time_zero;
79
80 struct itrace_synth_opts synth_opts;
81
82 bool sample_instructions;
83 u64 instructions_sample_type;
84 u64 instructions_id;
85
86 bool sample_branches;
87 u32 branches_filter;
88 u64 branches_sample_type;
89 u64 branches_id;
90
91 bool sample_transactions;
92 u64 transactions_sample_type;
93 u64 transactions_id;
94
95 bool sample_ptwrites;
96 u64 ptwrites_sample_type;
97 u64 ptwrites_id;
98
99 bool sample_pwr_events;
100 u64 pwr_events_sample_type;
101 u64 mwait_id;
102 u64 pwre_id;
103 u64 exstop_id;
104 u64 pwrx_id;
105 u64 cbr_id;
106
107 u64 tsc_bit;
108 u64 mtc_bit;
109 u64 mtc_freq_bits;
110 u32 tsc_ctc_ratio_n;
111 u32 tsc_ctc_ratio_d;
112 u64 cyc_bit;
113 u64 noretcomp_bit;
114 unsigned max_non_turbo_ratio;
115 unsigned cbr2khz;
116
117 unsigned long num_events;
118
119 char *filter;
120 struct addr_filters filts;
121 };
122
123 enum switch_state {
124 INTEL_PT_SS_NOT_TRACING,
125 INTEL_PT_SS_UNKNOWN,
126 INTEL_PT_SS_TRACING,
127 INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
128 INTEL_PT_SS_EXPECTING_SWITCH_IP,
129 };
130
131 struct intel_pt_queue {
132 struct intel_pt *pt;
133 unsigned int queue_nr;
134 struct auxtrace_buffer *buffer;
135 void *decoder;
136 const struct intel_pt_state *state;
137 struct ip_callchain *chain;
138 struct branch_stack *last_branch;
139 struct branch_stack *last_branch_rb;
140 size_t last_branch_pos;
141 union perf_event *event_buf;
142 bool on_heap;
143 bool stop;
144 bool step_through_buffers;
145 bool use_buffer_pid_tid;
146 pid_t pid, tid;
147 int cpu;
148 int switch_state;
149 pid_t next_tid;
150 struct thread *thread;
151 bool exclude_kernel;
152 bool have_sample;
153 u64 time;
154 u64 timestamp;
155 u32 flags;
156 u16 insn_len;
157 u64 last_insn_cnt;
158 char insn[INTEL_PT_INSN_BUF_SZ];
159 };
160
161 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
162 unsigned char *buf, size_t len)
163 {
164 struct intel_pt_pkt packet;
165 size_t pos = 0;
166 int ret, pkt_len, i;
167 char desc[INTEL_PT_PKT_DESC_MAX];
168 const char *color = PERF_COLOR_BLUE;
169
170 color_fprintf(stdout, color,
171 ". ... Intel Processor Trace data: size %zu bytes\n",
172 len);
173
174 while (len) {
175 ret = intel_pt_get_packet(buf, len, &packet);
176 if (ret > 0)
177 pkt_len = ret;
178 else
179 pkt_len = 1;
180 printf(".");
181 color_fprintf(stdout, color, " %08x: ", pos);
182 for (i = 0; i < pkt_len; i++)
183 color_fprintf(stdout, color, " %02x", buf[i]);
184 for (; i < 16; i++)
185 color_fprintf(stdout, color, " ");
186 if (ret > 0) {
187 ret = intel_pt_pkt_desc(&packet, desc,
188 INTEL_PT_PKT_DESC_MAX);
189 if (ret > 0)
190 color_fprintf(stdout, color, " %s\n", desc);
191 } else {
192 color_fprintf(stdout, color, " Bad packet!\n");
193 }
194 pos += pkt_len;
195 buf += pkt_len;
196 len -= pkt_len;
197 }
198 }
199
200 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
201 size_t len)
202 {
203 printf(".\n");
204 intel_pt_dump(pt, buf, len);
205 }
206
207 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
208 struct auxtrace_buffer *b)
209 {
210 void *start;
211
212 start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
213 pt->have_tsc);
214 if (!start)
215 return -EINVAL;
216 b->use_size = b->data + b->size - start;
217 b->use_data = start;
218 return 0;
219 }
220
221 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
222 struct auxtrace_queue *queue,
223 struct auxtrace_buffer *buffer)
224 {
225 if (queue->cpu == -1 && buffer->cpu != -1)
226 ptq->cpu = buffer->cpu;
227
228 ptq->pid = buffer->pid;
229 ptq->tid = buffer->tid;
230
231 intel_pt_log("queue %u cpu %d pid %d tid %d\n",
232 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
233
234 thread__zput(ptq->thread);
235
236 if (ptq->tid != -1) {
237 if (ptq->pid != -1)
238 ptq->thread = machine__findnew_thread(ptq->pt->machine,
239 ptq->pid,
240 ptq->tid);
241 else
242 ptq->thread = machine__find_thread(ptq->pt->machine, -1,
243 ptq->tid);
244 }
245 }
246
247 /* This function assumes data is processed sequentially only */
248 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
249 {
250 struct intel_pt_queue *ptq = data;
251 struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
252 struct auxtrace_queue *queue;
253
254 if (ptq->stop) {
255 b->len = 0;
256 return 0;
257 }
258
259 queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
260 next:
261 buffer = auxtrace_buffer__next(queue, buffer);
262 if (!buffer) {
263 if (old_buffer)
264 auxtrace_buffer__drop_data(old_buffer);
265 b->len = 0;
266 return 0;
267 }
268
269 ptq->buffer = buffer;
270
271 if (!buffer->data) {
272 int fd = perf_data__fd(ptq->pt->session->data);
273
274 buffer->data = auxtrace_buffer__get_data(buffer, fd);
275 if (!buffer->data)
276 return -ENOMEM;
277 }
278
279 if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
280 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
281 return -ENOMEM;
282
283 if (buffer->use_data) {
284 b->len = buffer->use_size;
285 b->buf = buffer->use_data;
286 } else {
287 b->len = buffer->size;
288 b->buf = buffer->data;
289 }
290 b->ref_timestamp = buffer->reference;
291
292 /*
293 * If in snapshot mode and the buffer has no usable data, get next
294 * buffer and again check overlap against old_buffer.
295 */
296 if (ptq->pt->snapshot_mode && !b->len)
297 goto next;
298
299 if (old_buffer)
300 auxtrace_buffer__drop_data(old_buffer);
301
302 if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
303 !buffer->consecutive)) {
304 b->consecutive = false;
305 b->trace_nr = buffer->buffer_nr + 1;
306 } else {
307 b->consecutive = true;
308 }
309
310 if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
311 ptq->tid != buffer->tid))
312 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
313
314 if (ptq->step_through_buffers)
315 ptq->stop = true;
316
317 if (!b->len)
318 return intel_pt_get_trace(b, data);
319
320 return 0;
321 }
322
323 struct intel_pt_cache_entry {
324 struct auxtrace_cache_entry entry;
325 u64 insn_cnt;
326 u64 byte_cnt;
327 enum intel_pt_insn_op op;
328 enum intel_pt_insn_branch branch;
329 int length;
330 int32_t rel;
331 char insn[INTEL_PT_INSN_BUF_SZ];
332 };
333
334 static int intel_pt_config_div(const char *var, const char *value, void *data)
335 {
336 int *d = data;
337 long val;
338
339 if (!strcmp(var, "intel-pt.cache-divisor")) {
340 val = strtol(value, NULL, 0);
341 if (val > 0 && val <= INT_MAX)
342 *d = val;
343 }
344
345 return 0;
346 }
347
348 static int intel_pt_cache_divisor(void)
349 {
350 static int d;
351
352 if (d)
353 return d;
354
355 perf_config(intel_pt_config_div, &d);
356
357 if (!d)
358 d = 64;
359
360 return d;
361 }
362
363 static unsigned int intel_pt_cache_size(struct dso *dso,
364 struct machine *machine)
365 {
366 off_t size;
367
368 size = dso__data_size(dso, machine);
369 size /= intel_pt_cache_divisor();
370 if (size < 1000)
371 return 10;
372 if (size > (1 << 21))
373 return 21;
374 return 32 - __builtin_clz(size);
375 }
376
377 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
378 struct machine *machine)
379 {
380 struct auxtrace_cache *c;
381 unsigned int bits;
382
383 if (dso->auxtrace_cache)
384 return dso->auxtrace_cache;
385
386 bits = intel_pt_cache_size(dso, machine);
387
388 /* Ignoring cache creation failure */
389 c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
390
391 dso->auxtrace_cache = c;
392
393 return c;
394 }
395
396 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
397 u64 offset, u64 insn_cnt, u64 byte_cnt,
398 struct intel_pt_insn *intel_pt_insn)
399 {
400 struct auxtrace_cache *c = intel_pt_cache(dso, machine);
401 struct intel_pt_cache_entry *e;
402 int err;
403
404 if (!c)
405 return -ENOMEM;
406
407 e = auxtrace_cache__alloc_entry(c);
408 if (!e)
409 return -ENOMEM;
410
411 e->insn_cnt = insn_cnt;
412 e->byte_cnt = byte_cnt;
413 e->op = intel_pt_insn->op;
414 e->branch = intel_pt_insn->branch;
415 e->length = intel_pt_insn->length;
416 e->rel = intel_pt_insn->rel;
417 memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
418
419 err = auxtrace_cache__add(c, offset, &e->entry);
420 if (err)
421 auxtrace_cache__free_entry(c, e);
422
423 return err;
424 }
425
426 static struct intel_pt_cache_entry *
427 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
428 {
429 struct auxtrace_cache *c = intel_pt_cache(dso, machine);
430
431 if (!c)
432 return NULL;
433
434 return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
435 }
436
437 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
438 uint64_t *insn_cnt_ptr, uint64_t *ip,
439 uint64_t to_ip, uint64_t max_insn_cnt,
440 void *data)
441 {
442 struct intel_pt_queue *ptq = data;
443 struct machine *machine = ptq->pt->machine;
444 struct thread *thread;
445 struct addr_location al;
446 unsigned char buf[INTEL_PT_INSN_BUF_SZ];
447 ssize_t len;
448 int x86_64;
449 u8 cpumode;
450 u64 offset, start_offset, start_ip;
451 u64 insn_cnt = 0;
452 bool one_map = true;
453
454 intel_pt_insn->length = 0;
455
456 if (to_ip && *ip == to_ip)
457 goto out_no_cache;
458
459 if (*ip >= ptq->pt->kernel_start)
460 cpumode = PERF_RECORD_MISC_KERNEL;
461 else
462 cpumode = PERF_RECORD_MISC_USER;
463
464 thread = ptq->thread;
465 if (!thread) {
466 if (cpumode != PERF_RECORD_MISC_KERNEL)
467 return -EINVAL;
468 thread = ptq->pt->unknown_thread;
469 }
470
471 while (1) {
472 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
473 if (!al.map || !al.map->dso)
474 return -EINVAL;
475
476 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
477 dso__data_status_seen(al.map->dso,
478 DSO_DATA_STATUS_SEEN_ITRACE))
479 return -ENOENT;
480
481 offset = al.map->map_ip(al.map, *ip);
482
483 if (!to_ip && one_map) {
484 struct intel_pt_cache_entry *e;
485
486 e = intel_pt_cache_lookup(al.map->dso, machine, offset);
487 if (e &&
488 (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
489 *insn_cnt_ptr = e->insn_cnt;
490 *ip += e->byte_cnt;
491 intel_pt_insn->op = e->op;
492 intel_pt_insn->branch = e->branch;
493 intel_pt_insn->length = e->length;
494 intel_pt_insn->rel = e->rel;
495 memcpy(intel_pt_insn->buf, e->insn,
496 INTEL_PT_INSN_BUF_SZ);
497 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
498 return 0;
499 }
500 }
501
502 start_offset = offset;
503 start_ip = *ip;
504
505 /* Load maps to ensure dso->is_64_bit has been updated */
506 map__load(al.map);
507
508 x86_64 = al.map->dso->is_64_bit;
509
510 while (1) {
511 len = dso__data_read_offset(al.map->dso, machine,
512 offset, buf,
513 INTEL_PT_INSN_BUF_SZ);
514 if (len <= 0)
515 return -EINVAL;
516
517 if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
518 return -EINVAL;
519
520 intel_pt_log_insn(intel_pt_insn, *ip);
521
522 insn_cnt += 1;
523
524 if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
525 goto out;
526
527 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
528 goto out_no_cache;
529
530 *ip += intel_pt_insn->length;
531
532 if (to_ip && *ip == to_ip)
533 goto out_no_cache;
534
535 if (*ip >= al.map->end)
536 break;
537
538 offset += intel_pt_insn->length;
539 }
540 one_map = false;
541 }
542 out:
543 *insn_cnt_ptr = insn_cnt;
544
545 if (!one_map)
546 goto out_no_cache;
547
548 /*
549 * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
550 * entries.
551 */
552 if (to_ip) {
553 struct intel_pt_cache_entry *e;
554
555 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
556 if (e)
557 return 0;
558 }
559
560 /* Ignore cache errors */
561 intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
562 *ip - start_ip, intel_pt_insn);
563
564 return 0;
565
566 out_no_cache:
567 *insn_cnt_ptr = insn_cnt;
568 return 0;
569 }
570
571 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
572 uint64_t offset, const char *filename)
573 {
574 struct addr_filter *filt;
575 bool have_filter = false;
576 bool hit_tracestop = false;
577 bool hit_filter = false;
578
579 list_for_each_entry(filt, &pt->filts.head, list) {
580 if (filt->start)
581 have_filter = true;
582
583 if ((filename && !filt->filename) ||
584 (!filename && filt->filename) ||
585 (filename && strcmp(filename, filt->filename)))
586 continue;
587
588 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
589 continue;
590
591 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
592 ip, offset, filename ? filename : "[kernel]",
593 filt->start ? "filter" : "stop",
594 filt->addr, filt->size);
595
596 if (filt->start)
597 hit_filter = true;
598 else
599 hit_tracestop = true;
600 }
601
602 if (!hit_tracestop && !hit_filter)
603 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
604 ip, offset, filename ? filename : "[kernel]");
605
606 return hit_tracestop || (have_filter && !hit_filter);
607 }
608
609 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
610 {
611 struct intel_pt_queue *ptq = data;
612 struct thread *thread;
613 struct addr_location al;
614 u8 cpumode;
615 u64 offset;
616
617 if (ip >= ptq->pt->kernel_start)
618 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
619
620 cpumode = PERF_RECORD_MISC_USER;
621
622 thread = ptq->thread;
623 if (!thread)
624 return -EINVAL;
625
626 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
627 if (!al.map || !al.map->dso)
628 return -EINVAL;
629
630 offset = al.map->map_ip(al.map, ip);
631
632 return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
633 al.map->dso->long_name);
634 }
635
636 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
637 {
638 return __intel_pt_pgd_ip(ip, data) > 0;
639 }
640
641 static bool intel_pt_get_config(struct intel_pt *pt,
642 struct perf_event_attr *attr, u64 *config)
643 {
644 if (attr->type == pt->pmu_type) {
645 if (config)
646 *config = attr->config;
647 return true;
648 }
649
650 return false;
651 }
652
653 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
654 {
655 struct perf_evsel *evsel;
656
657 evlist__for_each_entry(pt->session->evlist, evsel) {
658 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
659 !evsel->attr.exclude_kernel)
660 return false;
661 }
662 return true;
663 }
664
665 static bool intel_pt_return_compression(struct intel_pt *pt)
666 {
667 struct perf_evsel *evsel;
668 u64 config;
669
670 if (!pt->noretcomp_bit)
671 return true;
672
673 evlist__for_each_entry(pt->session->evlist, evsel) {
674 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
675 (config & pt->noretcomp_bit))
676 return false;
677 }
678 return true;
679 }
680
681 static bool intel_pt_branch_enable(struct intel_pt *pt)
682 {
683 struct perf_evsel *evsel;
684 u64 config;
685
686 evlist__for_each_entry(pt->session->evlist, evsel) {
687 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
688 (config & 1) && !(config & 0x2000))
689 return false;
690 }
691 return true;
692 }
693
694 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
695 {
696 struct perf_evsel *evsel;
697 unsigned int shift;
698 u64 config;
699
700 if (!pt->mtc_freq_bits)
701 return 0;
702
703 for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
704 config >>= 1;
705
706 evlist__for_each_entry(pt->session->evlist, evsel) {
707 if (intel_pt_get_config(pt, &evsel->attr, &config))
708 return (config & pt->mtc_freq_bits) >> shift;
709 }
710 return 0;
711 }
712
713 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
714 {
715 struct perf_evsel *evsel;
716 bool timeless_decoding = true;
717 u64 config;
718
719 if (!pt->tsc_bit || !pt->cap_user_time_zero)
720 return true;
721
722 evlist__for_each_entry(pt->session->evlist, evsel) {
723 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
724 return true;
725 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
726 if (config & pt->tsc_bit)
727 timeless_decoding = false;
728 else
729 return true;
730 }
731 }
732 return timeless_decoding;
733 }
734
735 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
736 {
737 struct perf_evsel *evsel;
738
739 evlist__for_each_entry(pt->session->evlist, evsel) {
740 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
741 !evsel->attr.exclude_kernel)
742 return true;
743 }
744 return false;
745 }
746
747 static bool intel_pt_have_tsc(struct intel_pt *pt)
748 {
749 struct perf_evsel *evsel;
750 bool have_tsc = false;
751 u64 config;
752
753 if (!pt->tsc_bit)
754 return false;
755
756 evlist__for_each_entry(pt->session->evlist, evsel) {
757 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
758 if (config & pt->tsc_bit)
759 have_tsc = true;
760 else
761 return false;
762 }
763 }
764 return have_tsc;
765 }
766
767 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
768 {
769 u64 quot, rem;
770
771 quot = ns / pt->tc.time_mult;
772 rem = ns % pt->tc.time_mult;
773 return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
774 pt->tc.time_mult;
775 }
776
777 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
778 unsigned int queue_nr)
779 {
780 struct intel_pt_params params = { .get_trace = 0, };
781 struct intel_pt_queue *ptq;
782
783 ptq = zalloc(sizeof(struct intel_pt_queue));
784 if (!ptq)
785 return NULL;
786
787 if (pt->synth_opts.callchain) {
788 size_t sz = sizeof(struct ip_callchain);
789
790 sz += pt->synth_opts.callchain_sz * sizeof(u64);
791 ptq->chain = zalloc(sz);
792 if (!ptq->chain)
793 goto out_free;
794 }
795
796 if (pt->synth_opts.last_branch) {
797 size_t sz = sizeof(struct branch_stack);
798
799 sz += pt->synth_opts.last_branch_sz *
800 sizeof(struct branch_entry);
801 ptq->last_branch = zalloc(sz);
802 if (!ptq->last_branch)
803 goto out_free;
804 ptq->last_branch_rb = zalloc(sz);
805 if (!ptq->last_branch_rb)
806 goto out_free;
807 }
808
809 ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
810 if (!ptq->event_buf)
811 goto out_free;
812
813 ptq->pt = pt;
814 ptq->queue_nr = queue_nr;
815 ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
816 ptq->pid = -1;
817 ptq->tid = -1;
818 ptq->cpu = -1;
819 ptq->next_tid = -1;
820
821 params.get_trace = intel_pt_get_trace;
822 params.walk_insn = intel_pt_walk_next_insn;
823 params.data = ptq;
824 params.return_compression = intel_pt_return_compression(pt);
825 params.branch_enable = intel_pt_branch_enable(pt);
826 params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
827 params.mtc_period = intel_pt_mtc_period(pt);
828 params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
829 params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
830
831 if (pt->filts.cnt > 0)
832 params.pgd_ip = intel_pt_pgd_ip;
833
834 if (pt->synth_opts.instructions) {
835 if (pt->synth_opts.period) {
836 switch (pt->synth_opts.period_type) {
837 case PERF_ITRACE_PERIOD_INSTRUCTIONS:
838 params.period_type =
839 INTEL_PT_PERIOD_INSTRUCTIONS;
840 params.period = pt->synth_opts.period;
841 break;
842 case PERF_ITRACE_PERIOD_TICKS:
843 params.period_type = INTEL_PT_PERIOD_TICKS;
844 params.period = pt->synth_opts.period;
845 break;
846 case PERF_ITRACE_PERIOD_NANOSECS:
847 params.period_type = INTEL_PT_PERIOD_TICKS;
848 params.period = intel_pt_ns_to_ticks(pt,
849 pt->synth_opts.period);
850 break;
851 default:
852 break;
853 }
854 }
855
856 if (!params.period) {
857 params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
858 params.period = 1;
859 }
860 }
861
862 ptq->decoder = intel_pt_decoder_new(&params);
863 if (!ptq->decoder)
864 goto out_free;
865
866 return ptq;
867
868 out_free:
869 zfree(&ptq->event_buf);
870 zfree(&ptq->last_branch);
871 zfree(&ptq->last_branch_rb);
872 zfree(&ptq->chain);
873 free(ptq);
874 return NULL;
875 }
876
877 static void intel_pt_free_queue(void *priv)
878 {
879 struct intel_pt_queue *ptq = priv;
880
881 if (!ptq)
882 return;
883 thread__zput(ptq->thread);
884 intel_pt_decoder_free(ptq->decoder);
885 zfree(&ptq->event_buf);
886 zfree(&ptq->last_branch);
887 zfree(&ptq->last_branch_rb);
888 zfree(&ptq->chain);
889 free(ptq);
890 }
891
892 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
893 struct auxtrace_queue *queue)
894 {
895 struct intel_pt_queue *ptq = queue->priv;
896
897 if (queue->tid == -1 || pt->have_sched_switch) {
898 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
899 thread__zput(ptq->thread);
900 }
901
902 if (!ptq->thread && ptq->tid != -1)
903 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
904
905 if (ptq->thread) {
906 ptq->pid = ptq->thread->pid_;
907 if (queue->cpu == -1)
908 ptq->cpu = ptq->thread->cpu;
909 }
910 }
911
912 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
913 {
914 if (ptq->state->flags & INTEL_PT_ABORT_TX) {
915 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
916 } else if (ptq->state->flags & INTEL_PT_ASYNC) {
917 if (ptq->state->to_ip)
918 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
919 PERF_IP_FLAG_ASYNC |
920 PERF_IP_FLAG_INTERRUPT;
921 else
922 ptq->flags = PERF_IP_FLAG_BRANCH |
923 PERF_IP_FLAG_TRACE_END;
924 ptq->insn_len = 0;
925 } else {
926 if (ptq->state->from_ip)
927 ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
928 else
929 ptq->flags = PERF_IP_FLAG_BRANCH |
930 PERF_IP_FLAG_TRACE_BEGIN;
931 if (ptq->state->flags & INTEL_PT_IN_TX)
932 ptq->flags |= PERF_IP_FLAG_IN_TX;
933 ptq->insn_len = ptq->state->insn_len;
934 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
935 }
936 }
937
938 static int intel_pt_setup_queue(struct intel_pt *pt,
939 struct auxtrace_queue *queue,
940 unsigned int queue_nr)
941 {
942 struct intel_pt_queue *ptq = queue->priv;
943
944 if (list_empty(&queue->head))
945 return 0;
946
947 if (!ptq) {
948 ptq = intel_pt_alloc_queue(pt, queue_nr);
949 if (!ptq)
950 return -ENOMEM;
951 queue->priv = ptq;
952
953 if (queue->cpu != -1)
954 ptq->cpu = queue->cpu;
955 ptq->tid = queue->tid;
956
957 if (pt->sampling_mode) {
958 if (pt->timeless_decoding)
959 ptq->step_through_buffers = true;
960 if (pt->timeless_decoding || !pt->have_sched_switch)
961 ptq->use_buffer_pid_tid = true;
962 }
963 }
964
965 if (!ptq->on_heap &&
966 (!pt->sync_switch ||
967 ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
968 const struct intel_pt_state *state;
969 int ret;
970
971 if (pt->timeless_decoding)
972 return 0;
973
974 intel_pt_log("queue %u getting timestamp\n", queue_nr);
975 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
976 queue_nr, ptq->cpu, ptq->pid, ptq->tid);
977 while (1) {
978 state = intel_pt_decode(ptq->decoder);
979 if (state->err) {
980 if (state->err == INTEL_PT_ERR_NODATA) {
981 intel_pt_log("queue %u has no timestamp\n",
982 queue_nr);
983 return 0;
984 }
985 continue;
986 }
987 if (state->timestamp)
988 break;
989 }
990
991 ptq->timestamp = state->timestamp;
992 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
993 queue_nr, ptq->timestamp);
994 ptq->state = state;
995 ptq->have_sample = true;
996 intel_pt_sample_flags(ptq);
997 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
998 if (ret)
999 return ret;
1000 ptq->on_heap = true;
1001 }
1002
1003 return 0;
1004 }
1005
1006 static int intel_pt_setup_queues(struct intel_pt *pt)
1007 {
1008 unsigned int i;
1009 int ret;
1010
1011 for (i = 0; i < pt->queues.nr_queues; i++) {
1012 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1013 if (ret)
1014 return ret;
1015 }
1016 return 0;
1017 }
1018
1019 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1020 {
1021 struct branch_stack *bs_src = ptq->last_branch_rb;
1022 struct branch_stack *bs_dst = ptq->last_branch;
1023 size_t nr = 0;
1024
1025 bs_dst->nr = bs_src->nr;
1026
1027 if (!bs_src->nr)
1028 return;
1029
1030 nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1031 memcpy(&bs_dst->entries[0],
1032 &bs_src->entries[ptq->last_branch_pos],
1033 sizeof(struct branch_entry) * nr);
1034
1035 if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1036 memcpy(&bs_dst->entries[nr],
1037 &bs_src->entries[0],
1038 sizeof(struct branch_entry) * ptq->last_branch_pos);
1039 }
1040 }
1041
1042 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1043 {
1044 ptq->last_branch_pos = 0;
1045 ptq->last_branch_rb->nr = 0;
1046 }
1047
1048 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1049 {
1050 const struct intel_pt_state *state = ptq->state;
1051 struct branch_stack *bs = ptq->last_branch_rb;
1052 struct branch_entry *be;
1053
1054 if (!ptq->last_branch_pos)
1055 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1056
1057 ptq->last_branch_pos -= 1;
1058
1059 be = &bs->entries[ptq->last_branch_pos];
1060 be->from = state->from_ip;
1061 be->to = state->to_ip;
1062 be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1063 be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1064 /* No support for mispredict */
1065 be->flags.mispred = ptq->pt->mispred_all;
1066
1067 if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1068 bs->nr += 1;
1069 }
1070
1071 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1072 {
1073 return pt->synth_opts.initial_skip &&
1074 pt->num_events++ < pt->synth_opts.initial_skip;
1075 }
1076
1077 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1078 struct intel_pt_queue *ptq,
1079 union perf_event *event,
1080 struct perf_sample *sample)
1081 {
1082 event->sample.header.type = PERF_RECORD_SAMPLE;
1083 event->sample.header.misc = PERF_RECORD_MISC_USER;
1084 event->sample.header.size = sizeof(struct perf_event_header);
1085
1086 if (!pt->timeless_decoding)
1087 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1088
1089 sample->cpumode = PERF_RECORD_MISC_USER;
1090 sample->ip = ptq->state->from_ip;
1091 sample->pid = ptq->pid;
1092 sample->tid = ptq->tid;
1093 sample->addr = ptq->state->to_ip;
1094 sample->period = 1;
1095 sample->cpu = ptq->cpu;
1096 sample->flags = ptq->flags;
1097 sample->insn_len = ptq->insn_len;
1098 memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1099 }
1100
1101 static int intel_pt_inject_event(union perf_event *event,
1102 struct perf_sample *sample, u64 type)
1103 {
1104 event->header.size = perf_event__sample_event_size(sample, type, 0);
1105 return perf_event__synthesize_sample(event, type, 0, sample);
1106 }
1107
1108 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1109 union perf_event *event,
1110 struct perf_sample *sample, u64 type)
1111 {
1112 if (!pt->synth_opts.inject)
1113 return 0;
1114
1115 return intel_pt_inject_event(event, sample, type);
1116 }
1117
1118 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1119 union perf_event *event,
1120 struct perf_sample *sample, u64 type)
1121 {
1122 int ret;
1123
1124 ret = intel_pt_opt_inject(pt, event, sample, type);
1125 if (ret)
1126 return ret;
1127
1128 ret = perf_session__deliver_synth_event(pt->session, event, sample);
1129 if (ret)
1130 pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1131
1132 return ret;
1133 }
1134
1135 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1136 {
1137 struct intel_pt *pt = ptq->pt;
1138 union perf_event *event = ptq->event_buf;
1139 struct perf_sample sample = { .ip = 0, };
1140 struct dummy_branch_stack {
1141 u64 nr;
1142 struct branch_entry entries;
1143 } dummy_bs;
1144
1145 if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1146 return 0;
1147
1148 if (intel_pt_skip_event(pt))
1149 return 0;
1150
1151 intel_pt_prep_b_sample(pt, ptq, event, &sample);
1152
1153 sample.id = ptq->pt->branches_id;
1154 sample.stream_id = ptq->pt->branches_id;
1155
1156 /*
1157 * perf report cannot handle events without a branch stack when using
1158 * SORT_MODE__BRANCH so make a dummy one.
1159 */
1160 if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1161 dummy_bs = (struct dummy_branch_stack){
1162 .nr = 1,
1163 .entries = {
1164 .from = sample.ip,
1165 .to = sample.addr,
1166 },
1167 };
1168 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1169 }
1170
1171 return intel_pt_deliver_synth_b_event(pt, event, &sample,
1172 pt->branches_sample_type);
1173 }
1174
1175 static void intel_pt_prep_sample(struct intel_pt *pt,
1176 struct intel_pt_queue *ptq,
1177 union perf_event *event,
1178 struct perf_sample *sample)
1179 {
1180 intel_pt_prep_b_sample(pt, ptq, event, sample);
1181
1182 if (pt->synth_opts.callchain) {
1183 thread_stack__sample(ptq->thread, ptq->chain,
1184 pt->synth_opts.callchain_sz, sample->ip);
1185 sample->callchain = ptq->chain;
1186 }
1187
1188 if (pt->synth_opts.last_branch) {
1189 intel_pt_copy_last_branch_rb(ptq);
1190 sample->branch_stack = ptq->last_branch;
1191 }
1192 }
1193
1194 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1195 struct intel_pt_queue *ptq,
1196 union perf_event *event,
1197 struct perf_sample *sample,
1198 u64 type)
1199 {
1200 int ret;
1201
1202 ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1203
1204 if (pt->synth_opts.last_branch)
1205 intel_pt_reset_last_branch_rb(ptq);
1206
1207 return ret;
1208 }
1209
1210 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1211 {
1212 struct intel_pt *pt = ptq->pt;
1213 union perf_event *event = ptq->event_buf;
1214 struct perf_sample sample = { .ip = 0, };
1215
1216 if (intel_pt_skip_event(pt))
1217 return 0;
1218
1219 intel_pt_prep_sample(pt, ptq, event, &sample);
1220
1221 sample.id = ptq->pt->instructions_id;
1222 sample.stream_id = ptq->pt->instructions_id;
1223 sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1224
1225 ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1226
1227 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1228 pt->instructions_sample_type);
1229 }
1230
1231 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1232 {
1233 struct intel_pt *pt = ptq->pt;
1234 union perf_event *event = ptq->event_buf;
1235 struct perf_sample sample = { .ip = 0, };
1236
1237 if (intel_pt_skip_event(pt))
1238 return 0;
1239
1240 intel_pt_prep_sample(pt, ptq, event, &sample);
1241
1242 sample.id = ptq->pt->transactions_id;
1243 sample.stream_id = ptq->pt->transactions_id;
1244
1245 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1246 pt->transactions_sample_type);
1247 }
1248
1249 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1250 struct intel_pt_queue *ptq,
1251 union perf_event *event,
1252 struct perf_sample *sample)
1253 {
1254 intel_pt_prep_sample(pt, ptq, event, sample);
1255
1256 /*
1257 * Zero IP is used to mean "trace start" but that is not the case for
1258 * power or PTWRITE events with no IP, so clear the flags.
1259 */
1260 if (!sample->ip)
1261 sample->flags = 0;
1262 }
1263
1264 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1265 {
1266 struct intel_pt *pt = ptq->pt;
1267 union perf_event *event = ptq->event_buf;
1268 struct perf_sample sample = { .ip = 0, };
1269 struct perf_synth_intel_ptwrite raw;
1270
1271 if (intel_pt_skip_event(pt))
1272 return 0;
1273
1274 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1275
1276 sample.id = ptq->pt->ptwrites_id;
1277 sample.stream_id = ptq->pt->ptwrites_id;
1278
1279 raw.flags = 0;
1280 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1281 raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1282
1283 sample.raw_size = perf_synth__raw_size(raw);
1284 sample.raw_data = perf_synth__raw_data(&raw);
1285
1286 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1287 pt->ptwrites_sample_type);
1288 }
1289
1290 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1291 {
1292 struct intel_pt *pt = ptq->pt;
1293 union perf_event *event = ptq->event_buf;
1294 struct perf_sample sample = { .ip = 0, };
1295 struct perf_synth_intel_cbr raw;
1296 u32 flags;
1297
1298 if (intel_pt_skip_event(pt))
1299 return 0;
1300
1301 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1302
1303 sample.id = ptq->pt->cbr_id;
1304 sample.stream_id = ptq->pt->cbr_id;
1305
1306 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1307 raw.flags = cpu_to_le32(flags);
1308 raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1309 raw.reserved3 = 0;
1310
1311 sample.raw_size = perf_synth__raw_size(raw);
1312 sample.raw_data = perf_synth__raw_data(&raw);
1313
1314 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1315 pt->pwr_events_sample_type);
1316 }
1317
1318 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1319 {
1320 struct intel_pt *pt = ptq->pt;
1321 union perf_event *event = ptq->event_buf;
1322 struct perf_sample sample = { .ip = 0, };
1323 struct perf_synth_intel_mwait raw;
1324
1325 if (intel_pt_skip_event(pt))
1326 return 0;
1327
1328 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1329
1330 sample.id = ptq->pt->mwait_id;
1331 sample.stream_id = ptq->pt->mwait_id;
1332
1333 raw.reserved = 0;
1334 raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1335
1336 sample.raw_size = perf_synth__raw_size(raw);
1337 sample.raw_data = perf_synth__raw_data(&raw);
1338
1339 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1340 pt->pwr_events_sample_type);
1341 }
1342
1343 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1344 {
1345 struct intel_pt *pt = ptq->pt;
1346 union perf_event *event = ptq->event_buf;
1347 struct perf_sample sample = { .ip = 0, };
1348 struct perf_synth_intel_pwre raw;
1349
1350 if (intel_pt_skip_event(pt))
1351 return 0;
1352
1353 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1354
1355 sample.id = ptq->pt->pwre_id;
1356 sample.stream_id = ptq->pt->pwre_id;
1357
1358 raw.reserved = 0;
1359 raw.payload = cpu_to_le64(ptq->state->pwre_payload);
1360
1361 sample.raw_size = perf_synth__raw_size(raw);
1362 sample.raw_data = perf_synth__raw_data(&raw);
1363
1364 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1365 pt->pwr_events_sample_type);
1366 }
1367
1368 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1369 {
1370 struct intel_pt *pt = ptq->pt;
1371 union perf_event *event = ptq->event_buf;
1372 struct perf_sample sample = { .ip = 0, };
1373 struct perf_synth_intel_exstop raw;
1374
1375 if (intel_pt_skip_event(pt))
1376 return 0;
1377
1378 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1379
1380 sample.id = ptq->pt->exstop_id;
1381 sample.stream_id = ptq->pt->exstop_id;
1382
1383 raw.flags = 0;
1384 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1385
1386 sample.raw_size = perf_synth__raw_size(raw);
1387 sample.raw_data = perf_synth__raw_data(&raw);
1388
1389 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1390 pt->pwr_events_sample_type);
1391 }
1392
1393 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1394 {
1395 struct intel_pt *pt = ptq->pt;
1396 union perf_event *event = ptq->event_buf;
1397 struct perf_sample sample = { .ip = 0, };
1398 struct perf_synth_intel_pwrx raw;
1399
1400 if (intel_pt_skip_event(pt))
1401 return 0;
1402
1403 intel_pt_prep_p_sample(pt, ptq, event, &sample);
1404
1405 sample.id = ptq->pt->pwrx_id;
1406 sample.stream_id = ptq->pt->pwrx_id;
1407
1408 raw.reserved = 0;
1409 raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
1410
1411 sample.raw_size = perf_synth__raw_size(raw);
1412 sample.raw_data = perf_synth__raw_data(&raw);
1413
1414 return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1415 pt->pwr_events_sample_type);
1416 }
1417
1418 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1419 pid_t pid, pid_t tid, u64 ip)
1420 {
1421 union perf_event event;
1422 char msg[MAX_AUXTRACE_ERROR_MSG];
1423 int err;
1424
1425 intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1426
1427 auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1428 code, cpu, pid, tid, ip, msg);
1429
1430 err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1431 if (err)
1432 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1433 err);
1434
1435 return err;
1436 }
1437
1438 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1439 {
1440 struct auxtrace_queue *queue;
1441 pid_t tid = ptq->next_tid;
1442 int err;
1443
1444 if (tid == -1)
1445 return 0;
1446
1447 intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1448
1449 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1450
1451 queue = &pt->queues.queue_array[ptq->queue_nr];
1452 intel_pt_set_pid_tid_cpu(pt, queue);
1453
1454 ptq->next_tid = -1;
1455
1456 return err;
1457 }
1458
1459 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1460 {
1461 struct intel_pt *pt = ptq->pt;
1462
1463 return ip == pt->switch_ip &&
1464 (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1465 !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1466 PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1467 }
1468
1469 #define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP | INTEL_PT_PWR_ENTRY | \
1470 INTEL_PT_EX_STOP | INTEL_PT_PWR_EXIT | \
1471 INTEL_PT_CBR_CHG)
1472
1473 static int intel_pt_sample(struct intel_pt_queue *ptq)
1474 {
1475 const struct intel_pt_state *state = ptq->state;
1476 struct intel_pt *pt = ptq->pt;
1477 int err;
1478
1479 if (!ptq->have_sample)
1480 return 0;
1481
1482 ptq->have_sample = false;
1483
1484 if (pt->sample_pwr_events && (state->type & INTEL_PT_PWR_EVT)) {
1485 if (state->type & INTEL_PT_CBR_CHG) {
1486 err = intel_pt_synth_cbr_sample(ptq);
1487 if (err)
1488 return err;
1489 }
1490 if (state->type & INTEL_PT_MWAIT_OP) {
1491 err = intel_pt_synth_mwait_sample(ptq);
1492 if (err)
1493 return err;
1494 }
1495 if (state->type & INTEL_PT_PWR_ENTRY) {
1496 err = intel_pt_synth_pwre_sample(ptq);
1497 if (err)
1498 return err;
1499 }
1500 if (state->type & INTEL_PT_EX_STOP) {
1501 err = intel_pt_synth_exstop_sample(ptq);
1502 if (err)
1503 return err;
1504 }
1505 if (state->type & INTEL_PT_PWR_EXIT) {
1506 err = intel_pt_synth_pwrx_sample(ptq);
1507 if (err)
1508 return err;
1509 }
1510 }
1511
1512 if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1513 err = intel_pt_synth_instruction_sample(ptq);
1514 if (err)
1515 return err;
1516 }
1517
1518 if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
1519 err = intel_pt_synth_transaction_sample(ptq);
1520 if (err)
1521 return err;
1522 }
1523
1524 if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
1525 err = intel_pt_synth_ptwrite_sample(ptq);
1526 if (err)
1527 return err;
1528 }
1529
1530 if (!(state->type & INTEL_PT_BRANCH))
1531 return 0;
1532
1533 if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1534 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1535 state->to_ip, ptq->insn_len,
1536 state->trace_nr);
1537 else
1538 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1539
1540 if (pt->sample_branches) {
1541 err = intel_pt_synth_branch_sample(ptq);
1542 if (err)
1543 return err;
1544 }
1545
1546 if (pt->synth_opts.last_branch)
1547 intel_pt_update_last_branch_rb(ptq);
1548
1549 if (!pt->sync_switch)
1550 return 0;
1551
1552 if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1553 switch (ptq->switch_state) {
1554 case INTEL_PT_SS_UNKNOWN:
1555 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1556 err = intel_pt_next_tid(pt, ptq);
1557 if (err)
1558 return err;
1559 ptq->switch_state = INTEL_PT_SS_TRACING;
1560 break;
1561 default:
1562 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1563 return 1;
1564 }
1565 } else if (!state->to_ip) {
1566 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1567 } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1568 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1569 } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1570 state->to_ip == pt->ptss_ip &&
1571 (ptq->flags & PERF_IP_FLAG_CALL)) {
1572 ptq->switch_state = INTEL_PT_SS_TRACING;
1573 }
1574
1575 return 0;
1576 }
1577
1578 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1579 {
1580 struct machine *machine = pt->machine;
1581 struct map *map;
1582 struct symbol *sym, *start;
1583 u64 ip, switch_ip = 0;
1584 const char *ptss;
1585
1586 if (ptss_ip)
1587 *ptss_ip = 0;
1588
1589 map = machine__kernel_map(machine);
1590 if (!map)
1591 return 0;
1592
1593 if (map__load(map))
1594 return 0;
1595
1596 start = dso__first_symbol(map->dso, MAP__FUNCTION);
1597
1598 for (sym = start; sym; sym = dso__next_symbol(sym)) {
1599 if (sym->binding == STB_GLOBAL &&
1600 !strcmp(sym->name, "__switch_to")) {
1601 ip = map->unmap_ip(map, sym->start);
1602 if (ip >= map->start && ip < map->end) {
1603 switch_ip = ip;
1604 break;
1605 }
1606 }
1607 }
1608
1609 if (!switch_ip || !ptss_ip)
1610 return 0;
1611
1612 if (pt->have_sched_switch == 1)
1613 ptss = "perf_trace_sched_switch";
1614 else
1615 ptss = "__perf_event_task_sched_out";
1616
1617 for (sym = start; sym; sym = dso__next_symbol(sym)) {
1618 if (!strcmp(sym->name, ptss)) {
1619 ip = map->unmap_ip(map, sym->start);
1620 if (ip >= map->start && ip < map->end) {
1621 *ptss_ip = ip;
1622 break;
1623 }
1624 }
1625 }
1626
1627 return switch_ip;
1628 }
1629
1630 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1631 {
1632 const struct intel_pt_state *state = ptq->state;
1633 struct intel_pt *pt = ptq->pt;
1634 int err;
1635
1636 if (!pt->kernel_start) {
1637 pt->kernel_start = machine__kernel_start(pt->machine);
1638 if (pt->per_cpu_mmaps &&
1639 (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1640 !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1641 !pt->sampling_mode) {
1642 pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1643 if (pt->switch_ip) {
1644 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1645 pt->switch_ip, pt->ptss_ip);
1646 pt->sync_switch = true;
1647 }
1648 }
1649 }
1650
1651 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1652 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1653 while (1) {
1654 err = intel_pt_sample(ptq);
1655 if (err)
1656 return err;
1657
1658 state = intel_pt_decode(ptq->decoder);
1659 if (state->err) {
1660 if (state->err == INTEL_PT_ERR_NODATA)
1661 return 1;
1662 if (pt->sync_switch &&
1663 state->from_ip >= pt->kernel_start) {
1664 pt->sync_switch = false;
1665 intel_pt_next_tid(pt, ptq);
1666 }
1667 if (pt->synth_opts.errors) {
1668 err = intel_pt_synth_error(pt, state->err,
1669 ptq->cpu, ptq->pid,
1670 ptq->tid,
1671 state->from_ip);
1672 if (err)
1673 return err;
1674 }
1675 continue;
1676 }
1677
1678 ptq->state = state;
1679 ptq->have_sample = true;
1680 intel_pt_sample_flags(ptq);
1681
1682 /* Use estimated TSC upon return to user space */
1683 if (pt->est_tsc &&
1684 (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1685 state->to_ip && state->to_ip < pt->kernel_start) {
1686 intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1687 state->timestamp, state->est_timestamp);
1688 ptq->timestamp = state->est_timestamp;
1689 /* Use estimated TSC in unknown switch state */
1690 } else if (pt->sync_switch &&
1691 ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1692 intel_pt_is_switch_ip(ptq, state->to_ip) &&
1693 ptq->next_tid == -1) {
1694 intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1695 state->timestamp, state->est_timestamp);
1696 ptq->timestamp = state->est_timestamp;
1697 } else if (state->timestamp > ptq->timestamp) {
1698 ptq->timestamp = state->timestamp;
1699 }
1700
1701 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1702 *timestamp = ptq->timestamp;
1703 return 0;
1704 }
1705 }
1706 return 0;
1707 }
1708
1709 static inline int intel_pt_update_queues(struct intel_pt *pt)
1710 {
1711 if (pt->queues.new_data) {
1712 pt->queues.new_data = false;
1713 return intel_pt_setup_queues(pt);
1714 }
1715 return 0;
1716 }
1717
1718 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1719 {
1720 unsigned int queue_nr;
1721 u64 ts;
1722 int ret;
1723
1724 while (1) {
1725 struct auxtrace_queue *queue;
1726 struct intel_pt_queue *ptq;
1727
1728 if (!pt->heap.heap_cnt)
1729 return 0;
1730
1731 if (pt->heap.heap_array[0].ordinal >= timestamp)
1732 return 0;
1733
1734 queue_nr = pt->heap.heap_array[0].queue_nr;
1735 queue = &pt->queues.queue_array[queue_nr];
1736 ptq = queue->priv;
1737
1738 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1739 queue_nr, pt->heap.heap_array[0].ordinal,
1740 timestamp);
1741
1742 auxtrace_heap__pop(&pt->heap);
1743
1744 if (pt->heap.heap_cnt) {
1745 ts = pt->heap.heap_array[0].ordinal + 1;
1746 if (ts > timestamp)
1747 ts = timestamp;
1748 } else {
1749 ts = timestamp;
1750 }
1751
1752 intel_pt_set_pid_tid_cpu(pt, queue);
1753
1754 ret = intel_pt_run_decoder(ptq, &ts);
1755
1756 if (ret < 0) {
1757 auxtrace_heap__add(&pt->heap, queue_nr, ts);
1758 return ret;
1759 }
1760
1761 if (!ret) {
1762 ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1763 if (ret < 0)
1764 return ret;
1765 } else {
1766 ptq->on_heap = false;
1767 }
1768 }
1769
1770 return 0;
1771 }
1772
1773 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1774 u64 time_)
1775 {
1776 struct auxtrace_queues *queues = &pt->queues;
1777 unsigned int i;
1778 u64 ts = 0;
1779
1780 for (i = 0; i < queues->nr_queues; i++) {
1781 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1782 struct intel_pt_queue *ptq = queue->priv;
1783
1784 if (ptq && (tid == -1 || ptq->tid == tid)) {
1785 ptq->time = time_;
1786 intel_pt_set_pid_tid_cpu(pt, queue);
1787 intel_pt_run_decoder(ptq, &ts);
1788 }
1789 }
1790 return 0;
1791 }
1792
1793 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1794 {
1795 return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1796 sample->pid, sample->tid, 0);
1797 }
1798
1799 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1800 {
1801 unsigned i, j;
1802
1803 if (cpu < 0 || !pt->queues.nr_queues)
1804 return NULL;
1805
1806 if ((unsigned)cpu >= pt->queues.nr_queues)
1807 i = pt->queues.nr_queues - 1;
1808 else
1809 i = cpu;
1810
1811 if (pt->queues.queue_array[i].cpu == cpu)
1812 return pt->queues.queue_array[i].priv;
1813
1814 for (j = 0; i > 0; j++) {
1815 if (pt->queues.queue_array[--i].cpu == cpu)
1816 return pt->queues.queue_array[i].priv;
1817 }
1818
1819 for (; j < pt->queues.nr_queues; j++) {
1820 if (pt->queues.queue_array[j].cpu == cpu)
1821 return pt->queues.queue_array[j].priv;
1822 }
1823
1824 return NULL;
1825 }
1826
1827 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1828 u64 timestamp)
1829 {
1830 struct intel_pt_queue *ptq;
1831 int err;
1832
1833 if (!pt->sync_switch)
1834 return 1;
1835
1836 ptq = intel_pt_cpu_to_ptq(pt, cpu);
1837 if (!ptq)
1838 return 1;
1839
1840 switch (ptq->switch_state) {
1841 case INTEL_PT_SS_NOT_TRACING:
1842 ptq->next_tid = -1;
1843 break;
1844 case INTEL_PT_SS_UNKNOWN:
1845 case INTEL_PT_SS_TRACING:
1846 ptq->next_tid = tid;
1847 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1848 return 0;
1849 case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1850 if (!ptq->on_heap) {
1851 ptq->timestamp = perf_time_to_tsc(timestamp,
1852 &pt->tc);
1853 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1854 ptq->timestamp);
1855 if (err)
1856 return err;
1857 ptq->on_heap = true;
1858 }
1859 ptq->switch_state = INTEL_PT_SS_TRACING;
1860 break;
1861 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1862 ptq->next_tid = tid;
1863 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1864 break;
1865 default:
1866 break;
1867 }
1868
1869 return 1;
1870 }
1871
1872 static int intel_pt_process_switch(struct intel_pt *pt,
1873 struct perf_sample *sample)
1874 {
1875 struct perf_evsel *evsel;
1876 pid_t tid;
1877 int cpu, ret;
1878
1879 evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1880 if (evsel != pt->switch_evsel)
1881 return 0;
1882
1883 tid = perf_evsel__intval(evsel, sample, "next_pid");
1884 cpu = sample->cpu;
1885
1886 intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1887 cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1888 &pt->tc));
1889
1890 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1891 if (ret <= 0)
1892 return ret;
1893
1894 return machine__set_current_tid(pt->machine, cpu, -1, tid);
1895 }
1896
1897 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1898 struct perf_sample *sample)
1899 {
1900 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1901 pid_t pid, tid;
1902 int cpu, ret;
1903
1904 cpu = sample->cpu;
1905
1906 if (pt->have_sched_switch == 3) {
1907 if (!out)
1908 return 0;
1909 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1910 pr_err("Expecting CPU-wide context switch event\n");
1911 return -EINVAL;
1912 }
1913 pid = event->context_switch.next_prev_pid;
1914 tid = event->context_switch.next_prev_tid;
1915 } else {
1916 if (out)
1917 return 0;
1918 pid = sample->pid;
1919 tid = sample->tid;
1920 }
1921
1922 if (tid == -1) {
1923 pr_err("context_switch event has no tid\n");
1924 return -EINVAL;
1925 }
1926
1927 intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1928 cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1929 &pt->tc));
1930
1931 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1932 if (ret <= 0)
1933 return ret;
1934
1935 return machine__set_current_tid(pt->machine, cpu, pid, tid);
1936 }
1937
1938 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1939 union perf_event *event,
1940 struct perf_sample *sample)
1941 {
1942 if (!pt->per_cpu_mmaps)
1943 return 0;
1944
1945 intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1946 sample->cpu, event->itrace_start.pid,
1947 event->itrace_start.tid, sample->time,
1948 perf_time_to_tsc(sample->time, &pt->tc));
1949
1950 return machine__set_current_tid(pt->machine, sample->cpu,
1951 event->itrace_start.pid,
1952 event->itrace_start.tid);
1953 }
1954
1955 static int intel_pt_process_event(struct perf_session *session,
1956 union perf_event *event,
1957 struct perf_sample *sample,
1958 struct perf_tool *tool)
1959 {
1960 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1961 auxtrace);
1962 u64 timestamp;
1963 int err = 0;
1964
1965 if (dump_trace)
1966 return 0;
1967
1968 if (!tool->ordered_events) {
1969 pr_err("Intel Processor Trace requires ordered events\n");
1970 return -EINVAL;
1971 }
1972
1973 if (sample->time && sample->time != (u64)-1)
1974 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1975 else
1976 timestamp = 0;
1977
1978 if (timestamp || pt->timeless_decoding) {
1979 err = intel_pt_update_queues(pt);
1980 if (err)
1981 return err;
1982 }
1983
1984 if (pt->timeless_decoding) {
1985 if (event->header.type == PERF_RECORD_EXIT) {
1986 err = intel_pt_process_timeless_queues(pt,
1987 event->fork.tid,
1988 sample->time);
1989 }
1990 } else if (timestamp) {
1991 err = intel_pt_process_queues(pt, timestamp);
1992 }
1993 if (err)
1994 return err;
1995
1996 if (event->header.type == PERF_RECORD_AUX &&
1997 (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1998 pt->synth_opts.errors) {
1999 err = intel_pt_lost(pt, sample);
2000 if (err)
2001 return err;
2002 }
2003
2004 if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
2005 err = intel_pt_process_switch(pt, sample);
2006 else if (event->header.type == PERF_RECORD_ITRACE_START)
2007 err = intel_pt_process_itrace_start(pt, event, sample);
2008 else if (event->header.type == PERF_RECORD_SWITCH ||
2009 event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2010 err = intel_pt_context_switch(pt, event, sample);
2011
2012 intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
2013 perf_event__name(event->header.type), event->header.type,
2014 sample->cpu, sample->time, timestamp);
2015
2016 return err;
2017 }
2018
2019 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
2020 {
2021 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2022 auxtrace);
2023 int ret;
2024
2025 if (dump_trace)
2026 return 0;
2027
2028 if (!tool->ordered_events)
2029 return -EINVAL;
2030
2031 ret = intel_pt_update_queues(pt);
2032 if (ret < 0)
2033 return ret;
2034
2035 if (pt->timeless_decoding)
2036 return intel_pt_process_timeless_queues(pt, -1,
2037 MAX_TIMESTAMP - 1);
2038
2039 return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2040 }
2041
2042 static void intel_pt_free_events(struct perf_session *session)
2043 {
2044 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2045 auxtrace);
2046 struct auxtrace_queues *queues = &pt->queues;
2047 unsigned int i;
2048
2049 for (i = 0; i < queues->nr_queues; i++) {
2050 intel_pt_free_queue(queues->queue_array[i].priv);
2051 queues->queue_array[i].priv = NULL;
2052 }
2053 intel_pt_log_disable();
2054 auxtrace_queues__free(queues);
2055 }
2056
2057 static void intel_pt_free(struct perf_session *session)
2058 {
2059 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2060 auxtrace);
2061
2062 auxtrace_heap__free(&pt->heap);
2063 intel_pt_free_events(session);
2064 session->auxtrace = NULL;
2065 thread__put(pt->unknown_thread);
2066 addr_filters__exit(&pt->filts);
2067 zfree(&pt->filter);
2068 free(pt);
2069 }
2070
2071 static int intel_pt_process_auxtrace_event(struct perf_session *session,
2072 union perf_event *event,
2073 struct perf_tool *tool __maybe_unused)
2074 {
2075 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2076 auxtrace);
2077
2078 if (pt->sampling_mode)
2079 return 0;
2080
2081 if (!pt->data_queued) {
2082 struct auxtrace_buffer *buffer;
2083 off_t data_offset;
2084 int fd = perf_data__fd(session->data);
2085 int err;
2086
2087 if (perf_data__is_pipe(session->data)) {
2088 data_offset = 0;
2089 } else {
2090 data_offset = lseek(fd, 0, SEEK_CUR);
2091 if (data_offset == -1)
2092 return -errno;
2093 }
2094
2095 err = auxtrace_queues__add_event(&pt->queues, session, event,
2096 data_offset, &buffer);
2097 if (err)
2098 return err;
2099
2100 /* Dump here now we have copied a piped trace out of the pipe */
2101 if (dump_trace) {
2102 if (auxtrace_buffer__get_data(buffer, fd)) {
2103 intel_pt_dump_event(pt, buffer->data,
2104 buffer->size);
2105 auxtrace_buffer__put_data(buffer);
2106 }
2107 }
2108 }
2109
2110 return 0;
2111 }
2112
2113 struct intel_pt_synth {
2114 struct perf_tool dummy_tool;
2115 struct perf_session *session;
2116 };
2117
2118 static int intel_pt_event_synth(struct perf_tool *tool,
2119 union perf_event *event,
2120 struct perf_sample *sample __maybe_unused,
2121 struct machine *machine __maybe_unused)
2122 {
2123 struct intel_pt_synth *intel_pt_synth =
2124 container_of(tool, struct intel_pt_synth, dummy_tool);
2125
2126 return perf_session__deliver_synth_event(intel_pt_synth->session, event,
2127 NULL);
2128 }
2129
2130 static int intel_pt_synth_event(struct perf_session *session, const char *name,
2131 struct perf_event_attr *attr, u64 id)
2132 {
2133 struct intel_pt_synth intel_pt_synth;
2134 int err;
2135
2136 pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2137 name, id, (u64)attr->sample_type);
2138
2139 memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
2140 intel_pt_synth.session = session;
2141
2142 err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
2143 &id, intel_pt_event_synth);
2144 if (err)
2145 pr_err("%s: failed to synthesize '%s' event type\n",
2146 __func__, name);
2147
2148 return err;
2149 }
2150
2151 static void intel_pt_set_event_name(struct perf_evlist *evlist, u64 id,
2152 const char *name)
2153 {
2154 struct perf_evsel *evsel;
2155
2156 evlist__for_each_entry(evlist, evsel) {
2157 if (evsel->id && evsel->id[0] == id) {
2158 if (evsel->name)
2159 zfree(&evsel->name);
2160 evsel->name = strdup(name);
2161 break;
2162 }
2163 }
2164 }
2165
2166 static struct perf_evsel *intel_pt_evsel(struct intel_pt *pt,
2167 struct perf_evlist *evlist)
2168 {
2169 struct perf_evsel *evsel;
2170
2171 evlist__for_each_entry(evlist, evsel) {
2172 if (evsel->attr.type == pt->pmu_type && evsel->ids)
2173 return evsel;
2174 }
2175
2176 return NULL;
2177 }
2178
2179 static int intel_pt_synth_events(struct intel_pt *pt,
2180 struct perf_session *session)
2181 {
2182 struct perf_evlist *evlist = session->evlist;
2183 struct perf_evsel *evsel = intel_pt_evsel(pt, evlist);
2184 struct perf_event_attr attr;
2185 u64 id;
2186 int err;
2187
2188 if (!evsel) {
2189 pr_debug("There are no selected events with Intel Processor Trace data\n");
2190 return 0;
2191 }
2192
2193 memset(&attr, 0, sizeof(struct perf_event_attr));
2194 attr.size = sizeof(struct perf_event_attr);
2195 attr.type = PERF_TYPE_HARDWARE;
2196 attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
2197 attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
2198 PERF_SAMPLE_PERIOD;
2199 if (pt->timeless_decoding)
2200 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
2201 else
2202 attr.sample_type |= PERF_SAMPLE_TIME;
2203 if (!pt->per_cpu_mmaps)
2204 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
2205 attr.exclude_user = evsel->attr.exclude_user;
2206 attr.exclude_kernel = evsel->attr.exclude_kernel;
2207 attr.exclude_hv = evsel->attr.exclude_hv;
2208 attr.exclude_host = evsel->attr.exclude_host;
2209 attr.exclude_guest = evsel->attr.exclude_guest;
2210 attr.sample_id_all = evsel->attr.sample_id_all;
2211 attr.read_format = evsel->attr.read_format;
2212
2213 id = evsel->id[0] + 1000000000;
2214 if (!id)
2215 id = 1;
2216
2217 if (pt->synth_opts.branches) {
2218 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2219 attr.sample_period = 1;
2220 attr.sample_type |= PERF_SAMPLE_ADDR;
2221 err = intel_pt_synth_event(session, "branches", &attr, id);
2222 if (err)
2223 return err;
2224 pt->sample_branches = true;
2225 pt->branches_sample_type = attr.sample_type;
2226 pt->branches_id = id;
2227 id += 1;
2228 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
2229 }
2230
2231 if (pt->synth_opts.callchain)
2232 attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2233 if (pt->synth_opts.last_branch)
2234 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2235
2236 if (pt->synth_opts.instructions) {
2237 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2238 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2239 attr.sample_period =
2240 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2241 else
2242 attr.sample_period = pt->synth_opts.period;
2243 err = intel_pt_synth_event(session, "instructions", &attr, id);
2244 if (err)
2245 return err;
2246 pt->sample_instructions = true;
2247 pt->instructions_sample_type = attr.sample_type;
2248 pt->instructions_id = id;
2249 id += 1;
2250 }
2251
2252 attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
2253 attr.sample_period = 1;
2254
2255 if (pt->synth_opts.transactions) {
2256 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2257 err = intel_pt_synth_event(session, "transactions", &attr, id);
2258 if (err)
2259 return err;
2260 pt->sample_transactions = true;
2261 pt->transactions_sample_type = attr.sample_type;
2262 pt->transactions_id = id;
2263 intel_pt_set_event_name(evlist, id, "transactions");
2264 id += 1;
2265 }
2266
2267 attr.type = PERF_TYPE_SYNTH;
2268 attr.sample_type |= PERF_SAMPLE_RAW;
2269
2270 if (pt->synth_opts.ptwrites) {
2271 attr.config = PERF_SYNTH_INTEL_PTWRITE;
2272 err = intel_pt_synth_event(session, "ptwrite", &attr, id);
2273 if (err)
2274 return err;
2275 pt->sample_ptwrites = true;
2276 pt->ptwrites_sample_type = attr.sample_type;
2277 pt->ptwrites_id = id;
2278 intel_pt_set_event_name(evlist, id, "ptwrite");
2279 id += 1;
2280 }
2281
2282 if (pt->synth_opts.pwr_events) {
2283 pt->sample_pwr_events = true;
2284 pt->pwr_events_sample_type = attr.sample_type;
2285
2286 attr.config = PERF_SYNTH_INTEL_CBR;
2287 err = intel_pt_synth_event(session, "cbr", &attr, id);
2288 if (err)
2289 return err;
2290 pt->cbr_id = id;
2291 intel_pt_set_event_name(evlist, id, "cbr");
2292 id += 1;
2293 }
2294
2295 if (pt->synth_opts.pwr_events && (evsel->attr.config & 0x10)) {
2296 attr.config = PERF_SYNTH_INTEL_MWAIT;
2297 err = intel_pt_synth_event(session, "mwait", &attr, id);
2298 if (err)
2299 return err;
2300 pt->mwait_id = id;
2301 intel_pt_set_event_name(evlist, id, "mwait");
2302 id += 1;
2303
2304 attr.config = PERF_SYNTH_INTEL_PWRE;
2305 err = intel_pt_synth_event(session, "pwre", &attr, id);
2306 if (err)
2307 return err;
2308 pt->pwre_id = id;
2309 intel_pt_set_event_name(evlist, id, "pwre");
2310 id += 1;
2311
2312 attr.config = PERF_SYNTH_INTEL_EXSTOP;
2313 err = intel_pt_synth_event(session, "exstop", &attr, id);
2314 if (err)
2315 return err;
2316 pt->exstop_id = id;
2317 intel_pt_set_event_name(evlist, id, "exstop");
2318 id += 1;
2319
2320 attr.config = PERF_SYNTH_INTEL_PWRX;
2321 err = intel_pt_synth_event(session, "pwrx", &attr, id);
2322 if (err)
2323 return err;
2324 pt->pwrx_id = id;
2325 intel_pt_set_event_name(evlist, id, "pwrx");
2326 id += 1;
2327 }
2328
2329 return 0;
2330 }
2331
2332 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2333 {
2334 struct perf_evsel *evsel;
2335
2336 evlist__for_each_entry_reverse(evlist, evsel) {
2337 const char *name = perf_evsel__name(evsel);
2338
2339 if (!strcmp(name, "sched:sched_switch"))
2340 return evsel;
2341 }
2342
2343 return NULL;
2344 }
2345
2346 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2347 {
2348 struct perf_evsel *evsel;
2349
2350 evlist__for_each_entry(evlist, evsel) {
2351 if (evsel->attr.context_switch)
2352 return true;
2353 }
2354
2355 return false;
2356 }
2357
2358 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2359 {
2360 struct intel_pt *pt = data;
2361
2362 if (!strcmp(var, "intel-pt.mispred-all"))
2363 pt->mispred_all = perf_config_bool(var, value);
2364
2365 return 0;
2366 }
2367
2368 static const char * const intel_pt_info_fmts[] = {
2369 [INTEL_PT_PMU_TYPE] = " PMU Type %"PRId64"\n",
2370 [INTEL_PT_TIME_SHIFT] = " Time Shift %"PRIu64"\n",
2371 [INTEL_PT_TIME_MULT] = " Time Muliplier %"PRIu64"\n",
2372 [INTEL_PT_TIME_ZERO] = " Time Zero %"PRIu64"\n",
2373 [INTEL_PT_CAP_USER_TIME_ZERO] = " Cap Time Zero %"PRId64"\n",
2374 [INTEL_PT_TSC_BIT] = " TSC bit %#"PRIx64"\n",
2375 [INTEL_PT_NORETCOMP_BIT] = " NoRETComp bit %#"PRIx64"\n",
2376 [INTEL_PT_HAVE_SCHED_SWITCH] = " Have sched_switch %"PRId64"\n",
2377 [INTEL_PT_SNAPSHOT_MODE] = " Snapshot mode %"PRId64"\n",
2378 [INTEL_PT_PER_CPU_MMAPS] = " Per-cpu maps %"PRId64"\n",
2379 [INTEL_PT_MTC_BIT] = " MTC bit %#"PRIx64"\n",
2380 [INTEL_PT_TSC_CTC_N] = " TSC:CTC numerator %"PRIu64"\n",
2381 [INTEL_PT_TSC_CTC_D] = " TSC:CTC denominator %"PRIu64"\n",
2382 [INTEL_PT_CYC_BIT] = " CYC bit %#"PRIx64"\n",
2383 [INTEL_PT_MAX_NONTURBO_RATIO] = " Max non-turbo ratio %"PRIu64"\n",
2384 [INTEL_PT_FILTER_STR_LEN] = " Filter string len. %"PRIu64"\n",
2385 };
2386
2387 static void intel_pt_print_info(u64 *arr, int start, int finish)
2388 {
2389 int i;
2390
2391 if (!dump_trace)
2392 return;
2393
2394 for (i = start; i <= finish; i++)
2395 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2396 }
2397
2398 static void intel_pt_print_info_str(const char *name, const char *str)
2399 {
2400 if (!dump_trace)
2401 return;
2402
2403 fprintf(stdout, " %-20s%s\n", name, str ? str : "");
2404 }
2405
2406 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2407 {
2408 return auxtrace_info->header.size >=
2409 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2410 }
2411
2412 int intel_pt_process_auxtrace_info(union perf_event *event,
2413 struct perf_session *session)
2414 {
2415 struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2416 size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2417 struct intel_pt *pt;
2418 void *info_end;
2419 u64 *info;
2420 int err;
2421
2422 if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2423 min_sz)
2424 return -EINVAL;
2425
2426 pt = zalloc(sizeof(struct intel_pt));
2427 if (!pt)
2428 return -ENOMEM;
2429
2430 addr_filters__init(&pt->filts);
2431
2432 err = perf_config(intel_pt_perf_config, pt);
2433 if (err)
2434 goto err_free;
2435
2436 err = auxtrace_queues__init(&pt->queues);
2437 if (err)
2438 goto err_free;
2439
2440 intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2441
2442 pt->session = session;
2443 pt->machine = &session->machines.host; /* No kvm support */
2444 pt->auxtrace_type = auxtrace_info->type;
2445 pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2446 pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2447 pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2448 pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2449 pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2450 pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2451 pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2452 pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2453 pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2454 pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2455 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2456 INTEL_PT_PER_CPU_MMAPS);
2457
2458 if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2459 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2460 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2461 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2462 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2463 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2464 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2465 INTEL_PT_CYC_BIT);
2466 }
2467
2468 if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2469 pt->max_non_turbo_ratio =
2470 auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2471 intel_pt_print_info(&auxtrace_info->priv[0],
2472 INTEL_PT_MAX_NONTURBO_RATIO,
2473 INTEL_PT_MAX_NONTURBO_RATIO);
2474 }
2475
2476 info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2477 info_end = (void *)info + auxtrace_info->header.size;
2478
2479 if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2480 size_t len;
2481
2482 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2483 intel_pt_print_info(&auxtrace_info->priv[0],
2484 INTEL_PT_FILTER_STR_LEN,
2485 INTEL_PT_FILTER_STR_LEN);
2486 if (len) {
2487 const char *filter = (const char *)info;
2488
2489 len = roundup(len + 1, 8);
2490 info += len >> 3;
2491 if ((void *)info > info_end) {
2492 pr_err("%s: bad filter string length\n", __func__);
2493 err = -EINVAL;
2494 goto err_free_queues;
2495 }
2496 pt->filter = memdup(filter, len);
2497 if (!pt->filter) {
2498 err = -ENOMEM;
2499 goto err_free_queues;
2500 }
2501 if (session->header.needs_swap)
2502 mem_bswap_64(pt->filter, len);
2503 if (pt->filter[len - 1]) {
2504 pr_err("%s: filter string not null terminated\n", __func__);
2505 err = -EINVAL;
2506 goto err_free_queues;
2507 }
2508 err = addr_filters__parse_bare_filter(&pt->filts,
2509 filter);
2510 if (err)
2511 goto err_free_queues;
2512 }
2513 intel_pt_print_info_str("Filter string", pt->filter);
2514 }
2515
2516 pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2517 pt->have_tsc = intel_pt_have_tsc(pt);
2518 pt->sampling_mode = false;
2519 pt->est_tsc = !pt->timeless_decoding;
2520
2521 pt->unknown_thread = thread__new(999999999, 999999999);
2522 if (!pt->unknown_thread) {
2523 err = -ENOMEM;
2524 goto err_free_queues;
2525 }
2526
2527 /*
2528 * Since this thread will not be kept in any rbtree not in a
2529 * list, initialize its list node so that at thread__put() the
2530 * current thread lifetime assuption is kept and we don't segfault
2531 * at list_del_init().
2532 */
2533 INIT_LIST_HEAD(&pt->unknown_thread->node);
2534
2535 err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2536 if (err)
2537 goto err_delete_thread;
2538 if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2539 err = -ENOMEM;
2540 goto err_delete_thread;
2541 }
2542
2543 pt->auxtrace.process_event = intel_pt_process_event;
2544 pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2545 pt->auxtrace.flush_events = intel_pt_flush;
2546 pt->auxtrace.free_events = intel_pt_free_events;
2547 pt->auxtrace.free = intel_pt_free;
2548 session->auxtrace = &pt->auxtrace;
2549
2550 if (dump_trace)
2551 return 0;
2552
2553 if (pt->have_sched_switch == 1) {
2554 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2555 if (!pt->switch_evsel) {
2556 pr_err("%s: missing sched_switch event\n", __func__);
2557 err = -EINVAL;
2558 goto err_delete_thread;
2559 }
2560 } else if (pt->have_sched_switch == 2 &&
2561 !intel_pt_find_switch(session->evlist)) {
2562 pr_err("%s: missing context_switch attribute flag\n", __func__);
2563 err = -EINVAL;
2564 goto err_delete_thread;
2565 }
2566
2567 if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2568 pt->synth_opts = *session->itrace_synth_opts;
2569 } else {
2570 itrace_synth_opts__set_default(&pt->synth_opts);
2571 if (use_browser != -1) {
2572 pt->synth_opts.branches = false;
2573 pt->synth_opts.callchain = true;
2574 }
2575 if (session->itrace_synth_opts)
2576 pt->synth_opts.thread_stack =
2577 session->itrace_synth_opts->thread_stack;
2578 }
2579
2580 if (pt->synth_opts.log)
2581 intel_pt_log_enable();
2582
2583 /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2584 if (pt->tc.time_mult) {
2585 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2586
2587 if (!pt->max_non_turbo_ratio)
2588 pt->max_non_turbo_ratio =
2589 (tsc_freq + 50000000) / 100000000;
2590 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2591 intel_pt_log("Maximum non-turbo ratio %u\n",
2592 pt->max_non_turbo_ratio);
2593 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
2594 }
2595
2596 if (pt->synth_opts.calls)
2597 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2598 PERF_IP_FLAG_TRACE_END;
2599 if (pt->synth_opts.returns)
2600 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2601 PERF_IP_FLAG_TRACE_BEGIN;
2602
2603 if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2604 symbol_conf.use_callchain = true;
2605 if (callchain_register_param(&callchain_param) < 0) {
2606 symbol_conf.use_callchain = false;
2607 pt->synth_opts.callchain = false;
2608 }
2609 }
2610
2611 err = intel_pt_synth_events(pt, session);
2612 if (err)
2613 goto err_delete_thread;
2614
2615 err = auxtrace_queues__process_index(&pt->queues, session);
2616 if (err)
2617 goto err_delete_thread;
2618
2619 if (pt->queues.populated)
2620 pt->data_queued = true;
2621
2622 if (pt->timeless_decoding)
2623 pr_debug2("Intel PT decoding without timestamps\n");
2624
2625 return 0;
2626
2627 err_delete_thread:
2628 thread__zput(pt->unknown_thread);
2629 err_free_queues:
2630 intel_pt_log_disable();
2631 auxtrace_queues__free(&pt->queues);
2632 session->auxtrace = NULL;
2633 err_free:
2634 addr_filters__exit(&pt->filts);
2635 zfree(&pt->filter);
2636 free(pt);
2637 return err;
2638 }
CRIS linux kernel tree