?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf intel-pt: Add lookahead callback
[linux/fpc-iii.git] / tools / perf / util / intel-pt-decoder / intel-pt-decoder.c
blob13123b1950831f058061ce74087aabd97c9b2a6d
1 /*
2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16 #ifndef _GNU_SOURCE
17 #define _GNU_SOURCE
18 #endif
19 #include <stdlib.h>
20 #include <stdbool.h>
21 #include <string.h>
22 #include <errno.h>
23 #include <stdint.h>
24 #include <inttypes.h>
25 #include <linux/compiler.h>
26
27 #include "../cache.h"
28 #include "../util.h"
29 #include "../auxtrace.h"
30
31 #include "intel-pt-insn-decoder.h"
32 #include "intel-pt-pkt-decoder.h"
33 #include "intel-pt-decoder.h"
34 #include "intel-pt-log.h"
35
36 #define INTEL_PT_BLK_SIZE 1024
37
38 #define BIT63 (((uint64_t)1 << 63))
39
40 #define INTEL_PT_RETURN 1
41
42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
43 #define INTEL_PT_MAX_LOOPS 10000
44
45 struct intel_pt_blk {
46 struct intel_pt_blk *prev;
47 uint64_t ip[INTEL_PT_BLK_SIZE];
48 };
49
50 struct intel_pt_stack {
51 struct intel_pt_blk *blk;
52 struct intel_pt_blk *spare;
53 int pos;
54 };
55
56 enum intel_pt_pkt_state {
57 INTEL_PT_STATE_NO_PSB,
58 INTEL_PT_STATE_NO_IP,
59 INTEL_PT_STATE_ERR_RESYNC,
60 INTEL_PT_STATE_IN_SYNC,
61 INTEL_PT_STATE_TNT_CONT,
62 INTEL_PT_STATE_TNT,
63 INTEL_PT_STATE_TIP,
64 INTEL_PT_STATE_TIP_PGD,
65 INTEL_PT_STATE_FUP,
66 INTEL_PT_STATE_FUP_NO_TIP,
67 };
68
69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
70 {
71 switch (pkt_state) {
72 case INTEL_PT_STATE_NO_PSB:
73 case INTEL_PT_STATE_NO_IP:
74 case INTEL_PT_STATE_ERR_RESYNC:
75 case INTEL_PT_STATE_IN_SYNC:
76 case INTEL_PT_STATE_TNT_CONT:
77 return true;
78 case INTEL_PT_STATE_TNT:
79 case INTEL_PT_STATE_TIP:
80 case INTEL_PT_STATE_TIP_PGD:
81 case INTEL_PT_STATE_FUP:
82 case INTEL_PT_STATE_FUP_NO_TIP:
83 return false;
84 default:
85 return true;
86 };
87 }
88
89 #ifdef INTEL_PT_STRICT
90 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
91 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
92 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
93 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
94 #else
95 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
96 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
97 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
98 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
99 #endif
100
101 struct intel_pt_decoder {
102 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
103 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
104 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
105 uint64_t max_insn_cnt, void *data);
106 bool (*pgd_ip)(uint64_t ip, void *data);
107 int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);
108 void *data;
109 struct intel_pt_state state;
110 const unsigned char *buf;
111 size_t len;
112 bool return_compression;
113 bool branch_enable;
114 bool mtc_insn;
115 bool pge;
116 bool have_tma;
117 bool have_cyc;
118 bool fixup_last_mtc;
119 bool have_last_ip;
120 bool in_psb;
121 enum intel_pt_param_flags flags;
122 uint64_t pos;
123 uint64_t last_ip;
124 uint64_t ip;
125 uint64_t cr3;
126 uint64_t timestamp;
127 uint64_t tsc_timestamp;
128 uint64_t ref_timestamp;
129 uint64_t sample_timestamp;
130 uint64_t ret_addr;
131 uint64_t ctc_timestamp;
132 uint64_t ctc_delta;
133 uint64_t cycle_cnt;
134 uint64_t cyc_ref_timestamp;
135 uint32_t last_mtc;
136 uint32_t tsc_ctc_ratio_n;
137 uint32_t tsc_ctc_ratio_d;
138 uint32_t tsc_ctc_mult;
139 uint32_t tsc_slip;
140 uint32_t ctc_rem_mask;
141 int mtc_shift;
142 struct intel_pt_stack stack;
143 enum intel_pt_pkt_state pkt_state;
144 struct intel_pt_pkt packet;
145 struct intel_pt_pkt tnt;
146 int pkt_step;
147 int pkt_len;
148 int last_packet_type;
149 unsigned int cbr;
150 unsigned int cbr_seen;
151 unsigned int max_non_turbo_ratio;
152 double max_non_turbo_ratio_fp;
153 double cbr_cyc_to_tsc;
154 double calc_cyc_to_tsc;
155 bool have_calc_cyc_to_tsc;
156 int exec_mode;
157 unsigned int insn_bytes;
158 uint64_t period;
159 enum intel_pt_period_type period_type;
160 uint64_t tot_insn_cnt;
161 uint64_t period_insn_cnt;
162 uint64_t period_mask;
163 uint64_t period_ticks;
164 uint64_t last_masked_timestamp;
165 uint64_t tot_cyc_cnt;
166 uint64_t sample_tot_cyc_cnt;
167 uint64_t base_cyc_cnt;
168 uint64_t cyc_cnt_timestamp;
169 double tsc_to_cyc;
170 bool continuous_period;
171 bool overflow;
172 bool set_fup_tx_flags;
173 bool set_fup_ptw;
174 bool set_fup_mwait;
175 bool set_fup_pwre;
176 bool set_fup_exstop;
177 bool sample_cyc;
178 unsigned int fup_tx_flags;
179 unsigned int tx_flags;
180 uint64_t fup_ptw_payload;
181 uint64_t fup_mwait_payload;
182 uint64_t fup_pwre_payload;
183 uint64_t cbr_payload;
184 uint64_t timestamp_insn_cnt;
185 uint64_t sample_insn_cnt;
186 uint64_t stuck_ip;
187 int no_progress;
188 int stuck_ip_prd;
189 int stuck_ip_cnt;
190 const unsigned char *next_buf;
191 size_t next_len;
192 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
193 };
194
195 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
196 {
197 int i;
198
199 for (i = 0; x != 1; i++)
200 x >>= 1;
201
202 return x << i;
203 }
204
205 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
206 {
207 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
208 uint64_t period;
209
210 period = intel_pt_lower_power_of_2(decoder->period);
211 decoder->period_mask = ~(period - 1);
212 decoder->period_ticks = period;
213 }
214 }
215
216 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
217 {
218 if (!d)
219 return 0;
220 return (t / d) * n + ((t % d) * n) / d;
221 }
222
223 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
224 {
225 struct intel_pt_decoder *decoder;
226
227 if (!params->get_trace || !params->walk_insn)
228 return NULL;
229
230 decoder = zalloc(sizeof(struct intel_pt_decoder));
231 if (!decoder)
232 return NULL;
233
234 decoder->get_trace = params->get_trace;
235 decoder->walk_insn = params->walk_insn;
236 decoder->pgd_ip = params->pgd_ip;
237 decoder->lookahead = params->lookahead;
238 decoder->data = params->data;
239 decoder->return_compression = params->return_compression;
240 decoder->branch_enable = params->branch_enable;
241
242 decoder->flags = params->flags;
243
244 decoder->period = params->period;
245 decoder->period_type = params->period_type;
246
247 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
248 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
249
250 intel_pt_setup_period(decoder);
251
252 decoder->mtc_shift = params->mtc_period;
253 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
254
255 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
256 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
257
258 if (!decoder->tsc_ctc_ratio_n)
259 decoder->tsc_ctc_ratio_d = 0;
260
261 if (decoder->tsc_ctc_ratio_d) {
262 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
263 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
264 decoder->tsc_ctc_ratio_d;
265 }
266
267 /*
268 * A TSC packet can slip past MTC packets so that the timestamp appears
269 * to go backwards. One estimate is that can be up to about 40 CPU
270 * cycles, which is certainly less than 0x1000 TSC ticks, but accept
271 * slippage an order of magnitude more to be on the safe side.
272 */
273 decoder->tsc_slip = 0x10000;
274
275 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
276 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
277 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
278 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
279 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
280
281 return decoder;
282 }
283
284 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
285 {
286 struct intel_pt_blk *blk = stack->blk;
287
288 stack->blk = blk->prev;
289 if (!stack->spare)
290 stack->spare = blk;
291 else
292 free(blk);
293 }
294
295 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
296 {
297 if (!stack->pos) {
298 if (!stack->blk)
299 return 0;
300 intel_pt_pop_blk(stack);
301 if (!stack->blk)
302 return 0;
303 stack->pos = INTEL_PT_BLK_SIZE;
304 }
305 return stack->blk->ip[--stack->pos];
306 }
307
308 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
309 {
310 struct intel_pt_blk *blk;
311
312 if (stack->spare) {
313 blk = stack->spare;
314 stack->spare = NULL;
315 } else {
316 blk = malloc(sizeof(struct intel_pt_blk));
317 if (!blk)
318 return -ENOMEM;
319 }
320
321 blk->prev = stack->blk;
322 stack->blk = blk;
323 stack->pos = 0;
324 return 0;
325 }
326
327 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
328 {
329 int err;
330
331 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
332 err = intel_pt_alloc_blk(stack);
333 if (err)
334 return err;
335 }
336
337 stack->blk->ip[stack->pos++] = ip;
338 return 0;
339 }
340
341 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
342 {
343 while (stack->blk)
344 intel_pt_pop_blk(stack);
345 stack->pos = 0;
346 }
347
348 static void intel_pt_free_stack(struct intel_pt_stack *stack)
349 {
350 intel_pt_clear_stack(stack);
351 zfree(&stack->blk);
352 zfree(&stack->spare);
353 }
354
355 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
356 {
357 intel_pt_free_stack(&decoder->stack);
358 free(decoder);
359 }
360
361 static int intel_pt_ext_err(int code)
362 {
363 switch (code) {
364 case -ENOMEM:
365 return INTEL_PT_ERR_NOMEM;
366 case -ENOSYS:
367 return INTEL_PT_ERR_INTERN;
368 case -EBADMSG:
369 return INTEL_PT_ERR_BADPKT;
370 case -ENODATA:
371 return INTEL_PT_ERR_NODATA;
372 case -EILSEQ:
373 return INTEL_PT_ERR_NOINSN;
374 case -ENOENT:
375 return INTEL_PT_ERR_MISMAT;
376 case -EOVERFLOW:
377 return INTEL_PT_ERR_OVR;
378 case -ENOSPC:
379 return INTEL_PT_ERR_LOST;
380 case -ELOOP:
381 return INTEL_PT_ERR_NELOOP;
382 default:
383 return INTEL_PT_ERR_UNK;
384 }
385 }
386
387 static const char *intel_pt_err_msgs[] = {
388 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
389 [INTEL_PT_ERR_INTERN] = "Internal error",
390 [INTEL_PT_ERR_BADPKT] = "Bad packet",
391 [INTEL_PT_ERR_NODATA] = "No more data",
392 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
393 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
394 [INTEL_PT_ERR_OVR] = "Overflow packet",
395 [INTEL_PT_ERR_LOST] = "Lost trace data",
396 [INTEL_PT_ERR_UNK] = "Unknown error!",
397 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
398 };
399
400 int intel_pt__strerror(int code, char *buf, size_t buflen)
401 {
402 if (code < 1 || code >= INTEL_PT_ERR_MAX)
403 code = INTEL_PT_ERR_UNK;
404 strlcpy(buf, intel_pt_err_msgs[code], buflen);
405 return 0;
406 }
407
408 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
409 uint64_t last_ip)
410 {
411 uint64_t ip;
412
413 switch (packet->count) {
414 case 1:
415 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
416 packet->payload;
417 break;
418 case 2:
419 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
420 packet->payload;
421 break;
422 case 3:
423 ip = packet->payload;
424 /* Sign-extend 6-byte ip */
425 if (ip & (uint64_t)0x800000000000ULL)
426 ip |= (uint64_t)0xffff000000000000ULL;
427 break;
428 case 4:
429 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
430 packet->payload;
431 break;
432 case 6:
433 ip = packet->payload;
434 break;
435 default:
436 return 0;
437 }
438
439 return ip;
440 }
441
442 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
443 {
444 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
445 decoder->have_last_ip = true;
446 }
447
448 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
449 {
450 intel_pt_set_last_ip(decoder);
451 decoder->ip = decoder->last_ip;
452 }
453
454 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
455 {
456 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
457 decoder->buf);
458 }
459
460 static int intel_pt_bug(struct intel_pt_decoder *decoder)
461 {
462 intel_pt_log("ERROR: Internal error\n");
463 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
464 return -ENOSYS;
465 }
466
467 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
468 {
469 decoder->tx_flags = 0;
470 }
471
472 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
473 {
474 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
475 }
476
477 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
478 {
479 intel_pt_clear_tx_flags(decoder);
480 decoder->have_tma = false;
481 decoder->pkt_len = 1;
482 decoder->pkt_step = 1;
483 intel_pt_decoder_log_packet(decoder);
484 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
485 intel_pt_log("ERROR: Bad packet\n");
486 decoder->pkt_state = INTEL_PT_STATE_ERR1;
487 }
488 return -EBADMSG;
489 }
490
491 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)
492 {
493 decoder->sample_timestamp = decoder->timestamp;
494 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
495 }
496
497 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
498 {
499 struct intel_pt_buffer buffer = { .buf = 0, };
500 int ret;
501
502 decoder->pkt_step = 0;
503
504 intel_pt_log("Getting more data\n");
505 ret = decoder->get_trace(&buffer, decoder->data);
506 if (ret)
507 return ret;
508 decoder->buf = buffer.buf;
509 decoder->len = buffer.len;
510 if (!decoder->len) {
511 intel_pt_log("No more data\n");
512 return -ENODATA;
513 }
514 if (!buffer.consecutive) {
515 decoder->ip = 0;
516 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
517 decoder->ref_timestamp = buffer.ref_timestamp;
518 decoder->timestamp = 0;
519 decoder->have_tma = false;
520 decoder->state.trace_nr = buffer.trace_nr;
521 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
522 decoder->ref_timestamp);
523 return -ENOLINK;
524 }
525
526 return 0;
527 }
528
529 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
530 {
531 if (!decoder->next_buf)
532 return intel_pt_get_data(decoder);
533
534 decoder->buf = decoder->next_buf;
535 decoder->len = decoder->next_len;
536 decoder->next_buf = 0;
537 decoder->next_len = 0;
538 return 0;
539 }
540
541 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
542 {
543 unsigned char *buf = decoder->temp_buf;
544 size_t old_len, len, n;
545 int ret;
546
547 old_len = decoder->len;
548 len = decoder->len;
549 memcpy(buf, decoder->buf, len);
550
551 ret = intel_pt_get_data(decoder);
552 if (ret) {
553 decoder->pos += old_len;
554 return ret < 0 ? ret : -EINVAL;
555 }
556
557 n = INTEL_PT_PKT_MAX_SZ - len;
558 if (n > decoder->len)
559 n = decoder->len;
560 memcpy(buf + len, decoder->buf, n);
561 len += n;
562
563 ret = intel_pt_get_packet(buf, len, &decoder->packet);
564 if (ret < (int)old_len) {
565 decoder->next_buf = decoder->buf;
566 decoder->next_len = decoder->len;
567 decoder->buf = buf;
568 decoder->len = old_len;
569 return intel_pt_bad_packet(decoder);
570 }
571
572 decoder->next_buf = decoder->buf + (ret - old_len);
573 decoder->next_len = decoder->len - (ret - old_len);
574
575 decoder->buf = buf;
576 decoder->len = ret;
577
578 return ret;
579 }
580
581 struct intel_pt_pkt_info {
582 struct intel_pt_decoder *decoder;
583 struct intel_pt_pkt packet;
584 uint64_t pos;
585 int pkt_len;
586 int last_packet_type;
587 void *data;
588 };
589
590 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
591
592 /* Lookahead packets in current buffer */
593 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
594 intel_pt_pkt_cb_t cb, void *data)
595 {
596 struct intel_pt_pkt_info pkt_info;
597 const unsigned char *buf = decoder->buf;
598 size_t len = decoder->len;
599 int ret;
600
601 pkt_info.decoder = decoder;
602 pkt_info.pos = decoder->pos;
603 pkt_info.pkt_len = decoder->pkt_step;
604 pkt_info.last_packet_type = decoder->last_packet_type;
605 pkt_info.data = data;
606
607 while (1) {
608 do {
609 pkt_info.pos += pkt_info.pkt_len;
610 buf += pkt_info.pkt_len;
611 len -= pkt_info.pkt_len;
612
613 if (!len)
614 return INTEL_PT_NEED_MORE_BYTES;
615
616 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
617 if (!ret)
618 return INTEL_PT_NEED_MORE_BYTES;
619 if (ret < 0)
620 return ret;
621
622 pkt_info.pkt_len = ret;
623 } while (pkt_info.packet.type == INTEL_PT_PAD);
624
625 ret = cb(&pkt_info);
626 if (ret)
627 return 0;
628
629 pkt_info.last_packet_type = pkt_info.packet.type;
630 }
631 }
632
633 struct intel_pt_calc_cyc_to_tsc_info {
634 uint64_t cycle_cnt;
635 unsigned int cbr;
636 uint32_t last_mtc;
637 uint64_t ctc_timestamp;
638 uint64_t ctc_delta;
639 uint64_t tsc_timestamp;
640 uint64_t timestamp;
641 bool have_tma;
642 bool fixup_last_mtc;
643 bool from_mtc;
644 double cbr_cyc_to_tsc;
645 };
646
647 /*
648 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
649 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
650 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
651 * packet by copying the missing bits from the current MTC assuming the least
652 * difference between the two, and that the current MTC comes after last_mtc.
653 */
654 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
655 uint32_t *last_mtc)
656 {
657 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
658 uint32_t mask = ~(first_missing_bit - 1);
659
660 *last_mtc |= mtc & mask;
661 if (*last_mtc >= mtc) {
662 *last_mtc -= first_missing_bit;
663 *last_mtc &= 0xff;
664 }
665 }
666
667 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
668 {
669 struct intel_pt_decoder *decoder = pkt_info->decoder;
670 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
671 uint64_t timestamp;
672 double cyc_to_tsc;
673 unsigned int cbr;
674 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
675
676 switch (pkt_info->packet.type) {
677 case INTEL_PT_TNT:
678 case INTEL_PT_TIP_PGE:
679 case INTEL_PT_TIP:
680 case INTEL_PT_FUP:
681 case INTEL_PT_PSB:
682 case INTEL_PT_PIP:
683 case INTEL_PT_MODE_EXEC:
684 case INTEL_PT_MODE_TSX:
685 case INTEL_PT_PSBEND:
686 case INTEL_PT_PAD:
687 case INTEL_PT_VMCS:
688 case INTEL_PT_MNT:
689 case INTEL_PT_PTWRITE:
690 case INTEL_PT_PTWRITE_IP:
691 return 0;
692
693 case INTEL_PT_MTC:
694 if (!data->have_tma)
695 return 0;
696
697 mtc = pkt_info->packet.payload;
698 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
699 data->fixup_last_mtc = false;
700 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
701 &data->last_mtc);
702 }
703 if (mtc > data->last_mtc)
704 mtc_delta = mtc - data->last_mtc;
705 else
706 mtc_delta = mtc + 256 - data->last_mtc;
707 data->ctc_delta += mtc_delta << decoder->mtc_shift;
708 data->last_mtc = mtc;
709
710 if (decoder->tsc_ctc_mult) {
711 timestamp = data->ctc_timestamp +
712 data->ctc_delta * decoder->tsc_ctc_mult;
713 } else {
714 timestamp = data->ctc_timestamp +
715 multdiv(data->ctc_delta,
716 decoder->tsc_ctc_ratio_n,
717 decoder->tsc_ctc_ratio_d);
718 }
719
720 if (timestamp < data->timestamp)
721 return 1;
722
723 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
724 data->timestamp = timestamp;
725 return 0;
726 }
727
728 break;
729
730 case INTEL_PT_TSC:
731 /*
732 * For now, do not support using TSC packets - refer
733 * intel_pt_calc_cyc_to_tsc().
734 */
735 if (data->from_mtc)
736 return 1;
737 timestamp = pkt_info->packet.payload |
738 (data->timestamp & (0xffULL << 56));
739 if (data->from_mtc && timestamp < data->timestamp &&
740 data->timestamp - timestamp < decoder->tsc_slip)
741 return 1;
742 if (timestamp < data->timestamp)
743 timestamp += (1ULL << 56);
744 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
745 if (data->from_mtc)
746 return 1;
747 data->tsc_timestamp = timestamp;
748 data->timestamp = timestamp;
749 return 0;
750 }
751 break;
752
753 case INTEL_PT_TMA:
754 if (data->from_mtc)
755 return 1;
756
757 if (!decoder->tsc_ctc_ratio_d)
758 return 0;
759
760 ctc = pkt_info->packet.payload;
761 fc = pkt_info->packet.count;
762 ctc_rem = ctc & decoder->ctc_rem_mask;
763
764 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
765
766 data->ctc_timestamp = data->tsc_timestamp - fc;
767 if (decoder->tsc_ctc_mult) {
768 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
769 } else {
770 data->ctc_timestamp -=
771 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
772 decoder->tsc_ctc_ratio_d);
773 }
774
775 data->ctc_delta = 0;
776 data->have_tma = true;
777 data->fixup_last_mtc = true;
778
779 return 0;
780
781 case INTEL_PT_CYC:
782 data->cycle_cnt += pkt_info->packet.payload;
783 return 0;
784
785 case INTEL_PT_CBR:
786 cbr = pkt_info->packet.payload;
787 if (data->cbr && data->cbr != cbr)
788 return 1;
789 data->cbr = cbr;
790 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
791 return 0;
792
793 case INTEL_PT_TIP_PGD:
794 case INTEL_PT_TRACESTOP:
795 case INTEL_PT_EXSTOP:
796 case INTEL_PT_EXSTOP_IP:
797 case INTEL_PT_MWAIT:
798 case INTEL_PT_PWRE:
799 case INTEL_PT_PWRX:
800 case INTEL_PT_OVF:
801 case INTEL_PT_BAD: /* Does not happen */
802 default:
803 return 1;
804 }
805
806 if (!data->cbr && decoder->cbr) {
807 data->cbr = decoder->cbr;
808 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
809 }
810
811 if (!data->cycle_cnt)
812 return 1;
813
814 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
815
816 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
817 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
818 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
819 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
820 return 1;
821 }
822
823 decoder->calc_cyc_to_tsc = cyc_to_tsc;
824 decoder->have_calc_cyc_to_tsc = true;
825
826 if (data->cbr) {
827 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
828 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
829 } else {
830 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
831 cyc_to_tsc, pkt_info->pos);
832 }
833
834 return 1;
835 }
836
837 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
838 bool from_mtc)
839 {
840 struct intel_pt_calc_cyc_to_tsc_info data = {
841 .cycle_cnt = 0,
842 .cbr = 0,
843 .last_mtc = decoder->last_mtc,
844 .ctc_timestamp = decoder->ctc_timestamp,
845 .ctc_delta = decoder->ctc_delta,
846 .tsc_timestamp = decoder->tsc_timestamp,
847 .timestamp = decoder->timestamp,
848 .have_tma = decoder->have_tma,
849 .fixup_last_mtc = decoder->fixup_last_mtc,
850 .from_mtc = from_mtc,
851 .cbr_cyc_to_tsc = 0,
852 };
853
854 /*
855 * For now, do not support using TSC packets for at least the reasons:
856 * 1) timing might have stopped
857 * 2) TSC packets within PSB+ can slip against CYC packets
858 */
859 if (!from_mtc)
860 return;
861
862 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
863 }
864
865 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
866 {
867 int ret;
868
869 decoder->last_packet_type = decoder->packet.type;
870
871 do {
872 decoder->pos += decoder->pkt_step;
873 decoder->buf += decoder->pkt_step;
874 decoder->len -= decoder->pkt_step;
875
876 if (!decoder->len) {
877 ret = intel_pt_get_next_data(decoder);
878 if (ret)
879 return ret;
880 }
881
882 ret = intel_pt_get_packet(decoder->buf, decoder->len,
883 &decoder->packet);
884 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
885 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
886 ret = intel_pt_get_split_packet(decoder);
887 if (ret < 0)
888 return ret;
889 }
890 if (ret <= 0)
891 return intel_pt_bad_packet(decoder);
892
893 decoder->pkt_len = ret;
894 decoder->pkt_step = ret;
895 intel_pt_decoder_log_packet(decoder);
896 } while (decoder->packet.type == INTEL_PT_PAD);
897
898 return 0;
899 }
900
901 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
902 {
903 uint64_t timestamp, masked_timestamp;
904
905 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
906 masked_timestamp = timestamp & decoder->period_mask;
907 if (decoder->continuous_period) {
908 if (masked_timestamp > decoder->last_masked_timestamp)
909 return 1;
910 } else {
911 timestamp += 1;
912 masked_timestamp = timestamp & decoder->period_mask;
913 if (masked_timestamp > decoder->last_masked_timestamp) {
914 decoder->last_masked_timestamp = masked_timestamp;
915 decoder->continuous_period = true;
916 }
917 }
918
919 if (masked_timestamp < decoder->last_masked_timestamp)
920 return decoder->period_ticks;
921
922 return decoder->period_ticks - (timestamp - masked_timestamp);
923 }
924
925 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
926 {
927 switch (decoder->period_type) {
928 case INTEL_PT_PERIOD_INSTRUCTIONS:
929 return decoder->period - decoder->period_insn_cnt;
930 case INTEL_PT_PERIOD_TICKS:
931 return intel_pt_next_period(decoder);
932 case INTEL_PT_PERIOD_NONE:
933 case INTEL_PT_PERIOD_MTC:
934 default:
935 return 0;
936 }
937 }
938
939 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
940 {
941 uint64_t timestamp, masked_timestamp;
942
943 switch (decoder->period_type) {
944 case INTEL_PT_PERIOD_INSTRUCTIONS:
945 decoder->period_insn_cnt = 0;
946 break;
947 case INTEL_PT_PERIOD_TICKS:
948 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
949 masked_timestamp = timestamp & decoder->period_mask;
950 if (masked_timestamp > decoder->last_masked_timestamp)
951 decoder->last_masked_timestamp = masked_timestamp;
952 else
953 decoder->last_masked_timestamp += decoder->period_ticks;
954 break;
955 case INTEL_PT_PERIOD_NONE:
956 case INTEL_PT_PERIOD_MTC:
957 default:
958 break;
959 }
960
961 decoder->state.type |= INTEL_PT_INSTRUCTION;
962 }
963
964 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
965 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
966 {
967 uint64_t max_insn_cnt, insn_cnt = 0;
968 int err;
969
970 if (!decoder->mtc_insn)
971 decoder->mtc_insn = true;
972
973 max_insn_cnt = intel_pt_next_sample(decoder);
974
975 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
976 max_insn_cnt, decoder->data);
977
978 decoder->tot_insn_cnt += insn_cnt;
979 decoder->timestamp_insn_cnt += insn_cnt;
980 decoder->sample_insn_cnt += insn_cnt;
981 decoder->period_insn_cnt += insn_cnt;
982
983 if (err) {
984 decoder->no_progress = 0;
985 decoder->pkt_state = INTEL_PT_STATE_ERR2;
986 intel_pt_log_at("ERROR: Failed to get instruction",
987 decoder->ip);
988 if (err == -ENOENT)
989 return -ENOLINK;
990 return -EILSEQ;
991 }
992
993 if (ip && decoder->ip == ip) {
994 err = -EAGAIN;
995 goto out;
996 }
997
998 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
999 intel_pt_sample_insn(decoder);
1000
1001 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
1002 decoder->state.type = INTEL_PT_INSTRUCTION;
1003 decoder->state.from_ip = decoder->ip;
1004 decoder->state.to_ip = 0;
1005 decoder->ip += intel_pt_insn->length;
1006 err = INTEL_PT_RETURN;
1007 goto out;
1008 }
1009
1010 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
1011 /* Zero-length calls are excluded */
1012 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
1013 intel_pt_insn->rel) {
1014 err = intel_pt_push(&decoder->stack, decoder->ip +
1015 intel_pt_insn->length);
1016 if (err)
1017 goto out;
1018 }
1019 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1020 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1021 }
1022
1023 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1024 int cnt = decoder->no_progress++;
1025
1026 decoder->state.from_ip = decoder->ip;
1027 decoder->ip += intel_pt_insn->length +
1028 intel_pt_insn->rel;
1029 decoder->state.to_ip = decoder->ip;
1030 err = INTEL_PT_RETURN;
1031
1032 /*
1033 * Check for being stuck in a loop. This can happen if a
1034 * decoder error results in the decoder erroneously setting the
1035 * ip to an address that is itself in an infinite loop that
1036 * consumes no packets. When that happens, there must be an
1037 * unconditional branch.
1038 */
1039 if (cnt) {
1040 if (cnt == 1) {
1041 decoder->stuck_ip = decoder->state.to_ip;
1042 decoder->stuck_ip_prd = 1;
1043 decoder->stuck_ip_cnt = 1;
1044 } else if (cnt > INTEL_PT_MAX_LOOPS ||
1045 decoder->state.to_ip == decoder->stuck_ip) {
1046 intel_pt_log_at("ERROR: Never-ending loop",
1047 decoder->state.to_ip);
1048 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1049 err = -ELOOP;
1050 goto out;
1051 } else if (!--decoder->stuck_ip_cnt) {
1052 decoder->stuck_ip_prd += 1;
1053 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1054 decoder->stuck_ip = decoder->state.to_ip;
1055 }
1056 }
1057 goto out_no_progress;
1058 }
1059 out:
1060 decoder->no_progress = 0;
1061 out_no_progress:
1062 decoder->state.insn_op = intel_pt_insn->op;
1063 decoder->state.insn_len = intel_pt_insn->length;
1064 memcpy(decoder->state.insn, intel_pt_insn->buf,
1065 INTEL_PT_INSN_BUF_SZ);
1066
1067 if (decoder->tx_flags & INTEL_PT_IN_TX)
1068 decoder->state.flags |= INTEL_PT_IN_TX;
1069
1070 return err;
1071 }
1072
1073 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1074 {
1075 bool ret = false;
1076
1077 if (decoder->set_fup_tx_flags) {
1078 decoder->set_fup_tx_flags = false;
1079 decoder->tx_flags = decoder->fup_tx_flags;
1080 decoder->state.type = INTEL_PT_TRANSACTION;
1081 decoder->state.from_ip = decoder->ip;
1082 decoder->state.to_ip = 0;
1083 decoder->state.flags = decoder->fup_tx_flags;
1084 return true;
1085 }
1086 if (decoder->set_fup_ptw) {
1087 decoder->set_fup_ptw = false;
1088 decoder->state.type = INTEL_PT_PTW;
1089 decoder->state.flags |= INTEL_PT_FUP_IP;
1090 decoder->state.from_ip = decoder->ip;
1091 decoder->state.to_ip = 0;
1092 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1093 return true;
1094 }
1095 if (decoder->set_fup_mwait) {
1096 decoder->set_fup_mwait = false;
1097 decoder->state.type = INTEL_PT_MWAIT_OP;
1098 decoder->state.from_ip = decoder->ip;
1099 decoder->state.to_ip = 0;
1100 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1101 ret = true;
1102 }
1103 if (decoder->set_fup_pwre) {
1104 decoder->set_fup_pwre = false;
1105 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1106 decoder->state.type &= ~INTEL_PT_BRANCH;
1107 decoder->state.from_ip = decoder->ip;
1108 decoder->state.to_ip = 0;
1109 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1110 ret = true;
1111 }
1112 if (decoder->set_fup_exstop) {
1113 decoder->set_fup_exstop = false;
1114 decoder->state.type |= INTEL_PT_EX_STOP;
1115 decoder->state.type &= ~INTEL_PT_BRANCH;
1116 decoder->state.flags |= INTEL_PT_FUP_IP;
1117 decoder->state.from_ip = decoder->ip;
1118 decoder->state.to_ip = 0;
1119 ret = true;
1120 }
1121 return ret;
1122 }
1123
1124 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1125 struct intel_pt_insn *intel_pt_insn,
1126 uint64_t ip, int err)
1127 {
1128 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1129 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1130 ip == decoder->ip + intel_pt_insn->length;
1131 }
1132
1133 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1134 {
1135 struct intel_pt_insn intel_pt_insn;
1136 uint64_t ip;
1137 int err;
1138
1139 ip = decoder->last_ip;
1140
1141 while (1) {
1142 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1143 if (err == INTEL_PT_RETURN)
1144 return 0;
1145 if (err == -EAGAIN ||
1146 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1147 if (intel_pt_fup_event(decoder))
1148 return 0;
1149 return -EAGAIN;
1150 }
1151 decoder->set_fup_tx_flags = false;
1152 if (err)
1153 return err;
1154
1155 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1156 intel_pt_log_at("ERROR: Unexpected indirect branch",
1157 decoder->ip);
1158 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1159 return -ENOENT;
1160 }
1161
1162 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1163 intel_pt_log_at("ERROR: Unexpected conditional branch",
1164 decoder->ip);
1165 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1166 return -ENOENT;
1167 }
1168
1169 intel_pt_bug(decoder);
1170 }
1171 }
1172
1173 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1174 {
1175 struct intel_pt_insn intel_pt_insn;
1176 int err;
1177
1178 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1179 if (err == INTEL_PT_RETURN &&
1180 decoder->pgd_ip &&
1181 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1182 (decoder->state.type & INTEL_PT_BRANCH) &&
1183 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1184 /* Unconditional branch leaving filter region */
1185 decoder->no_progress = 0;
1186 decoder->pge = false;
1187 decoder->continuous_period = false;
1188 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1189 decoder->state.type |= INTEL_PT_TRACE_END;
1190 return 0;
1191 }
1192 if (err == INTEL_PT_RETURN)
1193 return 0;
1194 if (err)
1195 return err;
1196
1197 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1198 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1199 decoder->pge = false;
1200 decoder->continuous_period = false;
1201 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1202 decoder->state.from_ip = decoder->ip;
1203 if (decoder->packet.count == 0) {
1204 decoder->state.to_ip = 0;
1205 } else {
1206 decoder->state.to_ip = decoder->last_ip;
1207 decoder->ip = decoder->last_ip;
1208 }
1209 decoder->state.type |= INTEL_PT_TRACE_END;
1210 } else {
1211 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1212 decoder->state.from_ip = decoder->ip;
1213 if (decoder->packet.count == 0) {
1214 decoder->state.to_ip = 0;
1215 } else {
1216 decoder->state.to_ip = decoder->last_ip;
1217 decoder->ip = decoder->last_ip;
1218 }
1219 }
1220 return 0;
1221 }
1222
1223 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1224 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1225 intel_pt_insn.rel;
1226
1227 if (decoder->pgd_ip &&
1228 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1229 decoder->pgd_ip(to_ip, decoder->data)) {
1230 /* Conditional branch leaving filter region */
1231 decoder->pge = false;
1232 decoder->continuous_period = false;
1233 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1234 decoder->ip = to_ip;
1235 decoder->state.from_ip = decoder->ip;
1236 decoder->state.to_ip = to_ip;
1237 decoder->state.type |= INTEL_PT_TRACE_END;
1238 return 0;
1239 }
1240 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1241 decoder->ip);
1242 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1243 return -ENOENT;
1244 }
1245
1246 return intel_pt_bug(decoder);
1247 }
1248
1249 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1250 {
1251 struct intel_pt_insn intel_pt_insn;
1252 int err;
1253
1254 while (1) {
1255 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1256 if (err == INTEL_PT_RETURN)
1257 return 0;
1258 if (err)
1259 return err;
1260
1261 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1262 if (!decoder->return_compression) {
1263 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1264 decoder->ip);
1265 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1266 return -ENOENT;
1267 }
1268 if (!decoder->ret_addr) {
1269 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1270 decoder->ip);
1271 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1272 return -ENOENT;
1273 }
1274 if (!(decoder->tnt.payload & BIT63)) {
1275 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1276 decoder->ip);
1277 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1278 return -ENOENT;
1279 }
1280 decoder->tnt.count -= 1;
1281 if (decoder->tnt.count)
1282 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1283 else
1284 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1285 decoder->tnt.payload <<= 1;
1286 decoder->state.from_ip = decoder->ip;
1287 decoder->ip = decoder->ret_addr;
1288 decoder->state.to_ip = decoder->ip;
1289 return 0;
1290 }
1291
1292 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1293 /* Handle deferred TIPs */
1294 err = intel_pt_get_next_packet(decoder);
1295 if (err)
1296 return err;
1297 if (decoder->packet.type != INTEL_PT_TIP ||
1298 decoder->packet.count == 0) {
1299 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1300 decoder->ip);
1301 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1302 decoder->pkt_step = 0;
1303 return -ENOENT;
1304 }
1305 intel_pt_set_last_ip(decoder);
1306 decoder->state.from_ip = decoder->ip;
1307 decoder->state.to_ip = decoder->last_ip;
1308 decoder->ip = decoder->last_ip;
1309 return 0;
1310 }
1311
1312 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1313 decoder->tnt.count -= 1;
1314 if (decoder->tnt.count)
1315 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1316 else
1317 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1318 if (decoder->tnt.payload & BIT63) {
1319 decoder->tnt.payload <<= 1;
1320 decoder->state.from_ip = decoder->ip;
1321 decoder->ip += intel_pt_insn.length +
1322 intel_pt_insn.rel;
1323 decoder->state.to_ip = decoder->ip;
1324 return 0;
1325 }
1326 /* Instruction sample for a non-taken branch */
1327 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1328 decoder->tnt.payload <<= 1;
1329 decoder->state.type = INTEL_PT_INSTRUCTION;
1330 decoder->state.from_ip = decoder->ip;
1331 decoder->state.to_ip = 0;
1332 decoder->ip += intel_pt_insn.length;
1333 return 0;
1334 }
1335 decoder->sample_cyc = false;
1336 decoder->ip += intel_pt_insn.length;
1337 if (!decoder->tnt.count) {
1338 intel_pt_update_sample_time(decoder);
1339 return -EAGAIN;
1340 }
1341 decoder->tnt.payload <<= 1;
1342 continue;
1343 }
1344
1345 return intel_pt_bug(decoder);
1346 }
1347 }
1348
1349 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1350 {
1351 unsigned int fup_tx_flags;
1352 int err;
1353
1354 fup_tx_flags = decoder->packet.payload &
1355 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1356 err = intel_pt_get_next_packet(decoder);
1357 if (err)
1358 return err;
1359 if (decoder->packet.type == INTEL_PT_FUP) {
1360 decoder->fup_tx_flags = fup_tx_flags;
1361 decoder->set_fup_tx_flags = true;
1362 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1363 *no_tip = true;
1364 } else {
1365 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1366 decoder->pos);
1367 intel_pt_update_in_tx(decoder);
1368 }
1369 return 0;
1370 }
1371
1372 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1373 {
1374 uint64_t timestamp;
1375
1376 decoder->have_tma = false;
1377
1378 if (decoder->ref_timestamp) {
1379 timestamp = decoder->packet.payload |
1380 (decoder->ref_timestamp & (0xffULL << 56));
1381 if (timestamp < decoder->ref_timestamp) {
1382 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1383 timestamp += (1ULL << 56);
1384 } else {
1385 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1386 timestamp -= (1ULL << 56);
1387 }
1388 decoder->tsc_timestamp = timestamp;
1389 decoder->timestamp = timestamp;
1390 decoder->ref_timestamp = 0;
1391 decoder->timestamp_insn_cnt = 0;
1392 } else if (decoder->timestamp) {
1393 timestamp = decoder->packet.payload |
1394 (decoder->timestamp & (0xffULL << 56));
1395 decoder->tsc_timestamp = timestamp;
1396 if (timestamp < decoder->timestamp &&
1397 decoder->timestamp - timestamp < decoder->tsc_slip) {
1398 intel_pt_log_to("Suppressing backwards timestamp",
1399 timestamp);
1400 timestamp = decoder->timestamp;
1401 }
1402 if (timestamp < decoder->timestamp) {
1403 intel_pt_log_to("Wraparound timestamp", timestamp);
1404 timestamp += (1ULL << 56);
1405 decoder->tsc_timestamp = timestamp;
1406 }
1407 decoder->timestamp = timestamp;
1408 decoder->timestamp_insn_cnt = 0;
1409 }
1410
1411 if (decoder->last_packet_type == INTEL_PT_CYC) {
1412 decoder->cyc_ref_timestamp = decoder->timestamp;
1413 decoder->cycle_cnt = 0;
1414 decoder->have_calc_cyc_to_tsc = false;
1415 intel_pt_calc_cyc_to_tsc(decoder, false);
1416 }
1417
1418 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1419 }
1420
1421 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1422 {
1423 intel_pt_log("ERROR: Buffer overflow\n");
1424 intel_pt_clear_tx_flags(decoder);
1425 decoder->timestamp_insn_cnt = 0;
1426 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1427 decoder->overflow = true;
1428 return -EOVERFLOW;
1429 }
1430
1431 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)
1432 {
1433 if (decoder->have_cyc)
1434 return;
1435
1436 decoder->cyc_cnt_timestamp = decoder->timestamp;
1437 decoder->base_cyc_cnt = decoder->tot_cyc_cnt;
1438 }
1439
1440 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)
1441 {
1442 decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;
1443
1444 if (decoder->pge)
1445 intel_pt_mtc_cyc_cnt_pge(decoder);
1446 }
1447
1448 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)
1449 {
1450 uint64_t tot_cyc_cnt, tsc_delta;
1451
1452 if (decoder->have_cyc)
1453 return;
1454
1455 decoder->sample_cyc = true;
1456
1457 if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)
1458 return;
1459
1460 tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;
1461 tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;
1462
1463 if (tot_cyc_cnt > decoder->tot_cyc_cnt)
1464 decoder->tot_cyc_cnt = tot_cyc_cnt;
1465 }
1466
1467 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1468 {
1469 uint32_t ctc = decoder->packet.payload;
1470 uint32_t fc = decoder->packet.count;
1471 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1472
1473 if (!decoder->tsc_ctc_ratio_d)
1474 return;
1475
1476 if (decoder->pge && !decoder->in_psb)
1477 intel_pt_mtc_cyc_cnt_pge(decoder);
1478 else
1479 intel_pt_mtc_cyc_cnt_upd(decoder);
1480
1481 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1482 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1483 if (decoder->tsc_ctc_mult) {
1484 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1485 } else {
1486 decoder->ctc_timestamp -= multdiv(ctc_rem,
1487 decoder->tsc_ctc_ratio_n,
1488 decoder->tsc_ctc_ratio_d);
1489 }
1490 decoder->ctc_delta = 0;
1491 decoder->have_tma = true;
1492 decoder->fixup_last_mtc = true;
1493 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1494 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1495 }
1496
1497 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1498 {
1499 uint64_t timestamp;
1500 uint32_t mtc, mtc_delta;
1501
1502 if (!decoder->have_tma)
1503 return;
1504
1505 mtc = decoder->packet.payload;
1506
1507 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1508 decoder->fixup_last_mtc = false;
1509 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1510 &decoder->last_mtc);
1511 }
1512
1513 if (mtc > decoder->last_mtc)
1514 mtc_delta = mtc - decoder->last_mtc;
1515 else
1516 mtc_delta = mtc + 256 - decoder->last_mtc;
1517
1518 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1519
1520 if (decoder->tsc_ctc_mult) {
1521 timestamp = decoder->ctc_timestamp +
1522 decoder->ctc_delta * decoder->tsc_ctc_mult;
1523 } else {
1524 timestamp = decoder->ctc_timestamp +
1525 multdiv(decoder->ctc_delta,
1526 decoder->tsc_ctc_ratio_n,
1527 decoder->tsc_ctc_ratio_d);
1528 }
1529
1530 if (timestamp < decoder->timestamp)
1531 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1532 timestamp, decoder->timestamp);
1533 else
1534 decoder->timestamp = timestamp;
1535
1536 intel_pt_mtc_cyc_cnt_upd(decoder);
1537
1538 decoder->timestamp_insn_cnt = 0;
1539 decoder->last_mtc = mtc;
1540
1541 if (decoder->last_packet_type == INTEL_PT_CYC) {
1542 decoder->cyc_ref_timestamp = decoder->timestamp;
1543 decoder->cycle_cnt = 0;
1544 decoder->have_calc_cyc_to_tsc = false;
1545 intel_pt_calc_cyc_to_tsc(decoder, true);
1546 }
1547
1548 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1549 }
1550
1551 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1552 {
1553 unsigned int cbr = decoder->packet.payload & 0xff;
1554
1555 decoder->cbr_payload = decoder->packet.payload;
1556
1557 if (decoder->cbr == cbr)
1558 return;
1559
1560 decoder->cbr = cbr;
1561 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1562
1563 intel_pt_mtc_cyc_cnt_cbr(decoder);
1564 }
1565
1566 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1567 {
1568 uint64_t timestamp = decoder->cyc_ref_timestamp;
1569
1570 decoder->have_cyc = true;
1571
1572 decoder->cycle_cnt += decoder->packet.payload;
1573 if (decoder->pge)
1574 decoder->tot_cyc_cnt += decoder->packet.payload;
1575 decoder->sample_cyc = true;
1576
1577 if (!decoder->cyc_ref_timestamp)
1578 return;
1579
1580 if (decoder->have_calc_cyc_to_tsc)
1581 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1582 else if (decoder->cbr)
1583 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1584 else
1585 return;
1586
1587 if (timestamp < decoder->timestamp)
1588 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1589 timestamp, decoder->timestamp);
1590 else
1591 decoder->timestamp = timestamp;
1592
1593 decoder->timestamp_insn_cnt = 0;
1594
1595 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1596 }
1597
1598 /* Walk PSB+ packets when already in sync. */
1599 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1600 {
1601 int err;
1602
1603 decoder->in_psb = true;
1604
1605 while (1) {
1606 err = intel_pt_get_next_packet(decoder);
1607 if (err)
1608 goto out;
1609
1610 switch (decoder->packet.type) {
1611 case INTEL_PT_PSBEND:
1612 err = 0;
1613 goto out;
1614
1615 case INTEL_PT_TIP_PGD:
1616 case INTEL_PT_TIP_PGE:
1617 case INTEL_PT_TIP:
1618 case INTEL_PT_TNT:
1619 case INTEL_PT_TRACESTOP:
1620 case INTEL_PT_BAD:
1621 case INTEL_PT_PSB:
1622 case INTEL_PT_PTWRITE:
1623 case INTEL_PT_PTWRITE_IP:
1624 case INTEL_PT_EXSTOP:
1625 case INTEL_PT_EXSTOP_IP:
1626 case INTEL_PT_MWAIT:
1627 case INTEL_PT_PWRE:
1628 case INTEL_PT_PWRX:
1629 decoder->have_tma = false;
1630 intel_pt_log("ERROR: Unexpected packet\n");
1631 err = -EAGAIN;
1632 goto out;
1633
1634 case INTEL_PT_OVF:
1635 err = intel_pt_overflow(decoder);
1636 goto out;
1637
1638 case INTEL_PT_TSC:
1639 intel_pt_calc_tsc_timestamp(decoder);
1640 break;
1641
1642 case INTEL_PT_TMA:
1643 intel_pt_calc_tma(decoder);
1644 break;
1645
1646 case INTEL_PT_CBR:
1647 intel_pt_calc_cbr(decoder);
1648 break;
1649
1650 case INTEL_PT_MODE_EXEC:
1651 decoder->exec_mode = decoder->packet.payload;
1652 break;
1653
1654 case INTEL_PT_PIP:
1655 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1656 break;
1657
1658 case INTEL_PT_FUP:
1659 decoder->pge = true;
1660 if (decoder->packet.count)
1661 intel_pt_set_last_ip(decoder);
1662 break;
1663
1664 case INTEL_PT_MODE_TSX:
1665 intel_pt_update_in_tx(decoder);
1666 break;
1667
1668 case INTEL_PT_MTC:
1669 intel_pt_calc_mtc_timestamp(decoder);
1670 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1671 decoder->state.type |= INTEL_PT_INSTRUCTION;
1672 break;
1673
1674 case INTEL_PT_CYC:
1675 case INTEL_PT_VMCS:
1676 case INTEL_PT_MNT:
1677 case INTEL_PT_PAD:
1678 default:
1679 break;
1680 }
1681 }
1682 out:
1683 decoder->in_psb = false;
1684
1685 return err;
1686 }
1687
1688 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1689 {
1690 int err;
1691
1692 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1693 decoder->tx_flags = 0;
1694 decoder->state.flags &= ~INTEL_PT_IN_TX;
1695 decoder->state.flags |= INTEL_PT_ABORT_TX;
1696 } else {
1697 decoder->state.flags |= INTEL_PT_ASYNC;
1698 }
1699
1700 while (1) {
1701 err = intel_pt_get_next_packet(decoder);
1702 if (err)
1703 return err;
1704
1705 switch (decoder->packet.type) {
1706 case INTEL_PT_TNT:
1707 case INTEL_PT_FUP:
1708 case INTEL_PT_TRACESTOP:
1709 case INTEL_PT_PSB:
1710 case INTEL_PT_TSC:
1711 case INTEL_PT_TMA:
1712 case INTEL_PT_MODE_TSX:
1713 case INTEL_PT_BAD:
1714 case INTEL_PT_PSBEND:
1715 case INTEL_PT_PTWRITE:
1716 case INTEL_PT_PTWRITE_IP:
1717 case INTEL_PT_EXSTOP:
1718 case INTEL_PT_EXSTOP_IP:
1719 case INTEL_PT_MWAIT:
1720 case INTEL_PT_PWRE:
1721 case INTEL_PT_PWRX:
1722 intel_pt_log("ERROR: Missing TIP after FUP\n");
1723 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1724 decoder->pkt_step = 0;
1725 return -ENOENT;
1726
1727 case INTEL_PT_CBR:
1728 intel_pt_calc_cbr(decoder);
1729 break;
1730
1731 case INTEL_PT_OVF:
1732 return intel_pt_overflow(decoder);
1733
1734 case INTEL_PT_TIP_PGD:
1735 decoder->state.from_ip = decoder->ip;
1736 if (decoder->packet.count == 0) {
1737 decoder->state.to_ip = 0;
1738 } else {
1739 intel_pt_set_ip(decoder);
1740 decoder->state.to_ip = decoder->ip;
1741 }
1742 decoder->pge = false;
1743 decoder->continuous_period = false;
1744 decoder->state.type |= INTEL_PT_TRACE_END;
1745 return 0;
1746
1747 case INTEL_PT_TIP_PGE:
1748 decoder->pge = true;
1749 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1750 decoder->ip);
1751 decoder->state.from_ip = 0;
1752 if (decoder->packet.count == 0) {
1753 decoder->state.to_ip = 0;
1754 } else {
1755 intel_pt_set_ip(decoder);
1756 decoder->state.to_ip = decoder->ip;
1757 }
1758 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1759 intel_pt_mtc_cyc_cnt_pge(decoder);
1760 return 0;
1761
1762 case INTEL_PT_TIP:
1763 decoder->state.from_ip = decoder->ip;
1764 if (decoder->packet.count == 0) {
1765 decoder->state.to_ip = 0;
1766 } else {
1767 intel_pt_set_ip(decoder);
1768 decoder->state.to_ip = decoder->ip;
1769 }
1770 return 0;
1771
1772 case INTEL_PT_PIP:
1773 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1774 break;
1775
1776 case INTEL_PT_MTC:
1777 intel_pt_calc_mtc_timestamp(decoder);
1778 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1779 decoder->state.type |= INTEL_PT_INSTRUCTION;
1780 break;
1781
1782 case INTEL_PT_CYC:
1783 intel_pt_calc_cyc_timestamp(decoder);
1784 break;
1785
1786 case INTEL_PT_MODE_EXEC:
1787 decoder->exec_mode = decoder->packet.payload;
1788 break;
1789
1790 case INTEL_PT_VMCS:
1791 case INTEL_PT_MNT:
1792 case INTEL_PT_PAD:
1793 break;
1794
1795 default:
1796 return intel_pt_bug(decoder);
1797 }
1798 }
1799 }
1800
1801 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1802 {
1803 bool no_tip = false;
1804 int err;
1805
1806 while (1) {
1807 err = intel_pt_get_next_packet(decoder);
1808 if (err)
1809 return err;
1810 next:
1811 switch (decoder->packet.type) {
1812 case INTEL_PT_TNT:
1813 if (!decoder->packet.count)
1814 break;
1815 decoder->tnt = decoder->packet;
1816 decoder->pkt_state = INTEL_PT_STATE_TNT;
1817 err = intel_pt_walk_tnt(decoder);
1818 if (err == -EAGAIN)
1819 break;
1820 return err;
1821
1822 case INTEL_PT_TIP_PGD:
1823 if (decoder->packet.count != 0)
1824 intel_pt_set_last_ip(decoder);
1825 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1826 return intel_pt_walk_tip(decoder);
1827
1828 case INTEL_PT_TIP_PGE: {
1829 decoder->pge = true;
1830 intel_pt_mtc_cyc_cnt_pge(decoder);
1831 if (decoder->packet.count == 0) {
1832 intel_pt_log_at("Skipping zero TIP.PGE",
1833 decoder->pos);
1834 break;
1835 }
1836 intel_pt_set_ip(decoder);
1837 decoder->state.from_ip = 0;
1838 decoder->state.to_ip = decoder->ip;
1839 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1840 return 0;
1841 }
1842
1843 case INTEL_PT_OVF:
1844 return intel_pt_overflow(decoder);
1845
1846 case INTEL_PT_TIP:
1847 if (decoder->packet.count != 0)
1848 intel_pt_set_last_ip(decoder);
1849 decoder->pkt_state = INTEL_PT_STATE_TIP;
1850 return intel_pt_walk_tip(decoder);
1851
1852 case INTEL_PT_FUP:
1853 if (decoder->packet.count == 0) {
1854 intel_pt_log_at("Skipping zero FUP",
1855 decoder->pos);
1856 no_tip = false;
1857 break;
1858 }
1859 intel_pt_set_last_ip(decoder);
1860 if (!decoder->branch_enable) {
1861 decoder->ip = decoder->last_ip;
1862 if (intel_pt_fup_event(decoder))
1863 return 0;
1864 no_tip = false;
1865 break;
1866 }
1867 if (decoder->set_fup_mwait)
1868 no_tip = true;
1869 err = intel_pt_walk_fup(decoder);
1870 if (err != -EAGAIN) {
1871 if (err)
1872 return err;
1873 if (no_tip)
1874 decoder->pkt_state =
1875 INTEL_PT_STATE_FUP_NO_TIP;
1876 else
1877 decoder->pkt_state = INTEL_PT_STATE_FUP;
1878 return 0;
1879 }
1880 if (no_tip) {
1881 no_tip = false;
1882 break;
1883 }
1884 return intel_pt_walk_fup_tip(decoder);
1885
1886 case INTEL_PT_TRACESTOP:
1887 decoder->pge = false;
1888 decoder->continuous_period = false;
1889 intel_pt_clear_tx_flags(decoder);
1890 decoder->have_tma = false;
1891 break;
1892
1893 case INTEL_PT_PSB:
1894 decoder->last_ip = 0;
1895 decoder->have_last_ip = true;
1896 intel_pt_clear_stack(&decoder->stack);
1897 err = intel_pt_walk_psbend(decoder);
1898 if (err == -EAGAIN)
1899 goto next;
1900 if (err)
1901 return err;
1902 break;
1903
1904 case INTEL_PT_PIP:
1905 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1906 break;
1907
1908 case INTEL_PT_MTC:
1909 intel_pt_calc_mtc_timestamp(decoder);
1910 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1911 break;
1912 /*
1913 * Ensure that there has been an instruction since the
1914 * last MTC.
1915 */
1916 if (!decoder->mtc_insn)
1917 break;
1918 decoder->mtc_insn = false;
1919 /* Ensure that there is a timestamp */
1920 if (!decoder->timestamp)
1921 break;
1922 decoder->state.type = INTEL_PT_INSTRUCTION;
1923 decoder->state.from_ip = decoder->ip;
1924 decoder->state.to_ip = 0;
1925 decoder->mtc_insn = false;
1926 return 0;
1927
1928 case INTEL_PT_TSC:
1929 intel_pt_calc_tsc_timestamp(decoder);
1930 break;
1931
1932 case INTEL_PT_TMA:
1933 intel_pt_calc_tma(decoder);
1934 break;
1935
1936 case INTEL_PT_CYC:
1937 intel_pt_calc_cyc_timestamp(decoder);
1938 break;
1939
1940 case INTEL_PT_CBR:
1941 intel_pt_calc_cbr(decoder);
1942 if (!decoder->branch_enable &&
1943 decoder->cbr != decoder->cbr_seen) {
1944 decoder->cbr_seen = decoder->cbr;
1945 decoder->state.type = INTEL_PT_CBR_CHG;
1946 decoder->state.from_ip = decoder->ip;
1947 decoder->state.to_ip = 0;
1948 decoder->state.cbr_payload =
1949 decoder->packet.payload;
1950 return 0;
1951 }
1952 break;
1953
1954 case INTEL_PT_MODE_EXEC:
1955 decoder->exec_mode = decoder->packet.payload;
1956 break;
1957
1958 case INTEL_PT_MODE_TSX:
1959 /* MODE_TSX need not be followed by FUP */
1960 if (!decoder->pge) {
1961 intel_pt_update_in_tx(decoder);
1962 break;
1963 }
1964 err = intel_pt_mode_tsx(decoder, &no_tip);
1965 if (err)
1966 return err;
1967 goto next;
1968
1969 case INTEL_PT_BAD: /* Does not happen */
1970 return intel_pt_bug(decoder);
1971
1972 case INTEL_PT_PSBEND:
1973 case INTEL_PT_VMCS:
1974 case INTEL_PT_MNT:
1975 case INTEL_PT_PAD:
1976 break;
1977
1978 case INTEL_PT_PTWRITE_IP:
1979 decoder->fup_ptw_payload = decoder->packet.payload;
1980 err = intel_pt_get_next_packet(decoder);
1981 if (err)
1982 return err;
1983 if (decoder->packet.type == INTEL_PT_FUP) {
1984 decoder->set_fup_ptw = true;
1985 no_tip = true;
1986 } else {
1987 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1988 decoder->pos);
1989 }
1990 goto next;
1991
1992 case INTEL_PT_PTWRITE:
1993 decoder->state.type = INTEL_PT_PTW;
1994 decoder->state.from_ip = decoder->ip;
1995 decoder->state.to_ip = 0;
1996 decoder->state.ptw_payload = decoder->packet.payload;
1997 return 0;
1998
1999 case INTEL_PT_MWAIT:
2000 decoder->fup_mwait_payload = decoder->packet.payload;
2001 decoder->set_fup_mwait = true;
2002 break;
2003
2004 case INTEL_PT_PWRE:
2005 if (decoder->set_fup_mwait) {
2006 decoder->fup_pwre_payload =
2007 decoder->packet.payload;
2008 decoder->set_fup_pwre = true;
2009 break;
2010 }
2011 decoder->state.type = INTEL_PT_PWR_ENTRY;
2012 decoder->state.from_ip = decoder->ip;
2013 decoder->state.to_ip = 0;
2014 decoder->state.pwrx_payload = decoder->packet.payload;
2015 return 0;
2016
2017 case INTEL_PT_EXSTOP_IP:
2018 err = intel_pt_get_next_packet(decoder);
2019 if (err)
2020 return err;
2021 if (decoder->packet.type == INTEL_PT_FUP) {
2022 decoder->set_fup_exstop = true;
2023 no_tip = true;
2024 } else {
2025 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
2026 decoder->pos);
2027 }
2028 goto next;
2029
2030 case INTEL_PT_EXSTOP:
2031 decoder->state.type = INTEL_PT_EX_STOP;
2032 decoder->state.from_ip = decoder->ip;
2033 decoder->state.to_ip = 0;
2034 return 0;
2035
2036 case INTEL_PT_PWRX:
2037 decoder->state.type = INTEL_PT_PWR_EXIT;
2038 decoder->state.from_ip = decoder->ip;
2039 decoder->state.to_ip = 0;
2040 decoder->state.pwrx_payload = decoder->packet.payload;
2041 return 0;
2042
2043 default:
2044 return intel_pt_bug(decoder);
2045 }
2046 }
2047 }
2048
2049 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
2050 {
2051 return decoder->packet.count &&
2052 (decoder->have_last_ip || decoder->packet.count == 3 ||
2053 decoder->packet.count == 6);
2054 }
2055
2056 /* Walk PSB+ packets to get in sync. */
2057 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
2058 {
2059 int err;
2060
2061 decoder->in_psb = true;
2062
2063 while (1) {
2064 err = intel_pt_get_next_packet(decoder);
2065 if (err)
2066 goto out;
2067
2068 switch (decoder->packet.type) {
2069 case INTEL_PT_TIP_PGD:
2070 decoder->continuous_period = false;
2071 __fallthrough;
2072 case INTEL_PT_TIP_PGE:
2073 case INTEL_PT_TIP:
2074 case INTEL_PT_PTWRITE:
2075 case INTEL_PT_PTWRITE_IP:
2076 case INTEL_PT_EXSTOP:
2077 case INTEL_PT_EXSTOP_IP:
2078 case INTEL_PT_MWAIT:
2079 case INTEL_PT_PWRE:
2080 case INTEL_PT_PWRX:
2081 intel_pt_log("ERROR: Unexpected packet\n");
2082 err = -ENOENT;
2083 goto out;
2084
2085 case INTEL_PT_FUP:
2086 decoder->pge = true;
2087 if (intel_pt_have_ip(decoder)) {
2088 uint64_t current_ip = decoder->ip;
2089
2090 intel_pt_set_ip(decoder);
2091 if (current_ip)
2092 intel_pt_log_to("Setting IP",
2093 decoder->ip);
2094 }
2095 break;
2096
2097 case INTEL_PT_MTC:
2098 intel_pt_calc_mtc_timestamp(decoder);
2099 break;
2100
2101 case INTEL_PT_TSC:
2102 intel_pt_calc_tsc_timestamp(decoder);
2103 break;
2104
2105 case INTEL_PT_TMA:
2106 intel_pt_calc_tma(decoder);
2107 break;
2108
2109 case INTEL_PT_CYC:
2110 intel_pt_calc_cyc_timestamp(decoder);
2111 break;
2112
2113 case INTEL_PT_CBR:
2114 intel_pt_calc_cbr(decoder);
2115 break;
2116
2117 case INTEL_PT_PIP:
2118 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2119 break;
2120
2121 case INTEL_PT_MODE_EXEC:
2122 decoder->exec_mode = decoder->packet.payload;
2123 break;
2124
2125 case INTEL_PT_MODE_TSX:
2126 intel_pt_update_in_tx(decoder);
2127 break;
2128
2129 case INTEL_PT_TRACESTOP:
2130 decoder->pge = false;
2131 decoder->continuous_period = false;
2132 intel_pt_clear_tx_flags(decoder);
2133 __fallthrough;
2134
2135 case INTEL_PT_TNT:
2136 decoder->have_tma = false;
2137 intel_pt_log("ERROR: Unexpected packet\n");
2138 if (decoder->ip)
2139 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2140 else
2141 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2142 err = -ENOENT;
2143 goto out;
2144
2145 case INTEL_PT_BAD: /* Does not happen */
2146 err = intel_pt_bug(decoder);
2147 goto out;
2148
2149 case INTEL_PT_OVF:
2150 err = intel_pt_overflow(decoder);
2151 goto out;
2152
2153 case INTEL_PT_PSBEND:
2154 err = 0;
2155 goto out;
2156
2157 case INTEL_PT_PSB:
2158 case INTEL_PT_VMCS:
2159 case INTEL_PT_MNT:
2160 case INTEL_PT_PAD:
2161 default:
2162 break;
2163 }
2164 }
2165 out:
2166 decoder->in_psb = false;
2167
2168 return err;
2169 }
2170
2171 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2172 {
2173 int err;
2174
2175 while (1) {
2176 err = intel_pt_get_next_packet(decoder);
2177 if (err)
2178 return err;
2179
2180 switch (decoder->packet.type) {
2181 case INTEL_PT_TIP_PGD:
2182 decoder->continuous_period = false;
2183 decoder->pge = false;
2184 if (intel_pt_have_ip(decoder))
2185 intel_pt_set_ip(decoder);
2186 if (!decoder->ip)
2187 break;
2188 decoder->state.type |= INTEL_PT_TRACE_END;
2189 return 0;
2190
2191 case INTEL_PT_TIP_PGE:
2192 decoder->pge = true;
2193 intel_pt_mtc_cyc_cnt_pge(decoder);
2194 if (intel_pt_have_ip(decoder))
2195 intel_pt_set_ip(decoder);
2196 if (!decoder->ip)
2197 break;
2198 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2199 return 0;
2200
2201 case INTEL_PT_TIP:
2202 decoder->pge = true;
2203 if (intel_pt_have_ip(decoder))
2204 intel_pt_set_ip(decoder);
2205 if (!decoder->ip)
2206 break;
2207 return 0;
2208
2209 case INTEL_PT_FUP:
2210 if (intel_pt_have_ip(decoder))
2211 intel_pt_set_ip(decoder);
2212 if (decoder->ip)
2213 return 0;
2214 break;
2215
2216 case INTEL_PT_MTC:
2217 intel_pt_calc_mtc_timestamp(decoder);
2218 break;
2219
2220 case INTEL_PT_TSC:
2221 intel_pt_calc_tsc_timestamp(decoder);
2222 break;
2223
2224 case INTEL_PT_TMA:
2225 intel_pt_calc_tma(decoder);
2226 break;
2227
2228 case INTEL_PT_CYC:
2229 intel_pt_calc_cyc_timestamp(decoder);
2230 break;
2231
2232 case INTEL_PT_CBR:
2233 intel_pt_calc_cbr(decoder);
2234 break;
2235
2236 case INTEL_PT_PIP:
2237 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2238 break;
2239
2240 case INTEL_PT_MODE_EXEC:
2241 decoder->exec_mode = decoder->packet.payload;
2242 break;
2243
2244 case INTEL_PT_MODE_TSX:
2245 intel_pt_update_in_tx(decoder);
2246 break;
2247
2248 case INTEL_PT_OVF:
2249 return intel_pt_overflow(decoder);
2250
2251 case INTEL_PT_BAD: /* Does not happen */
2252 return intel_pt_bug(decoder);
2253
2254 case INTEL_PT_TRACESTOP:
2255 decoder->pge = false;
2256 decoder->continuous_period = false;
2257 intel_pt_clear_tx_flags(decoder);
2258 decoder->have_tma = false;
2259 break;
2260
2261 case INTEL_PT_PSB:
2262 decoder->last_ip = 0;
2263 decoder->have_last_ip = true;
2264 intel_pt_clear_stack(&decoder->stack);
2265 err = intel_pt_walk_psb(decoder);
2266 if (err)
2267 return err;
2268 if (decoder->ip) {
2269 /* Do not have a sample */
2270 decoder->state.type = 0;
2271 return 0;
2272 }
2273 break;
2274
2275 case INTEL_PT_TNT:
2276 case INTEL_PT_PSBEND:
2277 case INTEL_PT_VMCS:
2278 case INTEL_PT_MNT:
2279 case INTEL_PT_PAD:
2280 case INTEL_PT_PTWRITE:
2281 case INTEL_PT_PTWRITE_IP:
2282 case INTEL_PT_EXSTOP:
2283 case INTEL_PT_EXSTOP_IP:
2284 case INTEL_PT_MWAIT:
2285 case INTEL_PT_PWRE:
2286 case INTEL_PT_PWRX:
2287 default:
2288 break;
2289 }
2290 }
2291 }
2292
2293 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2294 {
2295 int err;
2296
2297 decoder->set_fup_tx_flags = false;
2298 decoder->set_fup_ptw = false;
2299 decoder->set_fup_mwait = false;
2300 decoder->set_fup_pwre = false;
2301 decoder->set_fup_exstop = false;
2302
2303 if (!decoder->branch_enable) {
2304 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2305 decoder->overflow = false;
2306 decoder->state.type = 0; /* Do not have a sample */
2307 return 0;
2308 }
2309
2310 intel_pt_log("Scanning for full IP\n");
2311 err = intel_pt_walk_to_ip(decoder);
2312 if (err)
2313 return err;
2314
2315 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2316 decoder->overflow = false;
2317
2318 decoder->state.from_ip = 0;
2319 decoder->state.to_ip = decoder->ip;
2320 intel_pt_log_to("Setting IP", decoder->ip);
2321
2322 return 0;
2323 }
2324
2325 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2326 {
2327 const unsigned char *end = decoder->buf + decoder->len;
2328 size_t i;
2329
2330 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2331 if (i > decoder->len)
2332 continue;
2333 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2334 return i;
2335 }
2336 return 0;
2337 }
2338
2339 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2340 {
2341 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2342 const char *psb = INTEL_PT_PSB_STR;
2343
2344 if (rest_psb > decoder->len ||
2345 memcmp(decoder->buf, psb + part_psb, rest_psb))
2346 return 0;
2347
2348 return rest_psb;
2349 }
2350
2351 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2352 int part_psb)
2353 {
2354 int rest_psb, ret;
2355
2356 decoder->pos += decoder->len;
2357 decoder->len = 0;
2358
2359 ret = intel_pt_get_next_data(decoder);
2360 if (ret)
2361 return ret;
2362
2363 rest_psb = intel_pt_rest_psb(decoder, part_psb);
2364 if (!rest_psb)
2365 return 0;
2366
2367 decoder->pos -= part_psb;
2368 decoder->next_buf = decoder->buf + rest_psb;
2369 decoder->next_len = decoder->len - rest_psb;
2370 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2371 decoder->buf = decoder->temp_buf;
2372 decoder->len = INTEL_PT_PSB_LEN;
2373
2374 return 0;
2375 }
2376
2377 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2378 {
2379 unsigned char *next;
2380 int ret;
2381
2382 intel_pt_log("Scanning for PSB\n");
2383 while (1) {
2384 if (!decoder->len) {
2385 ret = intel_pt_get_next_data(decoder);
2386 if (ret)
2387 return ret;
2388 }
2389
2390 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2391 INTEL_PT_PSB_LEN);
2392 if (!next) {
2393 int part_psb;
2394
2395 part_psb = intel_pt_part_psb(decoder);
2396 if (part_psb) {
2397 ret = intel_pt_get_split_psb(decoder, part_psb);
2398 if (ret)
2399 return ret;
2400 } else {
2401 decoder->pos += decoder->len;
2402 decoder->len = 0;
2403 }
2404 continue;
2405 }
2406
2407 decoder->pkt_step = next - decoder->buf;
2408 return intel_pt_get_next_packet(decoder);
2409 }
2410 }
2411
2412 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2413 {
2414 int err;
2415
2416 decoder->pge = false;
2417 decoder->continuous_period = false;
2418 decoder->have_last_ip = false;
2419 decoder->last_ip = 0;
2420 decoder->ip = 0;
2421 intel_pt_clear_stack(&decoder->stack);
2422
2423 err = intel_pt_scan_for_psb(decoder);
2424 if (err)
2425 return err;
2426
2427 decoder->have_last_ip = true;
2428 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2429
2430 err = intel_pt_walk_psb(decoder);
2431 if (err)
2432 return err;
2433
2434 if (decoder->ip) {
2435 decoder->state.type = 0; /* Do not have a sample */
2436 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2437 } else {
2438 return intel_pt_sync_ip(decoder);
2439 }
2440
2441 return 0;
2442 }
2443
2444 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2445 {
2446 uint64_t est = decoder->sample_insn_cnt << 1;
2447
2448 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2449 goto out;
2450
2451 est *= decoder->max_non_turbo_ratio;
2452 est /= decoder->cbr;
2453 out:
2454 return decoder->sample_timestamp + est;
2455 }
2456
2457 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2458 {
2459 int err;
2460
2461 do {
2462 decoder->state.type = INTEL_PT_BRANCH;
2463 decoder->state.flags = 0;
2464
2465 switch (decoder->pkt_state) {
2466 case INTEL_PT_STATE_NO_PSB:
2467 err = intel_pt_sync(decoder);
2468 break;
2469 case INTEL_PT_STATE_NO_IP:
2470 decoder->have_last_ip = false;
2471 decoder->last_ip = 0;
2472 decoder->ip = 0;
2473 __fallthrough;
2474 case INTEL_PT_STATE_ERR_RESYNC:
2475 err = intel_pt_sync_ip(decoder);
2476 break;
2477 case INTEL_PT_STATE_IN_SYNC:
2478 err = intel_pt_walk_trace(decoder);
2479 break;
2480 case INTEL_PT_STATE_TNT:
2481 case INTEL_PT_STATE_TNT_CONT:
2482 err = intel_pt_walk_tnt(decoder);
2483 if (err == -EAGAIN)
2484 err = intel_pt_walk_trace(decoder);
2485 break;
2486 case INTEL_PT_STATE_TIP:
2487 case INTEL_PT_STATE_TIP_PGD:
2488 err = intel_pt_walk_tip(decoder);
2489 break;
2490 case INTEL_PT_STATE_FUP:
2491 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2492 err = intel_pt_walk_fup(decoder);
2493 if (err == -EAGAIN)
2494 err = intel_pt_walk_fup_tip(decoder);
2495 else if (!err)
2496 decoder->pkt_state = INTEL_PT_STATE_FUP;
2497 break;
2498 case INTEL_PT_STATE_FUP_NO_TIP:
2499 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2500 err = intel_pt_walk_fup(decoder);
2501 if (err == -EAGAIN)
2502 err = intel_pt_walk_trace(decoder);
2503 break;
2504 default:
2505 err = intel_pt_bug(decoder);
2506 break;
2507 }
2508 } while (err == -ENOLINK);
2509
2510 if (err) {
2511 decoder->state.err = intel_pt_ext_err(err);
2512 decoder->state.from_ip = decoder->ip;
2513 intel_pt_update_sample_time(decoder);
2514 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2515 } else {
2516 decoder->state.err = 0;
2517 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2518 decoder->cbr_seen = decoder->cbr;
2519 decoder->state.type |= INTEL_PT_CBR_CHG;
2520 decoder->state.cbr_payload = decoder->cbr_payload;
2521 }
2522 if (intel_pt_sample_time(decoder->pkt_state)) {
2523 intel_pt_update_sample_time(decoder);
2524 if (decoder->sample_cyc)
2525 decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;
2526 }
2527 }
2528
2529 decoder->state.timestamp = decoder->sample_timestamp;
2530 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2531 decoder->state.cr3 = decoder->cr3;
2532 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2533 decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;
2534
2535 return &decoder->state;
2536 }
2537
2538 /**
2539 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2540 * @buf: pointer to buffer pointer
2541 * @len: size of buffer
2542 *
2543 * Updates the buffer pointer to point to the start of the next PSB packet if
2544 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2545 * @len is adjusted accordingly.
2546 *
2547 * Return: %true if a PSB packet is found, %false otherwise.
2548 */
2549 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2550 {
2551 unsigned char *next;
2552
2553 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2554 if (next) {
2555 *len -= next - *buf;
2556 *buf = next;
2557 return true;
2558 }
2559 return false;
2560 }
2561
2562 /**
2563 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2564 * packet.
2565 * @buf: pointer to buffer pointer
2566 * @len: size of buffer
2567 *
2568 * Updates the buffer pointer to point to the start of the following PSB packet
2569 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2570 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2571 *
2572 * Return: %true if a PSB packet is found, %false otherwise.
2573 */
2574 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2575 {
2576 unsigned char *next;
2577
2578 if (!*len)
2579 return false;
2580
2581 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2582 if (next) {
2583 *len -= next - *buf;
2584 *buf = next;
2585 return true;
2586 }
2587 return false;
2588 }
2589
2590 /**
2591 * intel_pt_last_psb - find the last PSB packet in a buffer.
2592 * @buf: buffer
2593 * @len: size of buffer
2594 *
2595 * This function finds the last PSB in a buffer.
2596 *
2597 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2598 */
2599 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2600 {
2601 const char *n = INTEL_PT_PSB_STR;
2602 unsigned char *p;
2603 size_t k;
2604
2605 if (len < INTEL_PT_PSB_LEN)
2606 return NULL;
2607
2608 k = len - INTEL_PT_PSB_LEN + 1;
2609 while (1) {
2610 p = memrchr(buf, n[0], k);
2611 if (!p)
2612 return NULL;
2613 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2614 return p;
2615 k = p - buf;
2616 if (!k)
2617 return NULL;
2618 }
2619 }
2620
2621 /**
2622 * intel_pt_next_tsc - find and return next TSC.
2623 * @buf: buffer
2624 * @len: size of buffer
2625 * @tsc: TSC value returned
2626 * @rem: returns remaining size when TSC is found
2627 *
2628 * Find a TSC packet in @buf and return the TSC value. This function assumes
2629 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2630 * PSBEND packet is found.
2631 *
2632 * Return: %true if TSC is found, false otherwise.
2633 */
2634 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2635 size_t *rem)
2636 {
2637 struct intel_pt_pkt packet;
2638 int ret;
2639
2640 while (len) {
2641 ret = intel_pt_get_packet(buf, len, &packet);
2642 if (ret <= 0)
2643 return false;
2644 if (packet.type == INTEL_PT_TSC) {
2645 *tsc = packet.payload;
2646 *rem = len;
2647 return true;
2648 }
2649 if (packet.type == INTEL_PT_PSBEND)
2650 return false;
2651 buf += ret;
2652 len -= ret;
2653 }
2654 return false;
2655 }
2656
2657 /**
2658 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2659 * @tsc1: first TSC to compare
2660 * @tsc2: second TSC to compare
2661 *
2662 * This function compares 7-byte TSC values allowing for the possibility that
2663 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2664 * around so for that purpose this function assumes the absolute difference is
2665 * less than half the maximum difference.
2666 *
2667 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2668 * after @tsc2.
2669 */
2670 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2671 {
2672 const uint64_t halfway = (1ULL << 55);
2673
2674 if (tsc1 == tsc2)
2675 return 0;
2676
2677 if (tsc1 < tsc2) {
2678 if (tsc2 - tsc1 < halfway)
2679 return -1;
2680 else
2681 return 1;
2682 } else {
2683 if (tsc1 - tsc2 < halfway)
2684 return 1;
2685 else
2686 return -1;
2687 }
2688 }
2689
2690 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2691
2692 /**
2693 * adj_for_padding - adjust overlap to account for padding.
2694 * @buf_b: second buffer
2695 * @buf_a: first buffer
2696 * @len_a: size of first buffer
2697 *
2698 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2699 * accordingly.
2700 *
2701 * Return: A pointer into @buf_b from where non-overlapped data starts
2702 */
2703 static unsigned char *adj_for_padding(unsigned char *buf_b,
2704 unsigned char *buf_a, size_t len_a)
2705 {
2706 unsigned char *p = buf_b - MAX_PADDING;
2707 unsigned char *q = buf_a + len_a - MAX_PADDING;
2708 int i;
2709
2710 for (i = MAX_PADDING; i; i--, p++, q++) {
2711 if (*p != *q)
2712 break;
2713 }
2714
2715 return p;
2716 }
2717
2718 /**
2719 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2720 * using TSC.
2721 * @buf_a: first buffer
2722 * @len_a: size of first buffer
2723 * @buf_b: second buffer
2724 * @len_b: size of second buffer
2725 * @consecutive: returns true if there is data in buf_b that is consecutive
2726 * to buf_a
2727 *
2728 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2729 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2730 * walk forward in @buf_b until a later TSC is found. A precondition is that
2731 * @buf_a and @buf_b are positioned at a PSB.
2732 *
2733 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2734 * @buf_b + @len_b if there is no non-overlapped data.
2735 */
2736 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2737 size_t len_a,
2738 unsigned char *buf_b,
2739 size_t len_b, bool *consecutive)
2740 {
2741 uint64_t tsc_a, tsc_b;
2742 unsigned char *p;
2743 size_t len, rem_a, rem_b;
2744
2745 p = intel_pt_last_psb(buf_a, len_a);
2746 if (!p)
2747 return buf_b; /* No PSB in buf_a => no overlap */
2748
2749 len = len_a - (p - buf_a);
2750 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2751 /* The last PSB+ in buf_a is incomplete, so go back one more */
2752 len_a -= len;
2753 p = intel_pt_last_psb(buf_a, len_a);
2754 if (!p)
2755 return buf_b; /* No full PSB+ => assume no overlap */
2756 len = len_a - (p - buf_a);
2757 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2758 return buf_b; /* No TSC in buf_a => assume no overlap */
2759 }
2760
2761 while (1) {
2762 /* Ignore PSB+ with no TSC */
2763 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2764 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2765
2766 /* Same TSC, so buffers are consecutive */
2767 if (!cmp && rem_b >= rem_a) {
2768 unsigned char *start;
2769
2770 *consecutive = true;
2771 start = buf_b + len_b - (rem_b - rem_a);
2772 return adj_for_padding(start, buf_a, len_a);
2773 }
2774 if (cmp < 0)
2775 return buf_b; /* tsc_a < tsc_b => no overlap */
2776 }
2777
2778 if (!intel_pt_step_psb(&buf_b, &len_b))
2779 return buf_b + len_b; /* No PSB in buf_b => no data */
2780 }
2781 }
2782
2783 /**
2784 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2785 * @buf_a: first buffer
2786 * @len_a: size of first buffer
2787 * @buf_b: second buffer
2788 * @len_b: size of second buffer
2789 * @have_tsc: can use TSC packets to detect overlap
2790 * @consecutive: returns true if there is data in buf_b that is consecutive
2791 * to buf_a
2792 *
2793 * When trace samples or snapshots are recorded there is the possibility that
2794 * the data overlaps. Note that, for the purposes of decoding, data is only
2795 * useful if it begins with a PSB packet.
2796 *
2797 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2798 * @buf_b + @len_b if there is no non-overlapped data.
2799 */
2800 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2801 unsigned char *buf_b, size_t len_b,
2802 bool have_tsc, bool *consecutive)
2803 {
2804 unsigned char *found;
2805
2806 /* Buffer 'b' must start at PSB so throw away everything before that */
2807 if (!intel_pt_next_psb(&buf_b, &len_b))
2808 return buf_b + len_b; /* No PSB */
2809
2810 if (!intel_pt_next_psb(&buf_a, &len_a))
2811 return buf_b; /* No overlap */
2812
2813 if (have_tsc) {
2814 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2815 consecutive);
2816 if (found)
2817 return found;
2818 }
2819
2820 /*
2821 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2822 * we can ignore the first part of buffer 'a'.
2823 */
2824 while (len_b < len_a) {
2825 if (!intel_pt_step_psb(&buf_a, &len_a))
2826 return buf_b; /* No overlap */
2827 }
2828
2829 /* Now len_b >= len_a */
2830 while (1) {
2831 /* Potential overlap so check the bytes */
2832 found = memmem(buf_a, len_a, buf_b, len_a);
2833 if (found) {
2834 *consecutive = true;
2835 return adj_for_padding(buf_b + len_a, buf_a, len_a);
2836 }
2837
2838 /* Try again at next PSB in buffer 'a' */
2839 if (!intel_pt_step_psb(&buf_a, &len_a))
2840 return buf_b; /* No overlap */
2841 }
2842 }
Linux with added FPC-III support