?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf intel-pt: Fix overlap calculation for padding
[linux/fpc-iii.git] / tools / perf / util / intel-pt-decoder / intel-pt-decoder.c
bloba54d6c9a460184ffd096e5c6cb367cea301c369c
1 /*
2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16 #ifndef _GNU_SOURCE
17 #define _GNU_SOURCE
18 #endif
19 #include <stdlib.h>
20 #include <stdbool.h>
21 #include <string.h>
22 #include <errno.h>
23 #include <stdint.h>
24 #include <inttypes.h>
25 #include <linux/compiler.h>
26
27 #include "../cache.h"
28 #include "../util.h"
29 #include "../auxtrace.h"
30
31 #include "intel-pt-insn-decoder.h"
32 #include "intel-pt-pkt-decoder.h"
33 #include "intel-pt-decoder.h"
34 #include "intel-pt-log.h"
35
36 #define INTEL_PT_BLK_SIZE 1024
37
38 #define BIT63 (((uint64_t)1 << 63))
39
40 #define INTEL_PT_RETURN 1
41
42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
43 #define INTEL_PT_MAX_LOOPS 10000
44
45 struct intel_pt_blk {
46 struct intel_pt_blk *prev;
47 uint64_t ip[INTEL_PT_BLK_SIZE];
48 };
49
50 struct intel_pt_stack {
51 struct intel_pt_blk *blk;
52 struct intel_pt_blk *spare;
53 int pos;
54 };
55
56 enum intel_pt_pkt_state {
57 INTEL_PT_STATE_NO_PSB,
58 INTEL_PT_STATE_NO_IP,
59 INTEL_PT_STATE_ERR_RESYNC,
60 INTEL_PT_STATE_IN_SYNC,
61 INTEL_PT_STATE_TNT,
62 INTEL_PT_STATE_TIP,
63 INTEL_PT_STATE_TIP_PGD,
64 INTEL_PT_STATE_FUP,
65 INTEL_PT_STATE_FUP_NO_TIP,
66 };
67
68 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
69 {
70 switch (pkt_state) {
71 case INTEL_PT_STATE_NO_PSB:
72 case INTEL_PT_STATE_NO_IP:
73 case INTEL_PT_STATE_ERR_RESYNC:
74 case INTEL_PT_STATE_IN_SYNC:
75 case INTEL_PT_STATE_TNT:
76 return true;
77 case INTEL_PT_STATE_TIP:
78 case INTEL_PT_STATE_TIP_PGD:
79 case INTEL_PT_STATE_FUP:
80 case INTEL_PT_STATE_FUP_NO_TIP:
81 return false;
82 default:
83 return true;
84 };
85 }
86
87 #ifdef INTEL_PT_STRICT
88 #define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
89 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
90 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
91 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
92 #else
93 #define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
94 #define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
95 #define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
96 #define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
97 #endif
98
99 struct intel_pt_decoder {
100 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
101 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
102 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
103 uint64_t max_insn_cnt, void *data);
104 bool (*pgd_ip)(uint64_t ip, void *data);
105 void *data;
106 struct intel_pt_state state;
107 const unsigned char *buf;
108 size_t len;
109 bool return_compression;
110 bool branch_enable;
111 bool mtc_insn;
112 bool pge;
113 bool have_tma;
114 bool have_cyc;
115 bool fixup_last_mtc;
116 bool have_last_ip;
117 enum intel_pt_param_flags flags;
118 uint64_t pos;
119 uint64_t last_ip;
120 uint64_t ip;
121 uint64_t cr3;
122 uint64_t timestamp;
123 uint64_t tsc_timestamp;
124 uint64_t ref_timestamp;
125 uint64_t sample_timestamp;
126 uint64_t ret_addr;
127 uint64_t ctc_timestamp;
128 uint64_t ctc_delta;
129 uint64_t cycle_cnt;
130 uint64_t cyc_ref_timestamp;
131 uint32_t last_mtc;
132 uint32_t tsc_ctc_ratio_n;
133 uint32_t tsc_ctc_ratio_d;
134 uint32_t tsc_ctc_mult;
135 uint32_t tsc_slip;
136 uint32_t ctc_rem_mask;
137 int mtc_shift;
138 struct intel_pt_stack stack;
139 enum intel_pt_pkt_state pkt_state;
140 struct intel_pt_pkt packet;
141 struct intel_pt_pkt tnt;
142 int pkt_step;
143 int pkt_len;
144 int last_packet_type;
145 unsigned int cbr;
146 unsigned int cbr_seen;
147 unsigned int max_non_turbo_ratio;
148 double max_non_turbo_ratio_fp;
149 double cbr_cyc_to_tsc;
150 double calc_cyc_to_tsc;
151 bool have_calc_cyc_to_tsc;
152 int exec_mode;
153 unsigned int insn_bytes;
154 uint64_t period;
155 enum intel_pt_period_type period_type;
156 uint64_t tot_insn_cnt;
157 uint64_t period_insn_cnt;
158 uint64_t period_mask;
159 uint64_t period_ticks;
160 uint64_t last_masked_timestamp;
161 bool continuous_period;
162 bool overflow;
163 bool set_fup_tx_flags;
164 bool set_fup_ptw;
165 bool set_fup_mwait;
166 bool set_fup_pwre;
167 bool set_fup_exstop;
168 unsigned int fup_tx_flags;
169 unsigned int tx_flags;
170 uint64_t fup_ptw_payload;
171 uint64_t fup_mwait_payload;
172 uint64_t fup_pwre_payload;
173 uint64_t cbr_payload;
174 uint64_t timestamp_insn_cnt;
175 uint64_t sample_insn_cnt;
176 uint64_t stuck_ip;
177 int no_progress;
178 int stuck_ip_prd;
179 int stuck_ip_cnt;
180 const unsigned char *next_buf;
181 size_t next_len;
182 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
183 };
184
185 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
186 {
187 int i;
188
189 for (i = 0; x != 1; i++)
190 x >>= 1;
191
192 return x << i;
193 }
194
195 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
196 {
197 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
198 uint64_t period;
199
200 period = intel_pt_lower_power_of_2(decoder->period);
201 decoder->period_mask = ~(period - 1);
202 decoder->period_ticks = period;
203 }
204 }
205
206 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
207 {
208 if (!d)
209 return 0;
210 return (t / d) * n + ((t % d) * n) / d;
211 }
212
213 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
214 {
215 struct intel_pt_decoder *decoder;
216
217 if (!params->get_trace || !params->walk_insn)
218 return NULL;
219
220 decoder = zalloc(sizeof(struct intel_pt_decoder));
221 if (!decoder)
222 return NULL;
223
224 decoder->get_trace = params->get_trace;
225 decoder->walk_insn = params->walk_insn;
226 decoder->pgd_ip = params->pgd_ip;
227 decoder->data = params->data;
228 decoder->return_compression = params->return_compression;
229 decoder->branch_enable = params->branch_enable;
230
231 decoder->flags = params->flags;
232
233 decoder->period = params->period;
234 decoder->period_type = params->period_type;
235
236 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
237 decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
238
239 intel_pt_setup_period(decoder);
240
241 decoder->mtc_shift = params->mtc_period;
242 decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
243
244 decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
245 decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
246
247 if (!decoder->tsc_ctc_ratio_n)
248 decoder->tsc_ctc_ratio_d = 0;
249
250 if (decoder->tsc_ctc_ratio_d) {
251 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
252 decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
253 decoder->tsc_ctc_ratio_d;
254
255 /*
256 * Allow for timestamps appearing to backwards because a TSC
257 * packet has slipped past a MTC packet, so allow 2 MTC ticks
258 * or ...
259 */
260 decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,
261 decoder->tsc_ctc_ratio_n,
262 decoder->tsc_ctc_ratio_d);
263 }
264 /* ... or 0x100 paranoia */
265 if (decoder->tsc_slip < 0x100)
266 decoder->tsc_slip = 0x100;
267
268 intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
269 intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
270 intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
271 intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
272 intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
273
274 return decoder;
275 }
276
277 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
278 {
279 struct intel_pt_blk *blk = stack->blk;
280
281 stack->blk = blk->prev;
282 if (!stack->spare)
283 stack->spare = blk;
284 else
285 free(blk);
286 }
287
288 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
289 {
290 if (!stack->pos) {
291 if (!stack->blk)
292 return 0;
293 intel_pt_pop_blk(stack);
294 if (!stack->blk)
295 return 0;
296 stack->pos = INTEL_PT_BLK_SIZE;
297 }
298 return stack->blk->ip[--stack->pos];
299 }
300
301 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
302 {
303 struct intel_pt_blk *blk;
304
305 if (stack->spare) {
306 blk = stack->spare;
307 stack->spare = NULL;
308 } else {
309 blk = malloc(sizeof(struct intel_pt_blk));
310 if (!blk)
311 return -ENOMEM;
312 }
313
314 blk->prev = stack->blk;
315 stack->blk = blk;
316 stack->pos = 0;
317 return 0;
318 }
319
320 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
321 {
322 int err;
323
324 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
325 err = intel_pt_alloc_blk(stack);
326 if (err)
327 return err;
328 }
329
330 stack->blk->ip[stack->pos++] = ip;
331 return 0;
332 }
333
334 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
335 {
336 while (stack->blk)
337 intel_pt_pop_blk(stack);
338 stack->pos = 0;
339 }
340
341 static void intel_pt_free_stack(struct intel_pt_stack *stack)
342 {
343 intel_pt_clear_stack(stack);
344 zfree(&stack->blk);
345 zfree(&stack->spare);
346 }
347
348 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
349 {
350 intel_pt_free_stack(&decoder->stack);
351 free(decoder);
352 }
353
354 static int intel_pt_ext_err(int code)
355 {
356 switch (code) {
357 case -ENOMEM:
358 return INTEL_PT_ERR_NOMEM;
359 case -ENOSYS:
360 return INTEL_PT_ERR_INTERN;
361 case -EBADMSG:
362 return INTEL_PT_ERR_BADPKT;
363 case -ENODATA:
364 return INTEL_PT_ERR_NODATA;
365 case -EILSEQ:
366 return INTEL_PT_ERR_NOINSN;
367 case -ENOENT:
368 return INTEL_PT_ERR_MISMAT;
369 case -EOVERFLOW:
370 return INTEL_PT_ERR_OVR;
371 case -ENOSPC:
372 return INTEL_PT_ERR_LOST;
373 case -ELOOP:
374 return INTEL_PT_ERR_NELOOP;
375 default:
376 return INTEL_PT_ERR_UNK;
377 }
378 }
379
380 static const char *intel_pt_err_msgs[] = {
381 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
382 [INTEL_PT_ERR_INTERN] = "Internal error",
383 [INTEL_PT_ERR_BADPKT] = "Bad packet",
384 [INTEL_PT_ERR_NODATA] = "No more data",
385 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
386 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
387 [INTEL_PT_ERR_OVR] = "Overflow packet",
388 [INTEL_PT_ERR_LOST] = "Lost trace data",
389 [INTEL_PT_ERR_UNK] = "Unknown error!",
390 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
391 };
392
393 int intel_pt__strerror(int code, char *buf, size_t buflen)
394 {
395 if (code < 1 || code >= INTEL_PT_ERR_MAX)
396 code = INTEL_PT_ERR_UNK;
397 strlcpy(buf, intel_pt_err_msgs[code], buflen);
398 return 0;
399 }
400
401 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
402 uint64_t last_ip)
403 {
404 uint64_t ip;
405
406 switch (packet->count) {
407 case 1:
408 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
409 packet->payload;
410 break;
411 case 2:
412 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
413 packet->payload;
414 break;
415 case 3:
416 ip = packet->payload;
417 /* Sign-extend 6-byte ip */
418 if (ip & (uint64_t)0x800000000000ULL)
419 ip |= (uint64_t)0xffff000000000000ULL;
420 break;
421 case 4:
422 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
423 packet->payload;
424 break;
425 case 6:
426 ip = packet->payload;
427 break;
428 default:
429 return 0;
430 }
431
432 return ip;
433 }
434
435 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
436 {
437 decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
438 decoder->have_last_ip = true;
439 }
440
441 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
442 {
443 intel_pt_set_last_ip(decoder);
444 decoder->ip = decoder->last_ip;
445 }
446
447 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
448 {
449 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
450 decoder->buf);
451 }
452
453 static int intel_pt_bug(struct intel_pt_decoder *decoder)
454 {
455 intel_pt_log("ERROR: Internal error\n");
456 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
457 return -ENOSYS;
458 }
459
460 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
461 {
462 decoder->tx_flags = 0;
463 }
464
465 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
466 {
467 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
468 }
469
470 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
471 {
472 intel_pt_clear_tx_flags(decoder);
473 decoder->have_tma = false;
474 decoder->pkt_len = 1;
475 decoder->pkt_step = 1;
476 intel_pt_decoder_log_packet(decoder);
477 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
478 intel_pt_log("ERROR: Bad packet\n");
479 decoder->pkt_state = INTEL_PT_STATE_ERR1;
480 }
481 return -EBADMSG;
482 }
483
484 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
485 {
486 struct intel_pt_buffer buffer = { .buf = 0, };
487 int ret;
488
489 decoder->pkt_step = 0;
490
491 intel_pt_log("Getting more data\n");
492 ret = decoder->get_trace(&buffer, decoder->data);
493 if (ret)
494 return ret;
495 decoder->buf = buffer.buf;
496 decoder->len = buffer.len;
497 if (!decoder->len) {
498 intel_pt_log("No more data\n");
499 return -ENODATA;
500 }
501 if (!buffer.consecutive) {
502 decoder->ip = 0;
503 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
504 decoder->ref_timestamp = buffer.ref_timestamp;
505 decoder->timestamp = 0;
506 decoder->have_tma = false;
507 decoder->state.trace_nr = buffer.trace_nr;
508 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
509 decoder->ref_timestamp);
510 return -ENOLINK;
511 }
512
513 return 0;
514 }
515
516 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
517 {
518 if (!decoder->next_buf)
519 return intel_pt_get_data(decoder);
520
521 decoder->buf = decoder->next_buf;
522 decoder->len = decoder->next_len;
523 decoder->next_buf = 0;
524 decoder->next_len = 0;
525 return 0;
526 }
527
528 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
529 {
530 unsigned char *buf = decoder->temp_buf;
531 size_t old_len, len, n;
532 int ret;
533
534 old_len = decoder->len;
535 len = decoder->len;
536 memcpy(buf, decoder->buf, len);
537
538 ret = intel_pt_get_data(decoder);
539 if (ret) {
540 decoder->pos += old_len;
541 return ret < 0 ? ret : -EINVAL;
542 }
543
544 n = INTEL_PT_PKT_MAX_SZ - len;
545 if (n > decoder->len)
546 n = decoder->len;
547 memcpy(buf + len, decoder->buf, n);
548 len += n;
549
550 ret = intel_pt_get_packet(buf, len, &decoder->packet);
551 if (ret < (int)old_len) {
552 decoder->next_buf = decoder->buf;
553 decoder->next_len = decoder->len;
554 decoder->buf = buf;
555 decoder->len = old_len;
556 return intel_pt_bad_packet(decoder);
557 }
558
559 decoder->next_buf = decoder->buf + (ret - old_len);
560 decoder->next_len = decoder->len - (ret - old_len);
561
562 decoder->buf = buf;
563 decoder->len = ret;
564
565 return ret;
566 }
567
568 struct intel_pt_pkt_info {
569 struct intel_pt_decoder *decoder;
570 struct intel_pt_pkt packet;
571 uint64_t pos;
572 int pkt_len;
573 int last_packet_type;
574 void *data;
575 };
576
577 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
578
579 /* Lookahead packets in current buffer */
580 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
581 intel_pt_pkt_cb_t cb, void *data)
582 {
583 struct intel_pt_pkt_info pkt_info;
584 const unsigned char *buf = decoder->buf;
585 size_t len = decoder->len;
586 int ret;
587
588 pkt_info.decoder = decoder;
589 pkt_info.pos = decoder->pos;
590 pkt_info.pkt_len = decoder->pkt_step;
591 pkt_info.last_packet_type = decoder->last_packet_type;
592 pkt_info.data = data;
593
594 while (1) {
595 do {
596 pkt_info.pos += pkt_info.pkt_len;
597 buf += pkt_info.pkt_len;
598 len -= pkt_info.pkt_len;
599
600 if (!len)
601 return INTEL_PT_NEED_MORE_BYTES;
602
603 ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
604 if (!ret)
605 return INTEL_PT_NEED_MORE_BYTES;
606 if (ret < 0)
607 return ret;
608
609 pkt_info.pkt_len = ret;
610 } while (pkt_info.packet.type == INTEL_PT_PAD);
611
612 ret = cb(&pkt_info);
613 if (ret)
614 return 0;
615
616 pkt_info.last_packet_type = pkt_info.packet.type;
617 }
618 }
619
620 struct intel_pt_calc_cyc_to_tsc_info {
621 uint64_t cycle_cnt;
622 unsigned int cbr;
623 uint32_t last_mtc;
624 uint64_t ctc_timestamp;
625 uint64_t ctc_delta;
626 uint64_t tsc_timestamp;
627 uint64_t timestamp;
628 bool have_tma;
629 bool fixup_last_mtc;
630 bool from_mtc;
631 double cbr_cyc_to_tsc;
632 };
633
634 /*
635 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
636 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
637 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
638 * packet by copying the missing bits from the current MTC assuming the least
639 * difference between the two, and that the current MTC comes after last_mtc.
640 */
641 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
642 uint32_t *last_mtc)
643 {
644 uint32_t first_missing_bit = 1U << (16 - mtc_shift);
645 uint32_t mask = ~(first_missing_bit - 1);
646
647 *last_mtc |= mtc & mask;
648 if (*last_mtc >= mtc) {
649 *last_mtc -= first_missing_bit;
650 *last_mtc &= 0xff;
651 }
652 }
653
654 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
655 {
656 struct intel_pt_decoder *decoder = pkt_info->decoder;
657 struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
658 uint64_t timestamp;
659 double cyc_to_tsc;
660 unsigned int cbr;
661 uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
662
663 switch (pkt_info->packet.type) {
664 case INTEL_PT_TNT:
665 case INTEL_PT_TIP_PGE:
666 case INTEL_PT_TIP:
667 case INTEL_PT_FUP:
668 case INTEL_PT_PSB:
669 case INTEL_PT_PIP:
670 case INTEL_PT_MODE_EXEC:
671 case INTEL_PT_MODE_TSX:
672 case INTEL_PT_PSBEND:
673 case INTEL_PT_PAD:
674 case INTEL_PT_VMCS:
675 case INTEL_PT_MNT:
676 case INTEL_PT_PTWRITE:
677 case INTEL_PT_PTWRITE_IP:
678 return 0;
679
680 case INTEL_PT_MTC:
681 if (!data->have_tma)
682 return 0;
683
684 mtc = pkt_info->packet.payload;
685 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
686 data->fixup_last_mtc = false;
687 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
688 &data->last_mtc);
689 }
690 if (mtc > data->last_mtc)
691 mtc_delta = mtc - data->last_mtc;
692 else
693 mtc_delta = mtc + 256 - data->last_mtc;
694 data->ctc_delta += mtc_delta << decoder->mtc_shift;
695 data->last_mtc = mtc;
696
697 if (decoder->tsc_ctc_mult) {
698 timestamp = data->ctc_timestamp +
699 data->ctc_delta * decoder->tsc_ctc_mult;
700 } else {
701 timestamp = data->ctc_timestamp +
702 multdiv(data->ctc_delta,
703 decoder->tsc_ctc_ratio_n,
704 decoder->tsc_ctc_ratio_d);
705 }
706
707 if (timestamp < data->timestamp)
708 return 1;
709
710 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
711 data->timestamp = timestamp;
712 return 0;
713 }
714
715 break;
716
717 case INTEL_PT_TSC:
718 /*
719 * For now, do not support using TSC packets - refer
720 * intel_pt_calc_cyc_to_tsc().
721 */
722 if (data->from_mtc)
723 return 1;
724 timestamp = pkt_info->packet.payload |
725 (data->timestamp & (0xffULL << 56));
726 if (data->from_mtc && timestamp < data->timestamp &&
727 data->timestamp - timestamp < decoder->tsc_slip)
728 return 1;
729 if (timestamp < data->timestamp)
730 timestamp += (1ULL << 56);
731 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
732 if (data->from_mtc)
733 return 1;
734 data->tsc_timestamp = timestamp;
735 data->timestamp = timestamp;
736 return 0;
737 }
738 break;
739
740 case INTEL_PT_TMA:
741 if (data->from_mtc)
742 return 1;
743
744 if (!decoder->tsc_ctc_ratio_d)
745 return 0;
746
747 ctc = pkt_info->packet.payload;
748 fc = pkt_info->packet.count;
749 ctc_rem = ctc & decoder->ctc_rem_mask;
750
751 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
752
753 data->ctc_timestamp = data->tsc_timestamp - fc;
754 if (decoder->tsc_ctc_mult) {
755 data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
756 } else {
757 data->ctc_timestamp -=
758 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
759 decoder->tsc_ctc_ratio_d);
760 }
761
762 data->ctc_delta = 0;
763 data->have_tma = true;
764 data->fixup_last_mtc = true;
765
766 return 0;
767
768 case INTEL_PT_CYC:
769 data->cycle_cnt += pkt_info->packet.payload;
770 return 0;
771
772 case INTEL_PT_CBR:
773 cbr = pkt_info->packet.payload;
774 if (data->cbr && data->cbr != cbr)
775 return 1;
776 data->cbr = cbr;
777 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
778 return 0;
779
780 case INTEL_PT_TIP_PGD:
781 case INTEL_PT_TRACESTOP:
782 case INTEL_PT_EXSTOP:
783 case INTEL_PT_EXSTOP_IP:
784 case INTEL_PT_MWAIT:
785 case INTEL_PT_PWRE:
786 case INTEL_PT_PWRX:
787 case INTEL_PT_OVF:
788 case INTEL_PT_BAD: /* Does not happen */
789 default:
790 return 1;
791 }
792
793 if (!data->cbr && decoder->cbr) {
794 data->cbr = decoder->cbr;
795 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
796 }
797
798 if (!data->cycle_cnt)
799 return 1;
800
801 cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
802
803 if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
804 cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
805 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
806 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
807 return 1;
808 }
809
810 decoder->calc_cyc_to_tsc = cyc_to_tsc;
811 decoder->have_calc_cyc_to_tsc = true;
812
813 if (data->cbr) {
814 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
815 cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
816 } else {
817 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
818 cyc_to_tsc, pkt_info->pos);
819 }
820
821 return 1;
822 }
823
824 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
825 bool from_mtc)
826 {
827 struct intel_pt_calc_cyc_to_tsc_info data = {
828 .cycle_cnt = 0,
829 .cbr = 0,
830 .last_mtc = decoder->last_mtc,
831 .ctc_timestamp = decoder->ctc_timestamp,
832 .ctc_delta = decoder->ctc_delta,
833 .tsc_timestamp = decoder->tsc_timestamp,
834 .timestamp = decoder->timestamp,
835 .have_tma = decoder->have_tma,
836 .fixup_last_mtc = decoder->fixup_last_mtc,
837 .from_mtc = from_mtc,
838 .cbr_cyc_to_tsc = 0,
839 };
840
841 /*
842 * For now, do not support using TSC packets for at least the reasons:
843 * 1) timing might have stopped
844 * 2) TSC packets within PSB+ can slip against CYC packets
845 */
846 if (!from_mtc)
847 return;
848
849 intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
850 }
851
852 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
853 {
854 int ret;
855
856 decoder->last_packet_type = decoder->packet.type;
857
858 do {
859 decoder->pos += decoder->pkt_step;
860 decoder->buf += decoder->pkt_step;
861 decoder->len -= decoder->pkt_step;
862
863 if (!decoder->len) {
864 ret = intel_pt_get_next_data(decoder);
865 if (ret)
866 return ret;
867 }
868
869 ret = intel_pt_get_packet(decoder->buf, decoder->len,
870 &decoder->packet);
871 if (ret == INTEL_PT_NEED_MORE_BYTES &&
872 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
873 ret = intel_pt_get_split_packet(decoder);
874 if (ret < 0)
875 return ret;
876 }
877 if (ret <= 0)
878 return intel_pt_bad_packet(decoder);
879
880 decoder->pkt_len = ret;
881 decoder->pkt_step = ret;
882 intel_pt_decoder_log_packet(decoder);
883 } while (decoder->packet.type == INTEL_PT_PAD);
884
885 return 0;
886 }
887
888 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
889 {
890 uint64_t timestamp, masked_timestamp;
891
892 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
893 masked_timestamp = timestamp & decoder->period_mask;
894 if (decoder->continuous_period) {
895 if (masked_timestamp != decoder->last_masked_timestamp)
896 return 1;
897 } else {
898 timestamp += 1;
899 masked_timestamp = timestamp & decoder->period_mask;
900 if (masked_timestamp != decoder->last_masked_timestamp) {
901 decoder->last_masked_timestamp = masked_timestamp;
902 decoder->continuous_period = true;
903 }
904 }
905 return decoder->period_ticks - (timestamp - masked_timestamp);
906 }
907
908 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
909 {
910 switch (decoder->period_type) {
911 case INTEL_PT_PERIOD_INSTRUCTIONS:
912 return decoder->period - decoder->period_insn_cnt;
913 case INTEL_PT_PERIOD_TICKS:
914 return intel_pt_next_period(decoder);
915 case INTEL_PT_PERIOD_NONE:
916 case INTEL_PT_PERIOD_MTC:
917 default:
918 return 0;
919 }
920 }
921
922 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
923 {
924 uint64_t timestamp, masked_timestamp;
925
926 switch (decoder->period_type) {
927 case INTEL_PT_PERIOD_INSTRUCTIONS:
928 decoder->period_insn_cnt = 0;
929 break;
930 case INTEL_PT_PERIOD_TICKS:
931 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
932 masked_timestamp = timestamp & decoder->period_mask;
933 decoder->last_masked_timestamp = masked_timestamp;
934 break;
935 case INTEL_PT_PERIOD_NONE:
936 case INTEL_PT_PERIOD_MTC:
937 default:
938 break;
939 }
940
941 decoder->state.type |= INTEL_PT_INSTRUCTION;
942 }
943
944 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
945 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
946 {
947 uint64_t max_insn_cnt, insn_cnt = 0;
948 int err;
949
950 if (!decoder->mtc_insn)
951 decoder->mtc_insn = true;
952
953 max_insn_cnt = intel_pt_next_sample(decoder);
954
955 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
956 max_insn_cnt, decoder->data);
957
958 decoder->tot_insn_cnt += insn_cnt;
959 decoder->timestamp_insn_cnt += insn_cnt;
960 decoder->sample_insn_cnt += insn_cnt;
961 decoder->period_insn_cnt += insn_cnt;
962
963 if (err) {
964 decoder->no_progress = 0;
965 decoder->pkt_state = INTEL_PT_STATE_ERR2;
966 intel_pt_log_at("ERROR: Failed to get instruction",
967 decoder->ip);
968 if (err == -ENOENT)
969 return -ENOLINK;
970 return -EILSEQ;
971 }
972
973 if (ip && decoder->ip == ip) {
974 err = -EAGAIN;
975 goto out;
976 }
977
978 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
979 intel_pt_sample_insn(decoder);
980
981 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
982 decoder->state.type = INTEL_PT_INSTRUCTION;
983 decoder->state.from_ip = decoder->ip;
984 decoder->state.to_ip = 0;
985 decoder->ip += intel_pt_insn->length;
986 err = INTEL_PT_RETURN;
987 goto out;
988 }
989
990 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
991 /* Zero-length calls are excluded */
992 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
993 intel_pt_insn->rel) {
994 err = intel_pt_push(&decoder->stack, decoder->ip +
995 intel_pt_insn->length);
996 if (err)
997 goto out;
998 }
999 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1000 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1001 }
1002
1003 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1004 int cnt = decoder->no_progress++;
1005
1006 decoder->state.from_ip = decoder->ip;
1007 decoder->ip += intel_pt_insn->length +
1008 intel_pt_insn->rel;
1009 decoder->state.to_ip = decoder->ip;
1010 err = INTEL_PT_RETURN;
1011
1012 /*
1013 * Check for being stuck in a loop. This can happen if a
1014 * decoder error results in the decoder erroneously setting the
1015 * ip to an address that is itself in an infinite loop that
1016 * consumes no packets. When that happens, there must be an
1017 * unconditional branch.
1018 */
1019 if (cnt) {
1020 if (cnt == 1) {
1021 decoder->stuck_ip = decoder->state.to_ip;
1022 decoder->stuck_ip_prd = 1;
1023 decoder->stuck_ip_cnt = 1;
1024 } else if (cnt > INTEL_PT_MAX_LOOPS ||
1025 decoder->state.to_ip == decoder->stuck_ip) {
1026 intel_pt_log_at("ERROR: Never-ending loop",
1027 decoder->state.to_ip);
1028 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1029 err = -ELOOP;
1030 goto out;
1031 } else if (!--decoder->stuck_ip_cnt) {
1032 decoder->stuck_ip_prd += 1;
1033 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1034 decoder->stuck_ip = decoder->state.to_ip;
1035 }
1036 }
1037 goto out_no_progress;
1038 }
1039 out:
1040 decoder->no_progress = 0;
1041 out_no_progress:
1042 decoder->state.insn_op = intel_pt_insn->op;
1043 decoder->state.insn_len = intel_pt_insn->length;
1044 memcpy(decoder->state.insn, intel_pt_insn->buf,
1045 INTEL_PT_INSN_BUF_SZ);
1046
1047 if (decoder->tx_flags & INTEL_PT_IN_TX)
1048 decoder->state.flags |= INTEL_PT_IN_TX;
1049
1050 return err;
1051 }
1052
1053 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1054 {
1055 bool ret = false;
1056
1057 if (decoder->set_fup_tx_flags) {
1058 decoder->set_fup_tx_flags = false;
1059 decoder->tx_flags = decoder->fup_tx_flags;
1060 decoder->state.type = INTEL_PT_TRANSACTION;
1061 decoder->state.from_ip = decoder->ip;
1062 decoder->state.to_ip = 0;
1063 decoder->state.flags = decoder->fup_tx_flags;
1064 return true;
1065 }
1066 if (decoder->set_fup_ptw) {
1067 decoder->set_fup_ptw = false;
1068 decoder->state.type = INTEL_PT_PTW;
1069 decoder->state.flags |= INTEL_PT_FUP_IP;
1070 decoder->state.from_ip = decoder->ip;
1071 decoder->state.to_ip = 0;
1072 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1073 return true;
1074 }
1075 if (decoder->set_fup_mwait) {
1076 decoder->set_fup_mwait = false;
1077 decoder->state.type = INTEL_PT_MWAIT_OP;
1078 decoder->state.from_ip = decoder->ip;
1079 decoder->state.to_ip = 0;
1080 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1081 ret = true;
1082 }
1083 if (decoder->set_fup_pwre) {
1084 decoder->set_fup_pwre = false;
1085 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1086 decoder->state.type &= ~INTEL_PT_BRANCH;
1087 decoder->state.from_ip = decoder->ip;
1088 decoder->state.to_ip = 0;
1089 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1090 ret = true;
1091 }
1092 if (decoder->set_fup_exstop) {
1093 decoder->set_fup_exstop = false;
1094 decoder->state.type |= INTEL_PT_EX_STOP;
1095 decoder->state.type &= ~INTEL_PT_BRANCH;
1096 decoder->state.flags |= INTEL_PT_FUP_IP;
1097 decoder->state.from_ip = decoder->ip;
1098 decoder->state.to_ip = 0;
1099 ret = true;
1100 }
1101 return ret;
1102 }
1103
1104 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1105 struct intel_pt_insn *intel_pt_insn,
1106 uint64_t ip, int err)
1107 {
1108 return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1109 intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1110 ip == decoder->ip + intel_pt_insn->length;
1111 }
1112
1113 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1114 {
1115 struct intel_pt_insn intel_pt_insn;
1116 uint64_t ip;
1117 int err;
1118
1119 ip = decoder->last_ip;
1120
1121 while (1) {
1122 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1123 if (err == INTEL_PT_RETURN)
1124 return 0;
1125 if (err == -EAGAIN ||
1126 intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1127 if (intel_pt_fup_event(decoder))
1128 return 0;
1129 return -EAGAIN;
1130 }
1131 decoder->set_fup_tx_flags = false;
1132 if (err)
1133 return err;
1134
1135 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1136 intel_pt_log_at("ERROR: Unexpected indirect branch",
1137 decoder->ip);
1138 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1139 return -ENOENT;
1140 }
1141
1142 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1143 intel_pt_log_at("ERROR: Unexpected conditional branch",
1144 decoder->ip);
1145 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1146 return -ENOENT;
1147 }
1148
1149 intel_pt_bug(decoder);
1150 }
1151 }
1152
1153 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1154 {
1155 struct intel_pt_insn intel_pt_insn;
1156 int err;
1157
1158 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1159 if (err == INTEL_PT_RETURN &&
1160 decoder->pgd_ip &&
1161 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1162 (decoder->state.type & INTEL_PT_BRANCH) &&
1163 decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1164 /* Unconditional branch leaving filter region */
1165 decoder->no_progress = 0;
1166 decoder->pge = false;
1167 decoder->continuous_period = false;
1168 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1169 decoder->state.type |= INTEL_PT_TRACE_END;
1170 return 0;
1171 }
1172 if (err == INTEL_PT_RETURN)
1173 return 0;
1174 if (err)
1175 return err;
1176
1177 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1178 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1179 decoder->pge = false;
1180 decoder->continuous_period = false;
1181 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1182 decoder->state.from_ip = decoder->ip;
1183 if (decoder->packet.count == 0) {
1184 decoder->state.to_ip = 0;
1185 } else {
1186 decoder->state.to_ip = decoder->last_ip;
1187 decoder->ip = decoder->last_ip;
1188 }
1189 decoder->state.type |= INTEL_PT_TRACE_END;
1190 } else {
1191 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1192 decoder->state.from_ip = decoder->ip;
1193 if (decoder->packet.count == 0) {
1194 decoder->state.to_ip = 0;
1195 } else {
1196 decoder->state.to_ip = decoder->last_ip;
1197 decoder->ip = decoder->last_ip;
1198 }
1199 }
1200 return 0;
1201 }
1202
1203 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1204 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1205 intel_pt_insn.rel;
1206
1207 if (decoder->pgd_ip &&
1208 decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1209 decoder->pgd_ip(to_ip, decoder->data)) {
1210 /* Conditional branch leaving filter region */
1211 decoder->pge = false;
1212 decoder->continuous_period = false;
1213 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1214 decoder->ip = to_ip;
1215 decoder->state.from_ip = decoder->ip;
1216 decoder->state.to_ip = to_ip;
1217 decoder->state.type |= INTEL_PT_TRACE_END;
1218 return 0;
1219 }
1220 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1221 decoder->ip);
1222 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1223 return -ENOENT;
1224 }
1225
1226 return intel_pt_bug(decoder);
1227 }
1228
1229 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1230 {
1231 struct intel_pt_insn intel_pt_insn;
1232 int err;
1233
1234 while (1) {
1235 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1236 if (err == INTEL_PT_RETURN)
1237 return 0;
1238 if (err)
1239 return err;
1240
1241 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1242 if (!decoder->return_compression) {
1243 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1244 decoder->ip);
1245 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1246 return -ENOENT;
1247 }
1248 if (!decoder->ret_addr) {
1249 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1250 decoder->ip);
1251 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1252 return -ENOENT;
1253 }
1254 if (!(decoder->tnt.payload & BIT63)) {
1255 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1256 decoder->ip);
1257 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1258 return -ENOENT;
1259 }
1260 decoder->tnt.count -= 1;
1261 if (!decoder->tnt.count)
1262 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1263 decoder->tnt.payload <<= 1;
1264 decoder->state.from_ip = decoder->ip;
1265 decoder->ip = decoder->ret_addr;
1266 decoder->state.to_ip = decoder->ip;
1267 return 0;
1268 }
1269
1270 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1271 /* Handle deferred TIPs */
1272 err = intel_pt_get_next_packet(decoder);
1273 if (err)
1274 return err;
1275 if (decoder->packet.type != INTEL_PT_TIP ||
1276 decoder->packet.count == 0) {
1277 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1278 decoder->ip);
1279 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1280 decoder->pkt_step = 0;
1281 return -ENOENT;
1282 }
1283 intel_pt_set_last_ip(decoder);
1284 decoder->state.from_ip = decoder->ip;
1285 decoder->state.to_ip = decoder->last_ip;
1286 decoder->ip = decoder->last_ip;
1287 return 0;
1288 }
1289
1290 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1291 decoder->tnt.count -= 1;
1292 if (!decoder->tnt.count)
1293 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1294 if (decoder->tnt.payload & BIT63) {
1295 decoder->tnt.payload <<= 1;
1296 decoder->state.from_ip = decoder->ip;
1297 decoder->ip += intel_pt_insn.length +
1298 intel_pt_insn.rel;
1299 decoder->state.to_ip = decoder->ip;
1300 return 0;
1301 }
1302 /* Instruction sample for a non-taken branch */
1303 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1304 decoder->tnt.payload <<= 1;
1305 decoder->state.type = INTEL_PT_INSTRUCTION;
1306 decoder->state.from_ip = decoder->ip;
1307 decoder->state.to_ip = 0;
1308 decoder->ip += intel_pt_insn.length;
1309 return 0;
1310 }
1311 decoder->ip += intel_pt_insn.length;
1312 if (!decoder->tnt.count)
1313 return -EAGAIN;
1314 decoder->tnt.payload <<= 1;
1315 continue;
1316 }
1317
1318 return intel_pt_bug(decoder);
1319 }
1320 }
1321
1322 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1323 {
1324 unsigned int fup_tx_flags;
1325 int err;
1326
1327 fup_tx_flags = decoder->packet.payload &
1328 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1329 err = intel_pt_get_next_packet(decoder);
1330 if (err)
1331 return err;
1332 if (decoder->packet.type == INTEL_PT_FUP) {
1333 decoder->fup_tx_flags = fup_tx_flags;
1334 decoder->set_fup_tx_flags = true;
1335 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1336 *no_tip = true;
1337 } else {
1338 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1339 decoder->pos);
1340 intel_pt_update_in_tx(decoder);
1341 }
1342 return 0;
1343 }
1344
1345 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1346 {
1347 uint64_t timestamp;
1348
1349 decoder->have_tma = false;
1350
1351 if (decoder->ref_timestamp) {
1352 timestamp = decoder->packet.payload |
1353 (decoder->ref_timestamp & (0xffULL << 56));
1354 if (timestamp < decoder->ref_timestamp) {
1355 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1356 timestamp += (1ULL << 56);
1357 } else {
1358 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1359 timestamp -= (1ULL << 56);
1360 }
1361 decoder->tsc_timestamp = timestamp;
1362 decoder->timestamp = timestamp;
1363 decoder->ref_timestamp = 0;
1364 decoder->timestamp_insn_cnt = 0;
1365 } else if (decoder->timestamp) {
1366 timestamp = decoder->packet.payload |
1367 (decoder->timestamp & (0xffULL << 56));
1368 decoder->tsc_timestamp = timestamp;
1369 if (timestamp < decoder->timestamp &&
1370 decoder->timestamp - timestamp < decoder->tsc_slip) {
1371 intel_pt_log_to("Suppressing backwards timestamp",
1372 timestamp);
1373 timestamp = decoder->timestamp;
1374 }
1375 if (timestamp < decoder->timestamp) {
1376 intel_pt_log_to("Wraparound timestamp", timestamp);
1377 timestamp += (1ULL << 56);
1378 decoder->tsc_timestamp = timestamp;
1379 }
1380 decoder->timestamp = timestamp;
1381 decoder->timestamp_insn_cnt = 0;
1382 }
1383
1384 if (decoder->last_packet_type == INTEL_PT_CYC) {
1385 decoder->cyc_ref_timestamp = decoder->timestamp;
1386 decoder->cycle_cnt = 0;
1387 decoder->have_calc_cyc_to_tsc = false;
1388 intel_pt_calc_cyc_to_tsc(decoder, false);
1389 }
1390
1391 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1392 }
1393
1394 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1395 {
1396 intel_pt_log("ERROR: Buffer overflow\n");
1397 intel_pt_clear_tx_flags(decoder);
1398 decoder->timestamp_insn_cnt = 0;
1399 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1400 decoder->overflow = true;
1401 return -EOVERFLOW;
1402 }
1403
1404 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1405 {
1406 uint32_t ctc = decoder->packet.payload;
1407 uint32_t fc = decoder->packet.count;
1408 uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1409
1410 if (!decoder->tsc_ctc_ratio_d)
1411 return;
1412
1413 decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1414 decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1415 if (decoder->tsc_ctc_mult) {
1416 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1417 } else {
1418 decoder->ctc_timestamp -= multdiv(ctc_rem,
1419 decoder->tsc_ctc_ratio_n,
1420 decoder->tsc_ctc_ratio_d);
1421 }
1422 decoder->ctc_delta = 0;
1423 decoder->have_tma = true;
1424 decoder->fixup_last_mtc = true;
1425 intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x CTC rem %#x\n",
1426 decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1427 }
1428
1429 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1430 {
1431 uint64_t timestamp;
1432 uint32_t mtc, mtc_delta;
1433
1434 if (!decoder->have_tma)
1435 return;
1436
1437 mtc = decoder->packet.payload;
1438
1439 if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1440 decoder->fixup_last_mtc = false;
1441 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1442 &decoder->last_mtc);
1443 }
1444
1445 if (mtc > decoder->last_mtc)
1446 mtc_delta = mtc - decoder->last_mtc;
1447 else
1448 mtc_delta = mtc + 256 - decoder->last_mtc;
1449
1450 decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1451
1452 if (decoder->tsc_ctc_mult) {
1453 timestamp = decoder->ctc_timestamp +
1454 decoder->ctc_delta * decoder->tsc_ctc_mult;
1455 } else {
1456 timestamp = decoder->ctc_timestamp +
1457 multdiv(decoder->ctc_delta,
1458 decoder->tsc_ctc_ratio_n,
1459 decoder->tsc_ctc_ratio_d);
1460 }
1461
1462 if (timestamp < decoder->timestamp)
1463 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1464 timestamp, decoder->timestamp);
1465 else
1466 decoder->timestamp = timestamp;
1467
1468 decoder->timestamp_insn_cnt = 0;
1469 decoder->last_mtc = mtc;
1470
1471 if (decoder->last_packet_type == INTEL_PT_CYC) {
1472 decoder->cyc_ref_timestamp = decoder->timestamp;
1473 decoder->cycle_cnt = 0;
1474 decoder->have_calc_cyc_to_tsc = false;
1475 intel_pt_calc_cyc_to_tsc(decoder, true);
1476 }
1477
1478 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1479 }
1480
1481 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1482 {
1483 unsigned int cbr = decoder->packet.payload & 0xff;
1484
1485 decoder->cbr_payload = decoder->packet.payload;
1486
1487 if (decoder->cbr == cbr)
1488 return;
1489
1490 decoder->cbr = cbr;
1491 decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1492 }
1493
1494 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1495 {
1496 uint64_t timestamp = decoder->cyc_ref_timestamp;
1497
1498 decoder->have_cyc = true;
1499
1500 decoder->cycle_cnt += decoder->packet.payload;
1501
1502 if (!decoder->cyc_ref_timestamp)
1503 return;
1504
1505 if (decoder->have_calc_cyc_to_tsc)
1506 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1507 else if (decoder->cbr)
1508 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1509 else
1510 return;
1511
1512 if (timestamp < decoder->timestamp)
1513 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1514 timestamp, decoder->timestamp);
1515 else
1516 decoder->timestamp = timestamp;
1517
1518 decoder->timestamp_insn_cnt = 0;
1519
1520 intel_pt_log_to("Setting timestamp", decoder->timestamp);
1521 }
1522
1523 /* Walk PSB+ packets when already in sync. */
1524 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1525 {
1526 int err;
1527
1528 while (1) {
1529 err = intel_pt_get_next_packet(decoder);
1530 if (err)
1531 return err;
1532
1533 switch (decoder->packet.type) {
1534 case INTEL_PT_PSBEND:
1535 return 0;
1536
1537 case INTEL_PT_TIP_PGD:
1538 case INTEL_PT_TIP_PGE:
1539 case INTEL_PT_TIP:
1540 case INTEL_PT_TNT:
1541 case INTEL_PT_TRACESTOP:
1542 case INTEL_PT_BAD:
1543 case INTEL_PT_PSB:
1544 case INTEL_PT_PTWRITE:
1545 case INTEL_PT_PTWRITE_IP:
1546 case INTEL_PT_EXSTOP:
1547 case INTEL_PT_EXSTOP_IP:
1548 case INTEL_PT_MWAIT:
1549 case INTEL_PT_PWRE:
1550 case INTEL_PT_PWRX:
1551 decoder->have_tma = false;
1552 intel_pt_log("ERROR: Unexpected packet\n");
1553 return -EAGAIN;
1554
1555 case INTEL_PT_OVF:
1556 return intel_pt_overflow(decoder);
1557
1558 case INTEL_PT_TSC:
1559 intel_pt_calc_tsc_timestamp(decoder);
1560 break;
1561
1562 case INTEL_PT_TMA:
1563 intel_pt_calc_tma(decoder);
1564 break;
1565
1566 case INTEL_PT_CBR:
1567 intel_pt_calc_cbr(decoder);
1568 break;
1569
1570 case INTEL_PT_MODE_EXEC:
1571 decoder->exec_mode = decoder->packet.payload;
1572 break;
1573
1574 case INTEL_PT_PIP:
1575 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1576 break;
1577
1578 case INTEL_PT_FUP:
1579 decoder->pge = true;
1580 if (decoder->packet.count)
1581 intel_pt_set_last_ip(decoder);
1582 break;
1583
1584 case INTEL_PT_MODE_TSX:
1585 intel_pt_update_in_tx(decoder);
1586 break;
1587
1588 case INTEL_PT_MTC:
1589 intel_pt_calc_mtc_timestamp(decoder);
1590 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1591 decoder->state.type |= INTEL_PT_INSTRUCTION;
1592 break;
1593
1594 case INTEL_PT_CYC:
1595 case INTEL_PT_VMCS:
1596 case INTEL_PT_MNT:
1597 case INTEL_PT_PAD:
1598 default:
1599 break;
1600 }
1601 }
1602 }
1603
1604 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1605 {
1606 int err;
1607
1608 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1609 decoder->tx_flags = 0;
1610 decoder->state.flags &= ~INTEL_PT_IN_TX;
1611 decoder->state.flags |= INTEL_PT_ABORT_TX;
1612 } else {
1613 decoder->state.flags |= INTEL_PT_ASYNC;
1614 }
1615
1616 while (1) {
1617 err = intel_pt_get_next_packet(decoder);
1618 if (err)
1619 return err;
1620
1621 switch (decoder->packet.type) {
1622 case INTEL_PT_TNT:
1623 case INTEL_PT_FUP:
1624 case INTEL_PT_TRACESTOP:
1625 case INTEL_PT_PSB:
1626 case INTEL_PT_TSC:
1627 case INTEL_PT_TMA:
1628 case INTEL_PT_MODE_TSX:
1629 case INTEL_PT_BAD:
1630 case INTEL_PT_PSBEND:
1631 case INTEL_PT_PTWRITE:
1632 case INTEL_PT_PTWRITE_IP:
1633 case INTEL_PT_EXSTOP:
1634 case INTEL_PT_EXSTOP_IP:
1635 case INTEL_PT_MWAIT:
1636 case INTEL_PT_PWRE:
1637 case INTEL_PT_PWRX:
1638 intel_pt_log("ERROR: Missing TIP after FUP\n");
1639 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1640 decoder->pkt_step = 0;
1641 return -ENOENT;
1642
1643 case INTEL_PT_CBR:
1644 intel_pt_calc_cbr(decoder);
1645 break;
1646
1647 case INTEL_PT_OVF:
1648 return intel_pt_overflow(decoder);
1649
1650 case INTEL_PT_TIP_PGD:
1651 decoder->state.from_ip = decoder->ip;
1652 if (decoder->packet.count == 0) {
1653 decoder->state.to_ip = 0;
1654 } else {
1655 intel_pt_set_ip(decoder);
1656 decoder->state.to_ip = decoder->ip;
1657 }
1658 decoder->pge = false;
1659 decoder->continuous_period = false;
1660 decoder->state.type |= INTEL_PT_TRACE_END;
1661 return 0;
1662
1663 case INTEL_PT_TIP_PGE:
1664 decoder->pge = true;
1665 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1666 decoder->ip);
1667 decoder->state.from_ip = 0;
1668 if (decoder->packet.count == 0) {
1669 decoder->state.to_ip = 0;
1670 } else {
1671 intel_pt_set_ip(decoder);
1672 decoder->state.to_ip = decoder->ip;
1673 }
1674 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1675 return 0;
1676
1677 case INTEL_PT_TIP:
1678 decoder->state.from_ip = decoder->ip;
1679 if (decoder->packet.count == 0) {
1680 decoder->state.to_ip = 0;
1681 } else {
1682 intel_pt_set_ip(decoder);
1683 decoder->state.to_ip = decoder->ip;
1684 }
1685 return 0;
1686
1687 case INTEL_PT_PIP:
1688 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1689 break;
1690
1691 case INTEL_PT_MTC:
1692 intel_pt_calc_mtc_timestamp(decoder);
1693 if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1694 decoder->state.type |= INTEL_PT_INSTRUCTION;
1695 break;
1696
1697 case INTEL_PT_CYC:
1698 intel_pt_calc_cyc_timestamp(decoder);
1699 break;
1700
1701 case INTEL_PT_MODE_EXEC:
1702 decoder->exec_mode = decoder->packet.payload;
1703 break;
1704
1705 case INTEL_PT_VMCS:
1706 case INTEL_PT_MNT:
1707 case INTEL_PT_PAD:
1708 break;
1709
1710 default:
1711 return intel_pt_bug(decoder);
1712 }
1713 }
1714 }
1715
1716 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1717 {
1718 bool no_tip = false;
1719 int err;
1720
1721 while (1) {
1722 err = intel_pt_get_next_packet(decoder);
1723 if (err)
1724 return err;
1725 next:
1726 switch (decoder->packet.type) {
1727 case INTEL_PT_TNT:
1728 if (!decoder->packet.count)
1729 break;
1730 decoder->tnt = decoder->packet;
1731 decoder->pkt_state = INTEL_PT_STATE_TNT;
1732 err = intel_pt_walk_tnt(decoder);
1733 if (err == -EAGAIN)
1734 break;
1735 return err;
1736
1737 case INTEL_PT_TIP_PGD:
1738 if (decoder->packet.count != 0)
1739 intel_pt_set_last_ip(decoder);
1740 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1741 return intel_pt_walk_tip(decoder);
1742
1743 case INTEL_PT_TIP_PGE: {
1744 decoder->pge = true;
1745 if (decoder->packet.count == 0) {
1746 intel_pt_log_at("Skipping zero TIP.PGE",
1747 decoder->pos);
1748 break;
1749 }
1750 intel_pt_set_ip(decoder);
1751 decoder->state.from_ip = 0;
1752 decoder->state.to_ip = decoder->ip;
1753 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1754 return 0;
1755 }
1756
1757 case INTEL_PT_OVF:
1758 return intel_pt_overflow(decoder);
1759
1760 case INTEL_PT_TIP:
1761 if (decoder->packet.count != 0)
1762 intel_pt_set_last_ip(decoder);
1763 decoder->pkt_state = INTEL_PT_STATE_TIP;
1764 return intel_pt_walk_tip(decoder);
1765
1766 case INTEL_PT_FUP:
1767 if (decoder->packet.count == 0) {
1768 intel_pt_log_at("Skipping zero FUP",
1769 decoder->pos);
1770 no_tip = false;
1771 break;
1772 }
1773 intel_pt_set_last_ip(decoder);
1774 if (!decoder->branch_enable) {
1775 decoder->ip = decoder->last_ip;
1776 if (intel_pt_fup_event(decoder))
1777 return 0;
1778 no_tip = false;
1779 break;
1780 }
1781 if (decoder->set_fup_mwait)
1782 no_tip = true;
1783 err = intel_pt_walk_fup(decoder);
1784 if (err != -EAGAIN) {
1785 if (err)
1786 return err;
1787 if (no_tip)
1788 decoder->pkt_state =
1789 INTEL_PT_STATE_FUP_NO_TIP;
1790 else
1791 decoder->pkt_state = INTEL_PT_STATE_FUP;
1792 return 0;
1793 }
1794 if (no_tip) {
1795 no_tip = false;
1796 break;
1797 }
1798 return intel_pt_walk_fup_tip(decoder);
1799
1800 case INTEL_PT_TRACESTOP:
1801 decoder->pge = false;
1802 decoder->continuous_period = false;
1803 intel_pt_clear_tx_flags(decoder);
1804 decoder->have_tma = false;
1805 break;
1806
1807 case INTEL_PT_PSB:
1808 decoder->last_ip = 0;
1809 decoder->have_last_ip = true;
1810 intel_pt_clear_stack(&decoder->stack);
1811 err = intel_pt_walk_psbend(decoder);
1812 if (err == -EAGAIN)
1813 goto next;
1814 if (err)
1815 return err;
1816 break;
1817
1818 case INTEL_PT_PIP:
1819 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1820 break;
1821
1822 case INTEL_PT_MTC:
1823 intel_pt_calc_mtc_timestamp(decoder);
1824 if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1825 break;
1826 /*
1827 * Ensure that there has been an instruction since the
1828 * last MTC.
1829 */
1830 if (!decoder->mtc_insn)
1831 break;
1832 decoder->mtc_insn = false;
1833 /* Ensure that there is a timestamp */
1834 if (!decoder->timestamp)
1835 break;
1836 decoder->state.type = INTEL_PT_INSTRUCTION;
1837 decoder->state.from_ip = decoder->ip;
1838 decoder->state.to_ip = 0;
1839 decoder->mtc_insn = false;
1840 return 0;
1841
1842 case INTEL_PT_TSC:
1843 intel_pt_calc_tsc_timestamp(decoder);
1844 break;
1845
1846 case INTEL_PT_TMA:
1847 intel_pt_calc_tma(decoder);
1848 break;
1849
1850 case INTEL_PT_CYC:
1851 intel_pt_calc_cyc_timestamp(decoder);
1852 break;
1853
1854 case INTEL_PT_CBR:
1855 intel_pt_calc_cbr(decoder);
1856 if (!decoder->branch_enable &&
1857 decoder->cbr != decoder->cbr_seen) {
1858 decoder->cbr_seen = decoder->cbr;
1859 decoder->state.type = INTEL_PT_CBR_CHG;
1860 decoder->state.from_ip = decoder->ip;
1861 decoder->state.to_ip = 0;
1862 decoder->state.cbr_payload =
1863 decoder->packet.payload;
1864 return 0;
1865 }
1866 break;
1867
1868 case INTEL_PT_MODE_EXEC:
1869 decoder->exec_mode = decoder->packet.payload;
1870 break;
1871
1872 case INTEL_PT_MODE_TSX:
1873 /* MODE_TSX need not be followed by FUP */
1874 if (!decoder->pge) {
1875 intel_pt_update_in_tx(decoder);
1876 break;
1877 }
1878 err = intel_pt_mode_tsx(decoder, &no_tip);
1879 if (err)
1880 return err;
1881 goto next;
1882
1883 case INTEL_PT_BAD: /* Does not happen */
1884 return intel_pt_bug(decoder);
1885
1886 case INTEL_PT_PSBEND:
1887 case INTEL_PT_VMCS:
1888 case INTEL_PT_MNT:
1889 case INTEL_PT_PAD:
1890 break;
1891
1892 case INTEL_PT_PTWRITE_IP:
1893 decoder->fup_ptw_payload = decoder->packet.payload;
1894 err = intel_pt_get_next_packet(decoder);
1895 if (err)
1896 return err;
1897 if (decoder->packet.type == INTEL_PT_FUP) {
1898 decoder->set_fup_ptw = true;
1899 no_tip = true;
1900 } else {
1901 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1902 decoder->pos);
1903 }
1904 goto next;
1905
1906 case INTEL_PT_PTWRITE:
1907 decoder->state.type = INTEL_PT_PTW;
1908 decoder->state.from_ip = decoder->ip;
1909 decoder->state.to_ip = 0;
1910 decoder->state.ptw_payload = decoder->packet.payload;
1911 return 0;
1912
1913 case INTEL_PT_MWAIT:
1914 decoder->fup_mwait_payload = decoder->packet.payload;
1915 decoder->set_fup_mwait = true;
1916 break;
1917
1918 case INTEL_PT_PWRE:
1919 if (decoder->set_fup_mwait) {
1920 decoder->fup_pwre_payload =
1921 decoder->packet.payload;
1922 decoder->set_fup_pwre = true;
1923 break;
1924 }
1925 decoder->state.type = INTEL_PT_PWR_ENTRY;
1926 decoder->state.from_ip = decoder->ip;
1927 decoder->state.to_ip = 0;
1928 decoder->state.pwrx_payload = decoder->packet.payload;
1929 return 0;
1930
1931 case INTEL_PT_EXSTOP_IP:
1932 err = intel_pt_get_next_packet(decoder);
1933 if (err)
1934 return err;
1935 if (decoder->packet.type == INTEL_PT_FUP) {
1936 decoder->set_fup_exstop = true;
1937 no_tip = true;
1938 } else {
1939 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1940 decoder->pos);
1941 }
1942 goto next;
1943
1944 case INTEL_PT_EXSTOP:
1945 decoder->state.type = INTEL_PT_EX_STOP;
1946 decoder->state.from_ip = decoder->ip;
1947 decoder->state.to_ip = 0;
1948 return 0;
1949
1950 case INTEL_PT_PWRX:
1951 decoder->state.type = INTEL_PT_PWR_EXIT;
1952 decoder->state.from_ip = decoder->ip;
1953 decoder->state.to_ip = 0;
1954 decoder->state.pwrx_payload = decoder->packet.payload;
1955 return 0;
1956
1957 default:
1958 return intel_pt_bug(decoder);
1959 }
1960 }
1961 }
1962
1963 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1964 {
1965 return decoder->packet.count &&
1966 (decoder->have_last_ip || decoder->packet.count == 3 ||
1967 decoder->packet.count == 6);
1968 }
1969
1970 /* Walk PSB+ packets to get in sync. */
1971 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1972 {
1973 int err;
1974
1975 while (1) {
1976 err = intel_pt_get_next_packet(decoder);
1977 if (err)
1978 return err;
1979
1980 switch (decoder->packet.type) {
1981 case INTEL_PT_TIP_PGD:
1982 decoder->continuous_period = false;
1983 __fallthrough;
1984 case INTEL_PT_TIP_PGE:
1985 case INTEL_PT_TIP:
1986 case INTEL_PT_PTWRITE:
1987 case INTEL_PT_PTWRITE_IP:
1988 case INTEL_PT_EXSTOP:
1989 case INTEL_PT_EXSTOP_IP:
1990 case INTEL_PT_MWAIT:
1991 case INTEL_PT_PWRE:
1992 case INTEL_PT_PWRX:
1993 intel_pt_log("ERROR: Unexpected packet\n");
1994 return -ENOENT;
1995
1996 case INTEL_PT_FUP:
1997 decoder->pge = true;
1998 if (intel_pt_have_ip(decoder)) {
1999 uint64_t current_ip = decoder->ip;
2000
2001 intel_pt_set_ip(decoder);
2002 if (current_ip)
2003 intel_pt_log_to("Setting IP",
2004 decoder->ip);
2005 }
2006 break;
2007
2008 case INTEL_PT_MTC:
2009 intel_pt_calc_mtc_timestamp(decoder);
2010 break;
2011
2012 case INTEL_PT_TSC:
2013 intel_pt_calc_tsc_timestamp(decoder);
2014 break;
2015
2016 case INTEL_PT_TMA:
2017 intel_pt_calc_tma(decoder);
2018 break;
2019
2020 case INTEL_PT_CYC:
2021 intel_pt_calc_cyc_timestamp(decoder);
2022 break;
2023
2024 case INTEL_PT_CBR:
2025 intel_pt_calc_cbr(decoder);
2026 break;
2027
2028 case INTEL_PT_PIP:
2029 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2030 break;
2031
2032 case INTEL_PT_MODE_EXEC:
2033 decoder->exec_mode = decoder->packet.payload;
2034 break;
2035
2036 case INTEL_PT_MODE_TSX:
2037 intel_pt_update_in_tx(decoder);
2038 break;
2039
2040 case INTEL_PT_TRACESTOP:
2041 decoder->pge = false;
2042 decoder->continuous_period = false;
2043 intel_pt_clear_tx_flags(decoder);
2044 __fallthrough;
2045
2046 case INTEL_PT_TNT:
2047 decoder->have_tma = false;
2048 intel_pt_log("ERROR: Unexpected packet\n");
2049 if (decoder->ip)
2050 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2051 else
2052 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2053 return -ENOENT;
2054
2055 case INTEL_PT_BAD: /* Does not happen */
2056 return intel_pt_bug(decoder);
2057
2058 case INTEL_PT_OVF:
2059 return intel_pt_overflow(decoder);
2060
2061 case INTEL_PT_PSBEND:
2062 return 0;
2063
2064 case INTEL_PT_PSB:
2065 case INTEL_PT_VMCS:
2066 case INTEL_PT_MNT:
2067 case INTEL_PT_PAD:
2068 default:
2069 break;
2070 }
2071 }
2072 }
2073
2074 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2075 {
2076 int err;
2077
2078 while (1) {
2079 err = intel_pt_get_next_packet(decoder);
2080 if (err)
2081 return err;
2082
2083 switch (decoder->packet.type) {
2084 case INTEL_PT_TIP_PGD:
2085 decoder->continuous_period = false;
2086 __fallthrough;
2087 case INTEL_PT_TIP_PGE:
2088 case INTEL_PT_TIP:
2089 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2090 if (intel_pt_have_ip(decoder))
2091 intel_pt_set_ip(decoder);
2092 if (!decoder->ip)
2093 break;
2094 if (decoder->packet.type == INTEL_PT_TIP_PGE)
2095 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2096 if (decoder->packet.type == INTEL_PT_TIP_PGD)
2097 decoder->state.type |= INTEL_PT_TRACE_END;
2098 return 0;
2099
2100 case INTEL_PT_FUP:
2101 if (intel_pt_have_ip(decoder))
2102 intel_pt_set_ip(decoder);
2103 if (decoder->ip)
2104 return 0;
2105 break;
2106
2107 case INTEL_PT_MTC:
2108 intel_pt_calc_mtc_timestamp(decoder);
2109 break;
2110
2111 case INTEL_PT_TSC:
2112 intel_pt_calc_tsc_timestamp(decoder);
2113 break;
2114
2115 case INTEL_PT_TMA:
2116 intel_pt_calc_tma(decoder);
2117 break;
2118
2119 case INTEL_PT_CYC:
2120 intel_pt_calc_cyc_timestamp(decoder);
2121 break;
2122
2123 case INTEL_PT_CBR:
2124 intel_pt_calc_cbr(decoder);
2125 break;
2126
2127 case INTEL_PT_PIP:
2128 decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2129 break;
2130
2131 case INTEL_PT_MODE_EXEC:
2132 decoder->exec_mode = decoder->packet.payload;
2133 break;
2134
2135 case INTEL_PT_MODE_TSX:
2136 intel_pt_update_in_tx(decoder);
2137 break;
2138
2139 case INTEL_PT_OVF:
2140 return intel_pt_overflow(decoder);
2141
2142 case INTEL_PT_BAD: /* Does not happen */
2143 return intel_pt_bug(decoder);
2144
2145 case INTEL_PT_TRACESTOP:
2146 decoder->pge = false;
2147 decoder->continuous_period = false;
2148 intel_pt_clear_tx_flags(decoder);
2149 decoder->have_tma = false;
2150 break;
2151
2152 case INTEL_PT_PSB:
2153 decoder->last_ip = 0;
2154 decoder->have_last_ip = true;
2155 intel_pt_clear_stack(&decoder->stack);
2156 err = intel_pt_walk_psb(decoder);
2157 if (err)
2158 return err;
2159 if (decoder->ip) {
2160 /* Do not have a sample */
2161 decoder->state.type = 0;
2162 return 0;
2163 }
2164 break;
2165
2166 case INTEL_PT_TNT:
2167 case INTEL_PT_PSBEND:
2168 case INTEL_PT_VMCS:
2169 case INTEL_PT_MNT:
2170 case INTEL_PT_PAD:
2171 case INTEL_PT_PTWRITE:
2172 case INTEL_PT_PTWRITE_IP:
2173 case INTEL_PT_EXSTOP:
2174 case INTEL_PT_EXSTOP_IP:
2175 case INTEL_PT_MWAIT:
2176 case INTEL_PT_PWRE:
2177 case INTEL_PT_PWRX:
2178 default:
2179 break;
2180 }
2181 }
2182 }
2183
2184 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2185 {
2186 int err;
2187
2188 decoder->set_fup_tx_flags = false;
2189 decoder->set_fup_ptw = false;
2190 decoder->set_fup_mwait = false;
2191 decoder->set_fup_pwre = false;
2192 decoder->set_fup_exstop = false;
2193
2194 if (!decoder->branch_enable) {
2195 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2196 decoder->overflow = false;
2197 decoder->state.type = 0; /* Do not have a sample */
2198 return 0;
2199 }
2200
2201 intel_pt_log("Scanning for full IP\n");
2202 err = intel_pt_walk_to_ip(decoder);
2203 if (err)
2204 return err;
2205
2206 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2207 decoder->overflow = false;
2208
2209 decoder->state.from_ip = 0;
2210 decoder->state.to_ip = decoder->ip;
2211 intel_pt_log_to("Setting IP", decoder->ip);
2212
2213 return 0;
2214 }
2215
2216 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2217 {
2218 const unsigned char *end = decoder->buf + decoder->len;
2219 size_t i;
2220
2221 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2222 if (i > decoder->len)
2223 continue;
2224 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2225 return i;
2226 }
2227 return 0;
2228 }
2229
2230 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2231 {
2232 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2233 const char *psb = INTEL_PT_PSB_STR;
2234
2235 if (rest_psb > decoder->len ||
2236 memcmp(decoder->buf, psb + part_psb, rest_psb))
2237 return 0;
2238
2239 return rest_psb;
2240 }
2241
2242 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2243 int part_psb)
2244 {
2245 int rest_psb, ret;
2246
2247 decoder->pos += decoder->len;
2248 decoder->len = 0;
2249
2250 ret = intel_pt_get_next_data(decoder);
2251 if (ret)
2252 return ret;
2253
2254 rest_psb = intel_pt_rest_psb(decoder, part_psb);
2255 if (!rest_psb)
2256 return 0;
2257
2258 decoder->pos -= part_psb;
2259 decoder->next_buf = decoder->buf + rest_psb;
2260 decoder->next_len = decoder->len - rest_psb;
2261 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2262 decoder->buf = decoder->temp_buf;
2263 decoder->len = INTEL_PT_PSB_LEN;
2264
2265 return 0;
2266 }
2267
2268 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2269 {
2270 unsigned char *next;
2271 int ret;
2272
2273 intel_pt_log("Scanning for PSB\n");
2274 while (1) {
2275 if (!decoder->len) {
2276 ret = intel_pt_get_next_data(decoder);
2277 if (ret)
2278 return ret;
2279 }
2280
2281 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2282 INTEL_PT_PSB_LEN);
2283 if (!next) {
2284 int part_psb;
2285
2286 part_psb = intel_pt_part_psb(decoder);
2287 if (part_psb) {
2288 ret = intel_pt_get_split_psb(decoder, part_psb);
2289 if (ret)
2290 return ret;
2291 } else {
2292 decoder->pos += decoder->len;
2293 decoder->len = 0;
2294 }
2295 continue;
2296 }
2297
2298 decoder->pkt_step = next - decoder->buf;
2299 return intel_pt_get_next_packet(decoder);
2300 }
2301 }
2302
2303 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2304 {
2305 int err;
2306
2307 decoder->pge = false;
2308 decoder->continuous_period = false;
2309 decoder->have_last_ip = false;
2310 decoder->last_ip = 0;
2311 decoder->ip = 0;
2312 intel_pt_clear_stack(&decoder->stack);
2313
2314 err = intel_pt_scan_for_psb(decoder);
2315 if (err)
2316 return err;
2317
2318 decoder->have_last_ip = true;
2319 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2320
2321 err = intel_pt_walk_psb(decoder);
2322 if (err)
2323 return err;
2324
2325 if (decoder->ip) {
2326 decoder->state.type = 0; /* Do not have a sample */
2327 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2328 } else {
2329 return intel_pt_sync_ip(decoder);
2330 }
2331
2332 return 0;
2333 }
2334
2335 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2336 {
2337 uint64_t est = decoder->sample_insn_cnt << 1;
2338
2339 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2340 goto out;
2341
2342 est *= decoder->max_non_turbo_ratio;
2343 est /= decoder->cbr;
2344 out:
2345 return decoder->sample_timestamp + est;
2346 }
2347
2348 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2349 {
2350 int err;
2351
2352 do {
2353 decoder->state.type = INTEL_PT_BRANCH;
2354 decoder->state.flags = 0;
2355
2356 switch (decoder->pkt_state) {
2357 case INTEL_PT_STATE_NO_PSB:
2358 err = intel_pt_sync(decoder);
2359 break;
2360 case INTEL_PT_STATE_NO_IP:
2361 decoder->have_last_ip = false;
2362 decoder->last_ip = 0;
2363 decoder->ip = 0;
2364 __fallthrough;
2365 case INTEL_PT_STATE_ERR_RESYNC:
2366 err = intel_pt_sync_ip(decoder);
2367 break;
2368 case INTEL_PT_STATE_IN_SYNC:
2369 err = intel_pt_walk_trace(decoder);
2370 break;
2371 case INTEL_PT_STATE_TNT:
2372 err = intel_pt_walk_tnt(decoder);
2373 if (err == -EAGAIN)
2374 err = intel_pt_walk_trace(decoder);
2375 break;
2376 case INTEL_PT_STATE_TIP:
2377 case INTEL_PT_STATE_TIP_PGD:
2378 err = intel_pt_walk_tip(decoder);
2379 break;
2380 case INTEL_PT_STATE_FUP:
2381 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2382 err = intel_pt_walk_fup(decoder);
2383 if (err == -EAGAIN)
2384 err = intel_pt_walk_fup_tip(decoder);
2385 else if (!err)
2386 decoder->pkt_state = INTEL_PT_STATE_FUP;
2387 break;
2388 case INTEL_PT_STATE_FUP_NO_TIP:
2389 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2390 err = intel_pt_walk_fup(decoder);
2391 if (err == -EAGAIN)
2392 err = intel_pt_walk_trace(decoder);
2393 break;
2394 default:
2395 err = intel_pt_bug(decoder);
2396 break;
2397 }
2398 } while (err == -ENOLINK);
2399
2400 if (err) {
2401 decoder->state.err = intel_pt_ext_err(err);
2402 decoder->state.from_ip = decoder->ip;
2403 decoder->sample_timestamp = decoder->timestamp;
2404 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2405 } else {
2406 decoder->state.err = 0;
2407 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2408 decoder->cbr_seen = decoder->cbr;
2409 decoder->state.type |= INTEL_PT_CBR_CHG;
2410 decoder->state.cbr_payload = decoder->cbr_payload;
2411 }
2412 if (intel_pt_sample_time(decoder->pkt_state)) {
2413 decoder->sample_timestamp = decoder->timestamp;
2414 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2415 }
2416 }
2417
2418 decoder->state.timestamp = decoder->sample_timestamp;
2419 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2420 decoder->state.cr3 = decoder->cr3;
2421 decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2422
2423 return &decoder->state;
2424 }
2425
2426 /**
2427 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2428 * @buf: pointer to buffer pointer
2429 * @len: size of buffer
2430 *
2431 * Updates the buffer pointer to point to the start of the next PSB packet if
2432 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
2433 * @len is adjusted accordingly.
2434 *
2435 * Return: %true if a PSB packet is found, %false otherwise.
2436 */
2437 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2438 {
2439 unsigned char *next;
2440
2441 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2442 if (next) {
2443 *len -= next - *buf;
2444 *buf = next;
2445 return true;
2446 }
2447 return false;
2448 }
2449
2450 /**
2451 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2452 * packet.
2453 * @buf: pointer to buffer pointer
2454 * @len: size of buffer
2455 *
2456 * Updates the buffer pointer to point to the start of the following PSB packet
2457 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2458 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
2459 *
2460 * Return: %true if a PSB packet is found, %false otherwise.
2461 */
2462 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2463 {
2464 unsigned char *next;
2465
2466 if (!*len)
2467 return false;
2468
2469 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2470 if (next) {
2471 *len -= next - *buf;
2472 *buf = next;
2473 return true;
2474 }
2475 return false;
2476 }
2477
2478 /**
2479 * intel_pt_last_psb - find the last PSB packet in a buffer.
2480 * @buf: buffer
2481 * @len: size of buffer
2482 *
2483 * This function finds the last PSB in a buffer.
2484 *
2485 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2486 */
2487 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2488 {
2489 const char *n = INTEL_PT_PSB_STR;
2490 unsigned char *p;
2491 size_t k;
2492
2493 if (len < INTEL_PT_PSB_LEN)
2494 return NULL;
2495
2496 k = len - INTEL_PT_PSB_LEN + 1;
2497 while (1) {
2498 p = memrchr(buf, n[0], k);
2499 if (!p)
2500 return NULL;
2501 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2502 return p;
2503 k = p - buf;
2504 if (!k)
2505 return NULL;
2506 }
2507 }
2508
2509 /**
2510 * intel_pt_next_tsc - find and return next TSC.
2511 * @buf: buffer
2512 * @len: size of buffer
2513 * @tsc: TSC value returned
2514 * @rem: returns remaining size when TSC is found
2515 *
2516 * Find a TSC packet in @buf and return the TSC value. This function assumes
2517 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2518 * PSBEND packet is found.
2519 *
2520 * Return: %true if TSC is found, false otherwise.
2521 */
2522 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2523 size_t *rem)
2524 {
2525 struct intel_pt_pkt packet;
2526 int ret;
2527
2528 while (len) {
2529 ret = intel_pt_get_packet(buf, len, &packet);
2530 if (ret <= 0)
2531 return false;
2532 if (packet.type == INTEL_PT_TSC) {
2533 *tsc = packet.payload;
2534 *rem = len;
2535 return true;
2536 }
2537 if (packet.type == INTEL_PT_PSBEND)
2538 return false;
2539 buf += ret;
2540 len -= ret;
2541 }
2542 return false;
2543 }
2544
2545 /**
2546 * intel_pt_tsc_cmp - compare 7-byte TSCs.
2547 * @tsc1: first TSC to compare
2548 * @tsc2: second TSC to compare
2549 *
2550 * This function compares 7-byte TSC values allowing for the possibility that
2551 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
2552 * around so for that purpose this function assumes the absolute difference is
2553 * less than half the maximum difference.
2554 *
2555 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2556 * after @tsc2.
2557 */
2558 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2559 {
2560 const uint64_t halfway = (1ULL << 55);
2561
2562 if (tsc1 == tsc2)
2563 return 0;
2564
2565 if (tsc1 < tsc2) {
2566 if (tsc2 - tsc1 < halfway)
2567 return -1;
2568 else
2569 return 1;
2570 } else {
2571 if (tsc1 - tsc2 < halfway)
2572 return 1;
2573 else
2574 return -1;
2575 }
2576 }
2577
2578 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2579
2580 /**
2581 * adj_for_padding - adjust overlap to account for padding.
2582 * @buf_b: second buffer
2583 * @buf_a: first buffer
2584 * @len_a: size of first buffer
2585 *
2586 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2587 * accordingly.
2588 *
2589 * Return: A pointer into @buf_b from where non-overlapped data starts
2590 */
2591 static unsigned char *adj_for_padding(unsigned char *buf_b,
2592 unsigned char *buf_a, size_t len_a)
2593 {
2594 unsigned char *p = buf_b - MAX_PADDING;
2595 unsigned char *q = buf_a + len_a - MAX_PADDING;
2596 int i;
2597
2598 for (i = MAX_PADDING; i; i--, p++, q++) {
2599 if (*p != *q)
2600 break;
2601 }
2602
2603 return p;
2604 }
2605
2606 /**
2607 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2608 * using TSC.
2609 * @buf_a: first buffer
2610 * @len_a: size of first buffer
2611 * @buf_b: second buffer
2612 * @len_b: size of second buffer
2613 * @consecutive: returns true if there is data in buf_b that is consecutive
2614 * to buf_a
2615 *
2616 * If the trace contains TSC we can look at the last TSC of @buf_a and the
2617 * first TSC of @buf_b in order to determine if the buffers overlap, and then
2618 * walk forward in @buf_b until a later TSC is found. A precondition is that
2619 * @buf_a and @buf_b are positioned at a PSB.
2620 *
2621 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2622 * @buf_b + @len_b if there is no non-overlapped data.
2623 */
2624 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2625 size_t len_a,
2626 unsigned char *buf_b,
2627 size_t len_b, bool *consecutive)
2628 {
2629 uint64_t tsc_a, tsc_b;
2630 unsigned char *p;
2631 size_t len, rem_a, rem_b;
2632
2633 p = intel_pt_last_psb(buf_a, len_a);
2634 if (!p)
2635 return buf_b; /* No PSB in buf_a => no overlap */
2636
2637 len = len_a - (p - buf_a);
2638 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2639 /* The last PSB+ in buf_a is incomplete, so go back one more */
2640 len_a -= len;
2641 p = intel_pt_last_psb(buf_a, len_a);
2642 if (!p)
2643 return buf_b; /* No full PSB+ => assume no overlap */
2644 len = len_a - (p - buf_a);
2645 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2646 return buf_b; /* No TSC in buf_a => assume no overlap */
2647 }
2648
2649 while (1) {
2650 /* Ignore PSB+ with no TSC */
2651 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2652 int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2653
2654 /* Same TSC, so buffers are consecutive */
2655 if (!cmp && rem_b >= rem_a) {
2656 unsigned char *start;
2657
2658 *consecutive = true;
2659 start = buf_b + len_b - (rem_b - rem_a);
2660 return adj_for_padding(start, buf_a, len_a);
2661 }
2662 if (cmp < 0)
2663 return buf_b; /* tsc_a < tsc_b => no overlap */
2664 }
2665
2666 if (!intel_pt_step_psb(&buf_b, &len_b))
2667 return buf_b + len_b; /* No PSB in buf_b => no data */
2668 }
2669 }
2670
2671 /**
2672 * intel_pt_find_overlap - determine start of non-overlapped trace data.
2673 * @buf_a: first buffer
2674 * @len_a: size of first buffer
2675 * @buf_b: second buffer
2676 * @len_b: size of second buffer
2677 * @have_tsc: can use TSC packets to detect overlap
2678 * @consecutive: returns true if there is data in buf_b that is consecutive
2679 * to buf_a
2680 *
2681 * When trace samples or snapshots are recorded there is the possibility that
2682 * the data overlaps. Note that, for the purposes of decoding, data is only
2683 * useful if it begins with a PSB packet.
2684 *
2685 * Return: A pointer into @buf_b from where non-overlapped data starts, or
2686 * @buf_b + @len_b if there is no non-overlapped data.
2687 */
2688 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2689 unsigned char *buf_b, size_t len_b,
2690 bool have_tsc, bool *consecutive)
2691 {
2692 unsigned char *found;
2693
2694 /* Buffer 'b' must start at PSB so throw away everything before that */
2695 if (!intel_pt_next_psb(&buf_b, &len_b))
2696 return buf_b + len_b; /* No PSB */
2697
2698 if (!intel_pt_next_psb(&buf_a, &len_a))
2699 return buf_b; /* No overlap */
2700
2701 if (have_tsc) {
2702 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2703 consecutive);
2704 if (found)
2705 return found;
2706 }
2707
2708 /*
2709 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2710 * we can ignore the first part of buffer 'a'.
2711 */
2712 while (len_b < len_a) {
2713 if (!intel_pt_step_psb(&buf_a, &len_a))
2714 return buf_b; /* No overlap */
2715 }
2716
2717 /* Now len_b >= len_a */
2718 while (1) {
2719 /* Potential overlap so check the bytes */
2720 found = memmem(buf_a, len_a, buf_b, len_a);
2721 if (found) {
2722 *consecutive = true;
2723 return adj_for_padding(buf_b + len_a, buf_a, len_a);
2724 }
2725
2726 /* Try again at next PSB in buffer 'a' */
2727 if (!intel_pt_step_psb(&buf_a, &len_a))
2728 return buf_b; /* No overlap */
2729 }
2730 }
Linux with added FPC-III support