| blob | 0bf9e5c27b599bfc78f7f65021d0966ed25f008a |
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright(C) 2015-2018 Linaro Limited.
4 *
5 * Author: Tor Jeremiassen <tor@ti.com>
6 * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
7 */
9 #include <linux/kernel.h>
10 #include <linux/bitfield.h>
11 #include <linux/bitops.h>
12 #include <linux/coresight-pmu.h>
13 #include <linux/err.h>
14 #include <linux/log2.h>
15 #include <linux/types.h>
16 #include <linux/zalloc.h>
18 #include <stdlib.h>
39 #include <tools/libc_compat.h>
51 /*
52 * Timeless has no timestamps in the trace so overlapping mmap lookups
53 * are less accurate but produces smaller trace data. We use context IDs
54 * in the trace instead of matching timestamps with fork records so
55 * they're not really needed in the general case. Overlapping mmaps
56 * happen in cases like between a fork and an exec.
57 */
60 /*
61 * Per-thread ignores the trace channel ID and instead assumes that
62 * everything in a buffer comes from the same process regardless of
63 * which CPU it ran on. It also implies no context IDs so the TID is
64 * taken from the auxtrace buffer.
65 */
72 u64 latest_kernel_timestamp;
73 u32 auxtrace_type;
74 u64 branches_sample_type;
75 u64 branches_id;
76 u64 instructions_sample_type;
77 u64 instructions_sample_period;
78 u64 instructions_id;
82 };
85 u8 trace_chan_id;
86 u64 period_instructions;
91 ocsd_ex_level prev_packet_el;
92 ocsd_ex_level el;
98 };
101 UNSET,
102 FORMATTED,
103 UNFORMATTED
104 };
111 u8 pending_timestamp_chan_id;
113 u64 offset;
116 /* Conversion between traceID and index in traceid_queues array */
119 /* Conversion between traceID and metadata pointers */
121 /*
122 * Same as traceid_list, but traceid_list may be a reference to another
123 * queue's which has a matching sink ID.
124 */
126 u32 sink_id;
127 };
131 pid_t tid);
138 /* PTMs ETMIDR [11:8] set to b0011 */
139 #define ETMIDR_PTM_VERSION 0x00000300
141 /*
142 * A struct auxtrace_heap_item only has a queue_nr and a timestamp to
143 * work with. One option is to modify to auxtrace_heap_XYZ() API or simply
144 * encode the etm queue number as the upper 16 bit and the channel as
145 * the lower 16 bit.
146 */
147 #define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \
148 (queue_nr << 16 | trace_chan_id)
149 #define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16)
150 #define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff)
151 #define SINK_UNSET ((u32) -1)
154 {
161 }
164 {
175 }
178 {
189 }
191 /*
192 * The returned PID format is presented as an enum:
193 *
194 * CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced.
195 * CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced.
196 * CS_ETM_PIDFMT_NONE: No context IDs
197 *
198 * It's possible that the two bits ETM_OPT_CTXTID and ETM_OPT_CTXTID2
199 * are enabled at the same time when the session runs on an EL2 kernel.
200 * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be
201 * recorded in the trace data, the tool will selectively use
202 * CONTEXTIDR_EL2 as PID.
203 *
204 * The result is cached in etm->pid_fmt so this function only needs to be called
205 * when processing the aux info.
206 */
208 {
209 u64 val;
213 /* CONTEXTIDR is traced */
218 /* CONTEXTIDR_EL2 is traced */
221 /* CONTEXTIDR_EL1 is traced */
224 }
227 }
230 {
232 }
236 {
237 /* Get an RB node for this CPU */
240 /* Something went wrong, no need to continue */
244 /* Disallow re-mapping a different traceID to metadata pair. */
247 u8 curr_chan_id;
251 /*
252 * With > CORESIGHT_TRACE_IDS_MAX ETMs, overlapping IDs
253 * are expected (but not supported) in per-thread mode,
254 * rather than signifying an error.
255 */
258 else
262 }
264 /* check that the mapped ID matches */
272 }
274 /* Skip re-adding the same mappings if everything matched */
276 }
278 /* Not one we've seen before, associate the traceID with the metadata pointer */
282 }
285 {
288 else
290 }
294 {
297 /*
298 * If the queue is unformatted then only save one mapping in the
299 * queue associated with that CPU so only one decoder is made.
300 */
304 cpu_metadata);
306 /*
307 * Otherwise, version 0 trace IDs are global so save them into every
308 * queue.
309 */
315 cpu_metadata);
318 }
321 }
324 u64 hw_id)
325 {
338 /*
339 * if we are picking up the association from the packet, need to plug
340 * the correct trace ID into the metadata for setting up decoders later.
341 */
343 }
346 u64 hw_id)
347 {
354 /*
355 * Check sink id hasn't changed in per-cpu mode. In per-thread mode,
356 * let it pass for now until an actual overlapping trace ID is hit. In
357 * most cases IDs won't overlap even if the sink changes.
358 */
363 }
367 /* Find which other queues use this sink and link their ID maps */
371 /* Different sinks, skip */
375 /* Already linked, skip */
379 /* At the point of first linking, this one should be empty */
383 }
389 }
401 }
404 {
410 CORESIGHT_TRACE_ID_VAL_MASK);
415 CORESIGHT_TRACE_ID_VAL_MASK);
419 }
421 }
423 /*
424 * update metadata trace ID from the value found in the AUX_HW_INFO packet.
425 */
427 {
441 }
443 }
445 /*
446 * Get a metadata index for a specific cpu from an array.
447 *
448 */
450 {
456 }
457 }
460 }
462 /*
463 * Get a metadata for a specific cpu from an array.
464 *
465 */
467 {
471 }
473 /*
474 * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event.
475 *
476 * The payload associates the Trace ID and the CPU.
477 * The routine is tolerant of seeing multiple packets with the same association,
478 * but a CPU / Trace ID association changing during a session is an error.
479 */
482 {
486 u64 hw_id;
489 /* extract and parse the HW ID */
493 /* check that we can handle this version */
495 pr_err("CS ETM Trace: PERF_RECORD_AUX_OUTPUT_HW_ID version %d not supported. Please update Perf.\n",
496 version);
498 }
500 /* get access to the etm metadata */
505 /* parse the sample to get the CPU */
514 /* no CPU in the sample - possibly recorded with an old version of perf */
517 }
523 }
526 u8 trace_chan_id)
527 {
528 /*
529 * When a timestamp packet is encountered the backend code
530 * is stopped so that the front end has time to process packets
531 * that were accumulated in the traceID queue. Since there can
532 * be more than one channel per cs_etm_queue, we need to specify
533 * what traceID queue needs servicing.
534 */
536 }
540 {
554 /* Acknowledge pending status */
557 /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */
559 }
562 {
582 }
583 }
586 {
596 }
597 }
601 u8 trace_chan_id)
602 {
635 }
643 out_free:
648 out:
650 }
652 static struct cs_etm_traceid_queue
654 {
666 /*
667 * Check if the traceid_queue exist for this traceID by looking
668 * in the queue list.
669 */
674 }
676 /* We couldn't find a traceid_queue for this traceID, allocate one */
683 /* Get a valid index for the new traceid_queue */
685 /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */
690 /* Associate this traceID with this index */
696 /* Grow the traceid_queues array by one unit */
702 /*
703 * On failure reallocarray() returns NULL and the original block of
704 * memory is left untouched.
705 */
714 out_free:
715 /*
716 * Function intlist__remove() removes the inode from the list
717 * and delete the memory associated to it.
718 */
723 }
725 struct cs_etm_packet_queue
727 {
735 }
739 {
744 /*
745 * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for
746 * the next incoming packet.
747 *
748 * Threads and exception levels are also tracked for both the
749 * previous and current packets. This is because the previous
750 * packet is used for the 'from' IP for branch samples, so the
751 * thread at that time must also be assigned to that sample.
752 * Across discontinuity packets the thread can change, so by
753 * tracking the thread for the previous packet the branch sample
754 * will have the correct info.
755 */
762 }
763 }
766 {
774 else
779 else
783 }
787 {
791 }
795 {
803 }
807 {
816 }
820 {
826 u32 etmidr;
841 }
842 }
845 }
850 {
865 out:
867 }
871 {
895 }
899 {
902 auxtrace);
910 /*
911 * Pass tid = -1 to process all queues. But likely they will have
912 * already been processed on PERF_RECORD_EXIT anyway.
913 */
915 }
918 }
921 {
932 /* Free this traceid_queue from the array */
943 /*
944 * Function intlist__remove() removes the inode from the list
945 * and delete the memory associated to it.
946 */
948 }
950 /* Then the RB tree itself */
954 /* finally free the traceid_queues array */
956 }
959 {
970 /* First remove all traceID/metadata nodes for the RB tree */
974 /* Then the RB tree itself */
976 }
979 }
982 {
986 auxtrace);
992 }
995 }
998 {
1002 auxtrace);
1011 }
1015 {
1018 auxtrace);
1021 }
1024 ocsd_ex_level el)
1025 {
1028 /*
1029 * For any virtualisation based on nVHE (e.g. pKVM), or host kernels
1030 * running at EL1 assume everything is the host.
1031 */
1035 /*
1036 * Not perfect, but otherwise assume anything in EL1 is the default
1037 * guest, and everything else is the host. Distinguishing between guest
1038 * and host userspaces isn't currently supported either. Neither is
1039 * multiple guest support. All this does is reduce the likeliness of
1040 * decode errors where we look into the host kernel maps when it should
1041 * have been the guest maps.
1042 */
1046 DEFAULT_GUEST_KERNEL_ID);
1053 }
1054 }
1057 ocsd_ex_level el)
1058 {
1064 else
1070 /*
1071 * Can't really happen at the moment because
1072 * cs_etm__get_machine() will always return
1073 * machines.host for any non EL1 trace.
1074 */
1076 }
1077 }
1078 }
1083 {
1084 u8 cpumode;
1085 u64 offset;
1100 /*
1101 * We've already tracked EL along side the PID in cs_etm__set_thread()
1102 * so double check that it matches what OpenCSD thinks as well. It
1103 * doesn't distinguish between EL0 and EL1 for this mem access callback
1104 * so we had to do the extra tracking. Skip validation if it's any of
1105 * the 'any' values.
1106 */
1110 /* Includes both non secure EL1 and EL0 */
1116 }
1139 ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n"
1143 address,
1146 }
1148 }
1150 out:
1153 }
1156 {
1165 /*
1166 * Create an RB tree for traceID-metadata tuple. Since the conversion
1167 * has to be made for each packet that gets decoded, optimizing access
1168 * in anything other than a sequential array is worth doing.
1169 */
1176 out_free:
1181 }
1186 {
1205 }
1210 {
1213 u8 trace_chan_id;
1214 u64 cs_timestamp;
1216 /*
1217 * We are under a CPU-wide trace scenario. As such we need to know
1218 * when the code that generated the traces started to execute so that
1219 * it can be correlated with execution on other CPUs. So we get a
1220 * handle on the beginning of traces and decode until we find a
1221 * timestamp. The timestamp is then added to the auxtrace min heap
1222 * in order to know what nibble (of all the etmqs) to decode first.
1223 */
1225 /*
1226 * Fetch an aux_buffer from this etmq. Bail if no more
1227 * blocks or an error has been encountered.
1228 */
1233 /*
1234 * Run decoder on the trace block. The decoder will stop when
1235 * encountering a CS timestamp, a full packet queue or the end of
1236 * trace for that block.
1237 */
1242 /*
1243 * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all
1244 * the timestamp calculation for us.
1245 */
1248 /* We found a timestamp, no need to continue. */
1252 /*
1253 * We didn't find a timestamp so empty all the traceid packet
1254 * queues before looking for another timestamp packet, either
1255 * in the current data block or a new one. Packets that were
1256 * just decoded are useless since no timestamp has been
1257 * associated with them. As such simply discard them.
1258 */
1260 }
1262 /*
1263 * We have a timestamp. Add it to the min heap to reflect when
1264 * instructions conveyed by the range packets of this traceID queue
1265 * started to execute. Once the same has been done for all the traceID
1266 * queues of each etmq, redenring and decoding can start in
1267 * chronological order.
1268 *
1269 * Note that packets decoded above are still in the traceID's packet
1270 * queue and will be processed in cs_etm__process_timestamped_queues().
1271 */
1274 out:
1276 }
1281 {
1286 /*
1287 * Set the number of records before early exit: ->nr is used to
1288 * determine how many branches to copy from ->entries.
1289 */
1292 /*
1293 * Early exit when there is nothing to copy.
1294 */
1298 /*
1299 * As bs_src->entries is a circular buffer, we need to copy from it in
1300 * two steps. First, copy the branches from the most recently inserted
1301 * branch ->last_branch_pos until the end of bs_src->entries buffer.
1302 */
1308 /*
1309 * If we wrapped around at least once, the branches from the beginning
1310 * of the bs_src->entries buffer and until the ->last_branch_pos element
1311 * are older valid branches: copy them over. The total number of
1312 * branches copied over will be equal to the number of branches asked by
1313 * the user in last_branch_sz.
1314 */
1319 }
1320 }
1324 {
1327 }
1331 {
1336 /*
1337 * T32 instruction size is indicated by bits[15:11] of the first
1338 * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111
1339 * denote a 32-bit instruction.
1340 */
1342 }
1345 {
1346 /*
1347 * Return 0 for packets that have no addresses so that CS_ETM_INVAL_ADDR doesn't
1348 * appear in samples.
1349 */
1355 }
1359 {
1360 /* Returns 0 for the CS_ETM_DISCONTINUITY packet */
1365 }
1368 u64 trace_chan_id,
1370 u64 offset)
1371 {
1378 offset--;
1379 }
1381 }
1383 /* Assume a 4 byte instruction size (A32/A64) */
1385 }
1389 {
1393 /*
1394 * The branches are recorded in a circular buffer in reverse
1395 * chronological order: we start recording from the last element of the
1396 * buffer down. After writing the first element of the stack, move the
1397 * insert position back to the end of the buffer.
1398 */
1407 /* No support for mispredict */
1411 /*
1412 * Increment bs->nr until reaching the number of last branches asked by
1413 * the user on the command line.
1414 */
1417 }
1421 {
1424 }
1427 static int
1429 {
1438 /* If no more data, drop the previous auxtrace_buffer and return */
1444 }
1448 /* If the aux_buffer doesn't have data associated, try to load it */
1450 /* get the file desc associated with the perf data file */
1456 }
1458 /* If valid, drop the previous buffer */
1467 }
1471 ocsd_ex_level el)
1472 {
1478 }
1480 /* Couldn't find a known thread */
1485 }
1489 {
1498 }
1501 {
1503 }
1506 u64 trace_chan_id,
1509 {
1510 /*
1511 * It's pointless to read instructions for the CS_ETM_DISCONTINUITY
1512 * packet, so directly bail out with 'insn_len' = 0.
1513 */
1517 }
1519 /*
1520 * T32 instruction size might be 32-bit or 16-bit, decide by calling
1521 * cs_etm__t32_instr_size().
1522 */
1526 /* Otherwise, A64 and A32 instruction size are always 32-bit. */
1527 else
1532 }
1535 {
1540 else
1542 }
1546 {
1552 else
1554 }
1559 {
1569 /* Set time field based on etm auxtrace config. */
1592 }
1599 ret);
1602 }
1604 /*
1605 * The cs etm packet encodes an instruction range between a branch target
1606 * and the next taken branch. Generate sample accordingly.
1607 */
1610 {
1616 u64 nr;
1617 u64 hw_idx;
1620 u64 ip;
1629 /* Set time field based on etm auxtrace config. */
1646 /*
1647 * perf report cannot handle events without a branch stack
1648 */
1656 },
1657 };
1659 }
1666 }
1673 ret);
1676 }
1680 {
1685 u64 id;
1692 }
1693 }
1698 }
1705 PERF_SAMPLE_PERIOD;
1708 else
1719 /* create new id val to be a fixed offset from evsel id */
1736 }
1740 /*
1741 * We don't use the hardware index, but the sample generation
1742 * code uses the new format branch_stack with this field,
1743 * so the event attributes must indicate that it's present.
1744 */
1746 }
1758 }
1761 }
1765 {
1769 u64 instrs_prev;
1771 /* Get instructions remainder from previous packet */
1776 /*
1777 * Record a branch when the last instruction in
1778 * PREV_PACKET is a branch.
1779 */
1787 /*
1788 * Emit instruction sample periodically
1789 * TODO: allow period to be defined in cycles and clock time
1790 */
1792 /*
1793 * Below diagram demonstrates the instruction samples
1794 * generation flows:
1795 *
1796 * Instrs Instrs Instrs Instrs
1797 * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3)
1798 * | | | |
1799 * V V V V
1800 * --------------------------------------------------
1801 * ^ ^
1802 * | |
1803 * Period Period
1804 * instructions(Pi) instructions(Pi')
1805 *
1806 * | |
1807 * \---------------- -----------------/
1808 * V
1809 * tidq->packet->instr_count
1810 *
1811 * Instrs Sample(n...) are the synthesised samples occurring
1812 * every etm->instructions_sample_period instructions - as
1813 * defined on the perf command line. Sample(n) is being the
1814 * last sample before the current etm packet, n+1 to n+3
1815 * samples are generated from the current etm packet.
1816 *
1817 * tidq->packet->instr_count represents the number of
1818 * instructions in the current etm packet.
1819 *
1820 * Period instructions (Pi) contains the number of
1821 * instructions executed after the sample point(n) from the
1822 * previous etm packet. This will always be less than
1823 * etm->instructions_sample_period.
1824 *
1825 * When generate new samples, it combines with two parts
1826 * instructions, one is the tail of the old packet and another
1827 * is the head of the new coming packet, to generate
1828 * sample(n+1); sample(n+2) and sample(n+3) consume the
1829 * instructions with sample period. After sample(n+3), the rest
1830 * instructions will be used by later packet and it is assigned
1831 * to tidq->period_instructions for next round calculation.
1832 */
1834 /*
1835 * Get the initial offset into the current packet instructions;
1836 * entry conditions ensure that instrs_prev is less than
1837 * etm->instructions_sample_period.
1838 */
1840 u64 addr;
1842 /* Prepare last branches for instruction sample */
1848 /*
1849 * Calculate the address of the sampled instruction (-1
1850 * as sample is reported as though instruction has just
1851 * been executed, but PC has not advanced to next
1852 * instruction)
1853 */
1865 }
1866 }
1871 /* Generate sample for tracing on packet */
1875 /* Generate sample for branch taken packet */
1884 }
1885 }
1890 }
1893 {
1894 /*
1895 * When the exception packet is inserted, whether the last instruction
1896 * in previous range packet is taken branch or not, we need to force
1897 * to set 'prev_packet->last_instr_taken_branch' to true. This ensures
1898 * to generate branch sample for the instruction range before the
1899 * exception is trapped to kernel or before the exception returning.
1900 *
1901 * The exception packet includes the dummy address values, so don't
1902 * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful
1903 * for generating instruction and branch samples.
1904 */
1909 }
1913 {
1917 /* Handle start tracing packet */
1924 u64 addr;
1926 /* Prepare last branches for instruction sample */
1929 /*
1930 * Generate a last branch event for the branches left in the
1931 * circular buffer at the end of the trace.
1932 *
1933 * Use the address of the end of the last reported execution
1934 * range
1935 */
1946 }
1953 }
1955 swap_packet:
1958 /* Reset last branches after flush the trace */
1963 }
1967 {
1970 /*
1971 * It has no new packet coming and 'etmq->packet' contains the stale
1972 * packet which was set at the previous time with packets swapping;
1973 * so skip to generate branch sample to avoid stale packet.
1974 *
1975 * For this case only flush branch stack and generate a last branch
1976 * event for the branches left in the circular buffer at the end of
1977 * the trace.
1978 */
1982 u64 addr;
1984 /* Prepare last branches for instruction sample */
1987 /*
1988 * Use the address of the end of the last reported execution
1989 * range.
1990 */
2000 }
2003 }
2004 /*
2005 * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue
2006 * if need be.
2007 * Returns: < 0 if error
2008 * = 0 if no more auxtrace_buffer to read
2009 * > 0 if the current buffer isn't empty yet
2010 */
2012 {
2019 /*
2020 * We cannot assume consecutive blocks in the data file
2021 * are contiguous, reset the decoder to force re-sync.
2022 */
2026 }
2029 }
2033 u64 end_addr)
2034 {
2035 /* Initialise to keep compiler happy */
2038 u64 addr;
2042 /*
2043 * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247:
2044 *
2045 * b'15 b'8
2046 * +-----------------+--------+
2047 * | 1 1 0 1 1 1 1 1 | imm8 |
2048 * +-----------------+--------+
2049 *
2050 * According to the specification, it only defines SVC for T32
2051 * with 16 bits instruction and has no definition for 32bits;
2052 * so below only read 2 bytes as instruction size for T32.
2053 */
2062 /*
2063 * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247:
2064 *
2065 * b'31 b'28 b'27 b'24
2066 * +---------+---------+-------------------------+
2067 * | !1111 | 1 1 1 1 | imm24 |
2068 * +---------+---------+-------------------------+
2069 */
2079 /*
2080 * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294:
2081 *
2082 * b'31 b'21 b'4 b'0
2083 * +-----------------------+---------+-----------+
2084 * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 |
2085 * +-----------------------+---------+-----------+
2086 */
2097 }
2100 }
2104 {
2113 /*
2114 * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and
2115 * HVC cases; need to check if it's SVC instruction based on
2116 * packet address.
2117 */
2123 }
2126 }
2129 u64 magic)
2130 {
2152 }
2156 u64 magic)
2157 {
2179 /*
2180 * For CS_ETMV4_EXC_CALL, except SVC other instructions
2181 * (SMC, HVC) are taken as sync exceptions.
2182 */
2188 /*
2189 * ETMv4 has 5 bits for exception number; if the numbers
2190 * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ]
2191 * they are implementation defined exceptions.
2192 *
2193 * For this case, simply take it as sync exception.
2194 */
2198 }
2201 }
2205 {
2209 u64 magic;
2214 /*
2215 * Immediate branch instruction without neither link nor
2216 * return flag, it's normal branch instruction within
2217 * the function.
2218 */
2225 }
2227 /*
2228 * Immediate branch instruction with link (e.g. BL), this is
2229 * branch instruction for function call.
2230 */
2234 PERF_IP_FLAG_CALL;
2236 /*
2237 * Indirect branch instruction with link (e.g. BLR), this is
2238 * branch instruction for function call.
2239 */
2243 PERF_IP_FLAG_CALL;
2245 /*
2246 * Indirect branch instruction with subtype of
2247 * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for
2248 * function return for A32/T32.
2249 */
2253 PERF_IP_FLAG_RETURN;
2255 /*
2256 * Indirect branch instruction without link (e.g. BR), usually
2257 * this is used for function return, especially for functions
2258 * within dynamic link lib.
2259 */
2263 PERF_IP_FLAG_RETURN;
2265 /* Return instruction for function return. */
2269 PERF_IP_FLAG_RETURN;
2271 /*
2272 * Decoder might insert a discontinuity in the middle of
2273 * instruction packets, fixup prev_packet with flag
2274 * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace.
2275 */
2278 PERF_IP_FLAG_TRACE_BEGIN;
2280 /*
2281 * If the previous packet is an exception return packet
2282 * and the return address just follows SVC instruction,
2283 * it needs to calibrate the previous packet sample flags
2284 * as PERF_IP_FLAG_SYSCALLRET.
2285 */
2287 PERF_IP_FLAG_RETURN |
2288 PERF_IP_FLAG_INTERRUPT) &&
2292 PERF_IP_FLAG_RETURN |
2293 PERF_IP_FLAG_SYSCALLRET;
2296 /*
2297 * The trace is discontinuous, if the previous packet is
2298 * instruction packet, set flag PERF_IP_FLAG_TRACE_END
2299 * for previous packet.
2300 */
2303 PERF_IP_FLAG_TRACE_END;
2310 /* The exception is for system call. */
2313 PERF_IP_FLAG_CALL |
2314 PERF_IP_FLAG_SYSCALLRET;
2315 /*
2316 * The exceptions are triggered by external signals from bus,
2317 * interrupt controller, debug module, PE reset or halt.
2318 */
2321 PERF_IP_FLAG_CALL |
2322 PERF_IP_FLAG_ASYNC |
2323 PERF_IP_FLAG_INTERRUPT;
2324 /*
2325 * Otherwise, exception is caused by trap, instruction &
2326 * data fault, or alignment errors.
2327 */
2330 PERF_IP_FLAG_CALL |
2331 PERF_IP_FLAG_INTERRUPT;
2333 /*
2334 * When the exception packet is inserted, since exception
2335 * packet is not used standalone for generating samples
2336 * and it's affiliation to the previous instruction range
2337 * packet; so set previous range packet flags to tell perf
2338 * it is an exception taken branch.
2339 */
2344 /*
2345 * When the exception return packet is inserted, since
2346 * exception return packet is not used standalone for
2347 * generating samples and it's affiliation to the previous
2348 * instruction range packet; so set previous range packet
2349 * flags to tell perf it is an exception return branch.
2350 *
2351 * The exception return can be for either system call or
2352 * other exception types; unfortunately the packet doesn't
2353 * contain exception type related info so we cannot decide
2354 * the exception type purely based on exception return packet.
2355 * If we record the exception number from exception packet and
2356 * reuse it for exception return packet, this is not reliable
2357 * due the trace can be discontinuity or the interrupt can
2358 * be nested, thus the recorded exception number cannot be
2359 * used for exception return packet for these two cases.
2360 *
2361 * For exception return packet, we only need to distinguish the
2362 * packet is for system call or for other types. Thus the
2363 * decision can be deferred when receive the next packet which
2364 * contains the return address, based on the return address we
2365 * can read out the previous instruction and check if it's a
2366 * system call instruction and then calibrate the sample flag
2367 * as needed.
2368 */
2371 PERF_IP_FLAG_RETURN |
2372 PERF_IP_FLAG_INTERRUPT;
2377 }
2380 }
2383 {
2387 /*
2388 * Packets are decoded and added to the decoder's packet queue
2389 * until the decoder packet processing callback has requested that
2390 * processing stops or there is nothing left in the buffer. Normal
2391 * operations that stop processing are a timestamp packet or a full
2392 * decoder buffer queue.
2393 */
2406 out:
2408 }
2412 {
2418 /* Process each packet in this chunk */
2423 /*
2424 * Stop processing this chunk on
2425 * end of data or error
2426 */
2429 /*
2430 * Since packet addresses are swapped in packet
2431 * handling within below switch() statements,
2432 * thus setting sample flags must be called
2433 * prior to switch() statement to use address
2434 * information before packets swapping.
2435 */
2442 /*
2443 * If the packet contains an instruction
2444 * range, generate instruction sequence
2445 * events.
2446 */
2451 /*
2452 * If the exception packet is coming,
2453 * make sure the previous instruction
2454 * range packet to be handled properly.
2455 */
2459 /*
2460 * Discontinuity in trace, flush
2461 * previous branch stack
2462 */
2466 /*
2467 * Should not receive empty packet,
2468 * report error.
2469 */
2474 }
2475 }
2478 }
2481 {
2491 /* Ignore return value */
2493 }
2494 }
2497 {
2505 /* Go through each buffer in the queue and decode them one by one */
2511 /* Run trace decoder until buffer consumed or end of trace */
2517 /*
2518 * Process each packet in this chunk, nothing to do if
2519 * an error occurs other than hoping the next one will
2520 * be better.
2521 */
2527 /* Flush any remaining branch stack entries */
2529 }
2532 }
2535 {
2540 /* Go through each buffer in the queue and decode them one by one */
2546 /* Run trace decoder until buffer consumed or end of trace */
2552 /*
2553 * cs_etm__run_per_thread_timeless_decoder() runs on a
2554 * single traceID queue because each TID has a separate
2555 * buffer. But here in per-cpu mode we need to iterate
2556 * over each channel instead.
2557 */
2563 }
2569 /* Flush any remaining branch stack entries */
2573 }
2574 }
2577 }
2580 pid_t tid)
2581 {
2604 }
2607 }
2610 {
2613 u8 trace_chan_id;
2614 u64 cs_timestamp;
2619 /*
2620 * Pre-populate the heap with one entry from each queue so that we can
2621 * start processing in time order across all queues.
2622 */
2631 }
2637 /* Take the entry at the top of the min heap */
2644 /*
2645 * Remove the top entry from the heap since we are about
2646 * to process it.
2647 */
2652 /*
2653 * No traceID queue has been allocated for this traceID,
2654 * which means something somewhere went very wrong. No
2655 * other choice than simply exit.
2656 */
2659 }
2661 /*
2662 * Packets associated with this timestamp are already in
2663 * the etmq's traceID queue, so process them.
2664 */
2669 /*
2670 * Packets for this timestamp have been processed, time to
2671 * move on to the next timestamp, fetching a new auxtrace_buffer
2672 * if need be.
2673 */
2674 refetch:
2679 /*
2680 * No more auxtrace_buffers to process in this etmq, simply
2681 * move on to another entry in the auxtrace_heap.
2682 */
2693 /*
2694 * Function cs_etm__decode_data_block() returns when
2695 * there is no more traces to decode in the current
2696 * auxtrace_buffer OR when a timestamp has been
2697 * encountered on any of the traceID queues. Since we
2698 * did not get a timestamp, there is no more traces to
2699 * process in this auxtrace_buffer. As such empty and
2700 * flush all traceID queues.
2701 */
2704 /* Fetch another auxtrace_buffer for this etmq */
2706 }
2708 /*
2709 * Add to the min heap the timestamp for packets that have
2710 * just been decoded. They will be processed and synthesized
2711 * during the next call to cs_etm__process_traceid_queue() for
2712 * this queue/traceID.
2713 */
2716 }
2728 /* Flush any remaining branch stack entries */
2733 }
2734 }
2735 out:
2737 }
2741 {
2747 /*
2748 * Add the tid/pid to the log so that we can get a match when we get a
2749 * contextID from the decoder. Only track for the host: only kernel
2750 * trace is supported for guests which wouldn't need pids so this should
2751 * be fine.
2752 */
2762 }
2766 {
2770 /*
2771 * Context switch in per-thread mode are irrelevant since perf
2772 * will start/stop tracing as the process is scheduled.
2773 */
2777 /*
2778 * SWITCH_IN events carry the next process to be switched out while
2779 * SWITCH_OUT events carry the process to be switched in. As such
2780 * we don't care about IN events.
2781 */
2785 /*
2786 * Add the tid/pid to the log so that we can get a match when we get a
2787 * contextID from the decoder. Only track for the host: only kernel
2788 * trace is supported for guests which wouldn't need pids so this should
2789 * be fine.
2790 */
2800 }
2806 {
2809 auxtrace);
2817 }
2821 /*
2822 * Don't need to wait for cs_etm__flush_events() in per-thread mode to
2823 * start the decode because we know there will be no more trace from
2824 * this thread. All this does is emit samples earlier than waiting for
2825 * the flush in other modes, but with timestamps it makes sense to wait
2826 * for flush so that events from different threads are interleaved
2827 * properly.
2828 */
2841 /*
2842 * Record the latest kernel timestamp available in the header
2843 * for samples so that synthesised samples occur from this point
2844 * onwards.
2845 */
2852 }
2855 }
2859 {
2862 /*
2863 * Find all buffers with same reference in the queues and dump them.
2864 * This is because the queues can contain multiple entries of the same
2865 * buffer that were split on aux records.
2866 */
2871 }
2876 {
2879 auxtrace);
2882 off_t data_offset;
2894 }
2905 }
2910 }
2913 {
2917 /* Override timeless mode with user input from --itrace=Z */
2921 }
2923 /*
2924 * Find the cs_etm evsel and look at what its timestamp setting was
2925 */
2931 }
2935 }
2937 /*
2938 * Read a single cpu parameter block from the auxtrace_info priv block.
2939 *
2940 * For version 1 there is a per cpu nr_params entry. If we are handling
2941 * version 1 file, then there may be less, the same, or more params
2942 * indicated by this value than the compile time number we understand.
2943 *
2944 * For a version 0 info block, there are a fixed number, and we need to
2945 * fill out the nr_param value in the metadata we create.
2946 */
2949 {
2959 /* read block current index & version */
2964 /* read version 0 info block into a version 1 metadata block */
2969 /* remaining block params at offset +1 from source */
2972 /* version 0 has 2 common params */
2975 /* read version 1 info block - input and output nr_params may differ */
2976 /* version 1 has 3 common params */
2980 /* if input has more params than output - skip excess */
2988 /* record the actual nr params we copied */
2990 }
2992 /* adjust in offset by number of in params used */
2996 }
2998 /**
2999 * Puts a fragment of an auxtrace buffer into the auxtrace queues based
3000 * on the bounds of aux_event, if it matches with the buffer that's at
3001 * file_offset.
3002 *
3003 * Normally, whole auxtrace buffers would be added to the queue. But we
3004 * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder
3005 * is reset across each buffer, so splitting the buffers up in advance has
3006 * the same effect.
3007 */
3008 static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz,
3010 {
3021 auxtrace);
3023 /*
3024 * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got
3025 * from looping through the auxtrace index.
3026 */
3038 }
3040 /*
3041 * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See
3042 * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a
3043 * CPU as we set this always for the AUX_OUTPUT_HW_ID event.
3044 * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1.
3045 * Return 'not found' if mismatch.
3046 */
3053 /*
3054 * Found a per-cpu buffer after a per-thread one was
3055 * already found
3056 */
3059 }
3061 }
3064 /*
3065 * Clamp size in snapshot mode. The buffer size is clamped in
3066 * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect
3067 * the buffer size.
3068 */
3071 /*
3072 * In this mode, the head also points to the end of the buffer so aux_offset
3073 * needs to have the size subtracted so it points to the beginning as in normal mode
3074 */
3079 }
3085 /*
3086 * If this AUX event was inside this buffer somewhere, create a new auxtrace event
3087 * based on the sizes of the aux event, and queue that fragment.
3088 */
3106 }
3109 }
3111 /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */
3113 }
3117 {
3118 /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */
3122 }
3124 }
3128 {
3136 /* Don't care about any other events, we're only queuing buffers for AUX events */
3143 /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */
3147 /*
3148 * Parse the sample, we need the sample_id_all data that comes after the event so that the
3149 * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID.
3150 */
3158 /*
3159 * Loop through the auxtrace index to find the buffer that matches up with this aux event.
3160 */
3166 /*
3167 * Stop search on error or successful values. Continue search on
3168 * 1 ('not found')
3169 */
3172 }
3173 }
3175 /*
3176 * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but
3177 * don't exit with an error because it will still be possible to decode other aux records.
3178 */
3182 }
3185 {
3193 /*
3194 * We would get here if there are no entries in the index (either no auxtrace
3195 * buffers or no index at all). Fail silently as there is the possibility of
3196 * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still
3197 * false.
3198 *
3199 * In that scenario, buffers will not be split by AUX records.
3200 */
3202 }
3204 #define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \
3205 (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1))
3207 /*
3208 * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual
3209 * timestamps).
3210 */
3212 {
3226 /* Unknown / unsupported magic number. */
3228 }
3229 }
3231 }
3233 /* map trace ids to correct metadata block, from information in metadata */
3236 {
3237 u64 cs_etm_magic;
3238 u8 trace_chan_id;
3254 /* unknown magic number */
3256 }
3260 }
3262 }
3264 /*
3265 * Use the data gathered by the peeks for HW_ID (trace ID mappings) and AUX
3266 * (formatted or not) packets to create the decoders.
3267 */
3269 {
3277 /*
3278 * Each queue can only contain data from one CPU when unformatted, so only one decoder is
3279 * needed.
3280 */
3284 /* Use metadata to fill in trace parameters for trace decoder */
3293 /* Set decoder parameters to decode trace packets */
3296 CS_ETM_OPERATION_DECODE))
3300 t_params);
3305 /*
3306 * Register a function to handle all memory accesses required by
3307 * the trace decoder library.
3308 */
3311 cs_etm__mem_access))
3317 out_free_decoder:
3319 out_free:
3322 }
3325 {
3333 /*
3334 * Don't create decoders for empty queues, mainly because
3335 * etmq->format is unknown for empty queues.
3336 */
3344 }
3346 }
3350 {
3364 /* First the global part */
3371 /* Start parsing after the common part of the header */
3374 /*
3375 * The metadata is stored in the auxtrace_info section and encodes
3376 * the configuration of the ARM embedded trace macrocell which is
3377 * required by the trace decoder to properly decode the trace due
3378 * to its highly compressed nature.
3379 */
3384 CS_ETM_PRIV_MAX,
3385 CS_ETM_NR_TRC_PARAMS_V0);
3389 CS_ETMV4_PRIV_MAX,
3390 CS_ETMV4_NR_TRC_PARAMS_V0);
3394 ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n",
3398 }
3403 }
3407 }
3409 /*
3410 * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and
3411 * CS_ETMV4_PRIV_MAX mark how many double words are in the
3412 * global metadata, and each cpu's metadata respectively.
3413 * The following tests if the correct number of double words was
3414 * present in the auxtrace info section.
3415 */
3420 }
3427 }
3429 /*
3430 * As all the ETMs run at the same exception level, the system should
3431 * have the same PID format crossing CPUs. So cache the PID format
3432 * and reuse it for sequential decoding.
3433 */
3444 }
3452 }
3463 /*
3464 * Prior to Armv8.4, Arm CPUs don't support FEAT_TRF feature,
3465 * therefore the decoder cannot know if the timestamp trace is
3466 * same with the kernel time.
3467 *
3468 * If a user has knowledge for the working platform and can
3469 * specify itrace option 'T' to tell decoder to forcely use the
3470 * traced timestamp as the kernel time.
3471 */
3473 else
3474 /* Use virtual timestamps if all ETMs report ts_source = 1 */
3504 }
3513 /*
3514 * Map Trace ID values to CPU metadata.
3515 *
3516 * Trace metadata will always contain Trace ID values from the legacy algorithm
3517 * in case it's read by a version of Perf that doesn't know about HW_ID packets
3518 * or the kernel doesn't emit them.
3519 *
3520 * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use
3521 * the same IDs as the old algorithm as far as is possible, unless there are clashes
3522 * in which case a different value will be used. This means an older perf may still
3523 * be able to record and read files generate on a newer system.
3524 *
3525 * For a perf able to interpret AUX_HW_ID packets we first check for the presence of
3526 * those packets. If they are there then the values will be mapped and plugged into
3527 * the metadata and decoders are only created for each mapping received.
3528 *
3529 * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel
3530 * then we map Trace ID values to CPU directly from the metadata and create decoders
3531 * for all mappings.
3532 */
3534 /* Scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */
3542 /* if no HW ID found this is a file with metadata values only, map from metadata */
3547 }
3556 err_free_queues:
3559 err_free_etm:
3561 err_free_metadata:
3562 /* No need to check @metadata[j], free(NULL) is supported */
3567 }