1 // SPDX-License-Identifier: GPL-2.0
2 #include <sys/sysmacros.h>
14 #include <linux/stringify.h>
28 #include "../builtin.h"
30 #include "sane_ctype.h"
33 struct perf_data
*output
;
34 struct perf_session
*session
;
35 struct machine
*machine
;
36 union jr_entry
*entry
;
41 bool needs_bswap
; /* handles cross-endianess */
42 bool use_arch_timestamp
;
45 uint64_t unwinding_size
;
46 uint64_t unwinding_mapped_size
;
47 uint64_t eh_frame_hdr_size
;
48 size_t nr_debug_entries
;
49 uint32_t code_load_count
;
51 struct rb_root code_root
;
55 struct debug_line_info
{
58 /* The filename format is unspecified, absolute path, relative etc. */
59 char const filename
[0];
63 struct perf_tool tool
;
64 struct perf_data output
;
65 struct perf_data input
;
69 #define hmax(a, b) ((a) > (b) ? (a) : (b))
70 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
73 jit_emit_elf(char *filename
,
81 uint32_t unwinding_header_size
,
82 uint32_t unwinding_size
)
87 fprintf(stderr
, "write ELF image %s\n", filename
);
89 fd
= open(filename
, O_CREAT
|O_TRUNC
|O_WRONLY
, 0644);
91 pr_warning("cannot create jit ELF %s: %s\n", filename
, strerror(errno
));
95 ret
= jit_write_elf(fd
, code_addr
, sym
, (const void *)code
, csize
, debug
, nr_debug_entries
,
96 unwinding
, unwinding_header_size
, unwinding_size
);
107 jit_close(struct jit_buf_desc
*jd
)
117 jit_validate_events(struct perf_session
*session
)
119 struct perf_evsel
*evsel
;
122 * check that all events use CLOCK_MONOTONIC
124 evlist__for_each_entry(session
->evlist
, evsel
) {
125 if (evsel
->attr
.use_clockid
== 0 || evsel
->attr
.clockid
!= CLOCK_MONOTONIC
)
132 jit_open(struct jit_buf_desc
*jd
, const char *name
)
134 struct jitheader header
;
135 struct jr_prefix
*prefix
;
137 void *n
, *buf
= NULL
;
138 int ret
, retval
= -1;
140 jd
->in
= fopen(name
, "r");
144 bsz
= hmax(sizeof(header
), sizeof(*prefix
));
151 * protect from writer modifying the file while we are reading it
155 ret
= fread(buf
, sizeof(header
), 1, jd
->in
);
159 memcpy(&header
, buf
, sizeof(header
));
161 if (header
.magic
!= JITHEADER_MAGIC
) {
162 if (header
.magic
!= JITHEADER_MAGIC_SW
)
164 jd
->needs_bswap
= true;
167 if (jd
->needs_bswap
) {
168 header
.version
= bswap_32(header
.version
);
169 header
.total_size
= bswap_32(header
.total_size
);
170 header
.pid
= bswap_32(header
.pid
);
171 header
.elf_mach
= bswap_32(header
.elf_mach
);
172 header
.timestamp
= bswap_64(header
.timestamp
);
173 header
.flags
= bswap_64(header
.flags
);
176 jd
->use_arch_timestamp
= header
.flags
& JITDUMP_FLAGS_ARCH_TIMESTAMP
;
179 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
182 (unsigned long long)header
.timestamp
,
185 jd
->use_arch_timestamp
);
187 if (header
.version
> JITHEADER_VERSION
) {
188 pr_err("wrong jitdump version %u, expected " __stringify(JITHEADER_VERSION
),
193 if (header
.flags
& JITDUMP_FLAGS_RESERVED
) {
194 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
195 (unsigned long long)header
.flags
& JITDUMP_FLAGS_RESERVED
);
199 if (jd
->use_arch_timestamp
&& !jd
->session
->time_conv
.time_mult
) {
200 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
205 * validate event is using the correct clockid
207 if (!jd
->use_arch_timestamp
&& jit_validate_events(jd
->session
)) {
208 pr_err("error, jitted code must be sampled with perf record -k 1\n");
212 bs
= header
.total_size
- sizeof(header
);
215 n
= realloc(buf
, bs
);
220 /* read extra we do not know about */
221 ret
= fread(buf
, bs
- bsz
, 1, jd
->in
);
226 * keep dirname for generating files and mmap records
228 strcpy(jd
->dir
, name
);
238 static union jr_entry
*
239 jit_get_next_entry(struct jit_buf_desc
*jd
)
241 struct jr_prefix
*prefix
;
250 if (jd
->buf
== NULL
) {
251 size_t sz
= getpagesize();
252 if (sz
< sizeof(*prefix
))
253 sz
= sizeof(*prefix
);
255 jd
->buf
= malloc(sz
);
265 * file is still locked at this point
267 ret
= fread(prefix
, sizeof(*prefix
), 1, jd
->in
);
271 if (jd
->needs_bswap
) {
272 prefix
->id
= bswap_32(prefix
->id
);
273 prefix
->total_size
= bswap_32(prefix
->total_size
);
274 prefix
->timestamp
= bswap_64(prefix
->timestamp
);
277 size
= prefix
->total_size
;
280 if (bs
< sizeof(*prefix
))
283 if (id
>= JIT_CODE_MAX
) {
284 pr_warning("next_entry: unknown record type %d, skipping\n", id
);
286 if (bs
> jd
->bufsize
) {
288 n
= realloc(jd
->buf
, bs
);
295 addr
= ((void *)jd
->buf
) + sizeof(*prefix
);
297 ret
= fread(addr
, bs
- sizeof(*prefix
), 1, jd
->in
);
301 jr
= (union jr_entry
*)jd
->buf
;
304 case JIT_CODE_DEBUG_INFO
:
305 if (jd
->needs_bswap
) {
307 jr
->info
.code_addr
= bswap_64(jr
->info
.code_addr
);
308 jr
->info
.nr_entry
= bswap_64(jr
->info
.nr_entry
);
309 for (n
= 0 ; n
< jr
->info
.nr_entry
; n
++) {
310 jr
->info
.entries
[n
].addr
= bswap_64(jr
->info
.entries
[n
].addr
);
311 jr
->info
.entries
[n
].lineno
= bswap_32(jr
->info
.entries
[n
].lineno
);
312 jr
->info
.entries
[n
].discrim
= bswap_32(jr
->info
.entries
[n
].discrim
);
316 case JIT_CODE_UNWINDING_INFO
:
317 if (jd
->needs_bswap
) {
318 jr
->unwinding
.unwinding_size
= bswap_64(jr
->unwinding
.unwinding_size
);
319 jr
->unwinding
.eh_frame_hdr_size
= bswap_64(jr
->unwinding
.eh_frame_hdr_size
);
320 jr
->unwinding
.mapped_size
= bswap_64(jr
->unwinding
.mapped_size
);
326 if (jd
->needs_bswap
) {
327 jr
->load
.pid
= bswap_32(jr
->load
.pid
);
328 jr
->load
.tid
= bswap_32(jr
->load
.tid
);
329 jr
->load
.vma
= bswap_64(jr
->load
.vma
);
330 jr
->load
.code_addr
= bswap_64(jr
->load
.code_addr
);
331 jr
->load
.code_size
= bswap_64(jr
->load
.code_size
);
332 jr
->load
.code_index
= bswap_64(jr
->load
.code_index
);
334 jd
->code_load_count
++;
337 if (jd
->needs_bswap
) {
338 jr
->move
.pid
= bswap_32(jr
->move
.pid
);
339 jr
->move
.tid
= bswap_32(jr
->move
.tid
);
340 jr
->move
.vma
= bswap_64(jr
->move
.vma
);
341 jr
->move
.old_code_addr
= bswap_64(jr
->move
.old_code_addr
);
342 jr
->move
.new_code_addr
= bswap_64(jr
->move
.new_code_addr
);
343 jr
->move
.code_size
= bswap_64(jr
->move
.code_size
);
344 jr
->move
.code_index
= bswap_64(jr
->move
.code_index
);
349 /* skip unknown record (we have read them) */
356 jit_inject_event(struct jit_buf_desc
*jd
, union perf_event
*event
)
360 size
= perf_data__write(jd
->output
, event
, event
->header
.size
);
364 jd
->bytes_written
+= size
;
368 static uint64_t convert_timestamp(struct jit_buf_desc
*jd
, uint64_t timestamp
)
370 struct perf_tsc_conversion tc
;
372 if (!jd
->use_arch_timestamp
)
375 tc
.time_shift
= jd
->session
->time_conv
.time_shift
;
376 tc
.time_mult
= jd
->session
->time_conv
.time_mult
;
377 tc
.time_zero
= jd
->session
->time_conv
.time_zero
;
382 return tsc_to_perf_time(timestamp
, &tc
);
385 static int jit_repipe_code_load(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
387 struct perf_sample sample
;
388 union perf_event
*event
;
389 struct perf_tool
*tool
= jd
->session
->tool
;
398 int ret
, csize
, usize
;
407 csize
= jr
->load
.code_size
;
408 usize
= jd
->unwinding_mapped_size
;
409 addr
= jr
->load
.code_addr
;
410 sym
= (void *)((unsigned long)jr
+ sizeof(jr
->load
));
411 code
= (unsigned long)jr
+ jr
->load
.p
.total_size
- csize
;
412 count
= jr
->load
.code_index
;
413 idr_size
= jd
->machine
->id_hdr_size
;
415 event
= calloc(1, sizeof(*event
) + idr_size
);
419 filename
= event
->mmap2
.filename
;
420 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%u.so",
427 size
= PERF_ALIGN(size
, sizeof(u64
));
428 uaddr
= (uintptr_t)code
;
429 ret
= jit_emit_elf(filename
, sym
, addr
, (const void *)uaddr
, csize
, jd
->debug_data
, jd
->nr_debug_entries
,
430 jd
->unwinding_data
, jd
->eh_frame_hdr_size
, jd
->unwinding_size
);
432 if (jd
->debug_data
&& jd
->nr_debug_entries
) {
433 free(jd
->debug_data
);
434 jd
->debug_data
= NULL
;
435 jd
->nr_debug_entries
= 0;
438 if (jd
->unwinding_data
&& jd
->eh_frame_hdr_size
) {
439 free(jd
->unwinding_data
);
440 jd
->unwinding_data
= NULL
;
441 jd
->eh_frame_hdr_size
= 0;
442 jd
->unwinding_mapped_size
= 0;
443 jd
->unwinding_size
= 0;
450 if (stat(filename
, &st
))
451 memset(&st
, 0, sizeof(st
));
453 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
454 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
455 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
456 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
458 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
459 event
->mmap2
.start
= addr
;
460 event
->mmap2
.len
= usize
? ALIGN_8(csize
) + usize
: csize
;
461 event
->mmap2
.pid
= pid
;
462 event
->mmap2
.tid
= tid
;
463 event
->mmap2
.ino
= st
.st_ino
;
464 event
->mmap2
.maj
= major(st
.st_dev
);
465 event
->mmap2
.min
= minor(st
.st_dev
);
466 event
->mmap2
.prot
= st
.st_mode
;
467 event
->mmap2
.flags
= MAP_SHARED
;
468 event
->mmap2
.ino_generation
= 1;
470 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
471 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
475 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
476 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
479 * create pseudo sample to induce dso hit increment
480 * use first address as sample address
482 memset(&sample
, 0, sizeof(sample
));
483 sample
.cpumode
= PERF_RECORD_MISC_USER
;
486 sample
.time
= id
->time
;
489 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
493 ret
= jit_inject_event(jd
, event
);
495 * mark dso as use to generate buildid in the header
498 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
503 static int jit_repipe_code_move(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
505 struct perf_sample sample
;
506 union perf_event
*event
;
507 struct perf_tool
*tool
= jd
->session
->tool
;
522 usize
= jd
->unwinding_mapped_size
;
523 idr_size
= jd
->machine
->id_hdr_size
;
526 * +16 to account for sample_id_all (hack)
528 event
= calloc(1, sizeof(*event
) + 16);
532 filename
= event
->mmap2
.filename
;
533 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%"PRIu64
,
536 jr
->move
.code_index
);
540 if (stat(filename
, &st
))
541 memset(&st
, 0, sizeof(st
));
543 size
= PERF_ALIGN(size
, sizeof(u64
));
545 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
546 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
547 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
548 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
549 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
550 event
->mmap2
.start
= jr
->move
.new_code_addr
;
551 event
->mmap2
.len
= usize
? ALIGN_8(jr
->move
.code_size
) + usize
552 : jr
->move
.code_size
;
553 event
->mmap2
.pid
= pid
;
554 event
->mmap2
.tid
= tid
;
555 event
->mmap2
.ino
= st
.st_ino
;
556 event
->mmap2
.maj
= major(st
.st_dev
);
557 event
->mmap2
.min
= minor(st
.st_dev
);
558 event
->mmap2
.prot
= st
.st_mode
;
559 event
->mmap2
.flags
= MAP_SHARED
;
560 event
->mmap2
.ino_generation
= 1;
562 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
563 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
567 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
568 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
571 * create pseudo sample to induce dso hit increment
572 * use first address as sample address
574 memset(&sample
, 0, sizeof(sample
));
575 sample
.cpumode
= PERF_RECORD_MISC_USER
;
578 sample
.time
= id
->time
;
579 sample
.ip
= jr
->move
.new_code_addr
;
581 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
585 ret
= jit_inject_event(jd
, event
);
587 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
592 static int jit_repipe_debug_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
600 sz
= jr
->prefix
.total_size
- sizeof(jr
->info
);
605 memcpy(data
, &jr
->info
.entries
, sz
);
607 jd
->debug_data
= data
;
610 * we must use nr_entry instead of size here because
611 * we cannot distinguish actual entry from padding otherwise
613 jd
->nr_debug_entries
= jr
->info
.nr_entry
;
619 jit_repipe_unwinding_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
621 void *unwinding_data
;
622 uint32_t unwinding_data_size
;
627 unwinding_data_size
= jr
->prefix
.total_size
- sizeof(jr
->unwinding
);
628 unwinding_data
= malloc(unwinding_data_size
);
632 memcpy(unwinding_data
, &jr
->unwinding
.unwinding_data
,
633 unwinding_data_size
);
635 jd
->eh_frame_hdr_size
= jr
->unwinding
.eh_frame_hdr_size
;
636 jd
->unwinding_size
= jr
->unwinding
.unwinding_size
;
637 jd
->unwinding_mapped_size
= jr
->unwinding
.mapped_size
;
638 jd
->unwinding_data
= unwinding_data
;
644 jit_process_dump(struct jit_buf_desc
*jd
)
649 while ((jr
= jit_get_next_entry(jd
))) {
650 switch(jr
->prefix
.id
) {
652 ret
= jit_repipe_code_load(jd
, jr
);
655 ret
= jit_repipe_code_move(jd
, jr
);
657 case JIT_CODE_DEBUG_INFO
:
658 ret
= jit_repipe_debug_info(jd
, jr
);
660 case JIT_CODE_UNWINDING_INFO
:
661 ret
= jit_repipe_unwinding_info(jd
, jr
);
672 jit_inject(struct jit_buf_desc
*jd
, char *path
)
677 fprintf(stderr
, "injecting: %s\n", path
);
679 ret
= jit_open(jd
, path
);
683 ret
= jit_process_dump(jd
);
688 fprintf(stderr
, "injected: %s (%d)\n", path
, ret
);
694 * File must be with pattern .../jit-XXXX.dump
695 * where XXXX is the PID of the process which did the mmap()
696 * as captured in the RECORD_MMAP record
699 jit_detect(char *mmap_name
, pid_t pid
)
706 fprintf(stderr
, "jit marker trying : %s\n", mmap_name
);
710 p
= strrchr(mmap_name
, '/');
717 if (strncmp(p
, "/jit-", 5))
726 * must be followed by a pid
731 pid2
= (int)strtol(p
, &end
, 10);
736 * pid does not match mmap pid
737 * pid==0 in system-wide mode (synthesized)
739 if (pid
&& pid2
!= pid
)
744 if (strcmp(end
, ".dump"))
748 fprintf(stderr
, "jit marker found: %s\n", mmap_name
);
754 jit_process(struct perf_session
*session
,
755 struct perf_data
*output
,
756 struct machine
*machine
,
761 struct perf_evsel
*first
;
762 struct jit_buf_desc jd
;
766 * first, detect marker mmap (i.e., the jitdump mmap)
768 if (jit_detect(filename
, pid
))
771 memset(&jd
, 0, sizeof(jd
));
773 jd
.session
= session
;
775 jd
.machine
= machine
;
778 * track sample_type to compute id_all layout
779 * perf sets the same sample type to all events as of now
781 first
= perf_evlist__first(session
->evlist
);
782 jd
.sample_type
= first
->attr
.sample_type
;
786 ret
= jit_inject(&jd
, filename
);
788 *nbytes
= jd
.bytes_written
;