1 // SPDX-License-Identifier: GPL-2.0
2 #include <sys/sysmacros.h>
15 #include <linux/stringify.h>
29 #include "../builtin.h"
31 #include "sane_ctype.h"
34 struct perf_data
*output
;
35 struct perf_session
*session
;
36 struct machine
*machine
;
37 union jr_entry
*entry
;
42 bool needs_bswap
; /* handles cross-endianness */
43 bool use_arch_timestamp
;
46 uint64_t unwinding_size
;
47 uint64_t unwinding_mapped_size
;
48 uint64_t eh_frame_hdr_size
;
49 size_t nr_debug_entries
;
50 uint32_t code_load_count
;
52 struct rb_root code_root
;
56 struct debug_line_info
{
59 /* The filename format is unspecified, absolute path, relative etc. */
60 char const filename
[0];
64 struct perf_tool tool
;
65 struct perf_data output
;
66 struct perf_data input
;
70 #define hmax(a, b) ((a) > (b) ? (a) : (b))
71 #define get_jit_tool(t) (container_of(tool, struct jit_tool, tool))
74 jit_emit_elf(char *filename
,
82 uint32_t unwinding_header_size
,
83 uint32_t unwinding_size
)
88 fprintf(stderr
, "write ELF image %s\n", filename
);
90 fd
= open(filename
, O_CREAT
|O_TRUNC
|O_WRONLY
, 0644);
92 pr_warning("cannot create jit ELF %s: %s\n", filename
, strerror(errno
));
96 ret
= jit_write_elf(fd
, code_addr
, sym
, (const void *)code
, csize
, debug
, nr_debug_entries
,
97 unwinding
, unwinding_header_size
, unwinding_size
);
108 jit_close(struct jit_buf_desc
*jd
)
118 jit_validate_events(struct perf_session
*session
)
120 struct perf_evsel
*evsel
;
123 * check that all events use CLOCK_MONOTONIC
125 evlist__for_each_entry(session
->evlist
, evsel
) {
126 if (evsel
->attr
.use_clockid
== 0 || evsel
->attr
.clockid
!= CLOCK_MONOTONIC
)
133 jit_open(struct jit_buf_desc
*jd
, const char *name
)
135 struct jitheader header
;
136 struct jr_prefix
*prefix
;
138 void *n
, *buf
= NULL
;
139 int ret
, retval
= -1;
141 jd
->in
= fopen(name
, "r");
145 bsz
= hmax(sizeof(header
), sizeof(*prefix
));
152 * protect from writer modifying the file while we are reading it
156 ret
= fread(buf
, sizeof(header
), 1, jd
->in
);
160 memcpy(&header
, buf
, sizeof(header
));
162 if (header
.magic
!= JITHEADER_MAGIC
) {
163 if (header
.magic
!= JITHEADER_MAGIC_SW
)
165 jd
->needs_bswap
= true;
168 if (jd
->needs_bswap
) {
169 header
.version
= bswap_32(header
.version
);
170 header
.total_size
= bswap_32(header
.total_size
);
171 header
.pid
= bswap_32(header
.pid
);
172 header
.elf_mach
= bswap_32(header
.elf_mach
);
173 header
.timestamp
= bswap_64(header
.timestamp
);
174 header
.flags
= bswap_64(header
.flags
);
177 jd
->use_arch_timestamp
= header
.flags
& JITDUMP_FLAGS_ARCH_TIMESTAMP
;
180 pr_debug("version=%u\nhdr.size=%u\nts=0x%llx\npid=%d\nelf_mach=%d\nuse_arch_timestamp=%d\n",
183 (unsigned long long)header
.timestamp
,
186 jd
->use_arch_timestamp
);
188 if (header
.version
> JITHEADER_VERSION
) {
189 pr_err("wrong jitdump version %u, expected " __stringify(JITHEADER_VERSION
),
194 if (header
.flags
& JITDUMP_FLAGS_RESERVED
) {
195 pr_err("jitdump file contains invalid or unsupported flags 0x%llx\n",
196 (unsigned long long)header
.flags
& JITDUMP_FLAGS_RESERVED
);
200 if (jd
->use_arch_timestamp
&& !jd
->session
->time_conv
.time_mult
) {
201 pr_err("jitdump file uses arch timestamps but there is no timestamp conversion\n");
206 * validate event is using the correct clockid
208 if (!jd
->use_arch_timestamp
&& jit_validate_events(jd
->session
)) {
209 pr_err("error, jitted code must be sampled with perf record -k 1\n");
213 bs
= header
.total_size
- sizeof(header
);
216 n
= realloc(buf
, bs
);
221 /* read extra we do not know about */
222 ret
= fread(buf
, bs
- bsz
, 1, jd
->in
);
227 * keep dirname for generating files and mmap records
229 strcpy(jd
->dir
, name
);
239 static union jr_entry
*
240 jit_get_next_entry(struct jit_buf_desc
*jd
)
242 struct jr_prefix
*prefix
;
251 if (jd
->buf
== NULL
) {
252 size_t sz
= getpagesize();
253 if (sz
< sizeof(*prefix
))
254 sz
= sizeof(*prefix
);
256 jd
->buf
= malloc(sz
);
266 * file is still locked at this point
268 ret
= fread(prefix
, sizeof(*prefix
), 1, jd
->in
);
272 if (jd
->needs_bswap
) {
273 prefix
->id
= bswap_32(prefix
->id
);
274 prefix
->total_size
= bswap_32(prefix
->total_size
);
275 prefix
->timestamp
= bswap_64(prefix
->timestamp
);
278 size
= prefix
->total_size
;
281 if (bs
< sizeof(*prefix
))
284 if (id
>= JIT_CODE_MAX
) {
285 pr_warning("next_entry: unknown record type %d, skipping\n", id
);
287 if (bs
> jd
->bufsize
) {
289 n
= realloc(jd
->buf
, bs
);
296 addr
= ((void *)jd
->buf
) + sizeof(*prefix
);
298 ret
= fread(addr
, bs
- sizeof(*prefix
), 1, jd
->in
);
302 jr
= (union jr_entry
*)jd
->buf
;
305 case JIT_CODE_DEBUG_INFO
:
306 if (jd
->needs_bswap
) {
308 jr
->info
.code_addr
= bswap_64(jr
->info
.code_addr
);
309 jr
->info
.nr_entry
= bswap_64(jr
->info
.nr_entry
);
310 for (n
= 0 ; n
< jr
->info
.nr_entry
; n
++) {
311 jr
->info
.entries
[n
].addr
= bswap_64(jr
->info
.entries
[n
].addr
);
312 jr
->info
.entries
[n
].lineno
= bswap_32(jr
->info
.entries
[n
].lineno
);
313 jr
->info
.entries
[n
].discrim
= bswap_32(jr
->info
.entries
[n
].discrim
);
317 case JIT_CODE_UNWINDING_INFO
:
318 if (jd
->needs_bswap
) {
319 jr
->unwinding
.unwinding_size
= bswap_64(jr
->unwinding
.unwinding_size
);
320 jr
->unwinding
.eh_frame_hdr_size
= bswap_64(jr
->unwinding
.eh_frame_hdr_size
);
321 jr
->unwinding
.mapped_size
= bswap_64(jr
->unwinding
.mapped_size
);
327 if (jd
->needs_bswap
) {
328 jr
->load
.pid
= bswap_32(jr
->load
.pid
);
329 jr
->load
.tid
= bswap_32(jr
->load
.tid
);
330 jr
->load
.vma
= bswap_64(jr
->load
.vma
);
331 jr
->load
.code_addr
= bswap_64(jr
->load
.code_addr
);
332 jr
->load
.code_size
= bswap_64(jr
->load
.code_size
);
333 jr
->load
.code_index
= bswap_64(jr
->load
.code_index
);
335 jd
->code_load_count
++;
338 if (jd
->needs_bswap
) {
339 jr
->move
.pid
= bswap_32(jr
->move
.pid
);
340 jr
->move
.tid
= bswap_32(jr
->move
.tid
);
341 jr
->move
.vma
= bswap_64(jr
->move
.vma
);
342 jr
->move
.old_code_addr
= bswap_64(jr
->move
.old_code_addr
);
343 jr
->move
.new_code_addr
= bswap_64(jr
->move
.new_code_addr
);
344 jr
->move
.code_size
= bswap_64(jr
->move
.code_size
);
345 jr
->move
.code_index
= bswap_64(jr
->move
.code_index
);
350 /* skip unknown record (we have read them) */
357 jit_inject_event(struct jit_buf_desc
*jd
, union perf_event
*event
)
361 size
= perf_data__write(jd
->output
, event
, event
->header
.size
);
365 jd
->bytes_written
+= size
;
369 static uint64_t convert_timestamp(struct jit_buf_desc
*jd
, uint64_t timestamp
)
371 struct perf_tsc_conversion tc
;
373 if (!jd
->use_arch_timestamp
)
376 tc
.time_shift
= jd
->session
->time_conv
.time_shift
;
377 tc
.time_mult
= jd
->session
->time_conv
.time_mult
;
378 tc
.time_zero
= jd
->session
->time_conv
.time_zero
;
383 return tsc_to_perf_time(timestamp
, &tc
);
386 static int jit_repipe_code_load(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
388 struct perf_sample sample
;
389 union perf_event
*event
;
390 struct perf_tool
*tool
= jd
->session
->tool
;
399 int ret
, csize
, usize
;
408 csize
= jr
->load
.code_size
;
409 usize
= jd
->unwinding_mapped_size
;
410 addr
= jr
->load
.code_addr
;
411 sym
= (void *)((unsigned long)jr
+ sizeof(jr
->load
));
412 code
= (unsigned long)jr
+ jr
->load
.p
.total_size
- csize
;
413 count
= jr
->load
.code_index
;
414 idr_size
= jd
->machine
->id_hdr_size
;
416 event
= calloc(1, sizeof(*event
) + idr_size
);
420 filename
= event
->mmap2
.filename
;
421 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%u.so",
428 size
= PERF_ALIGN(size
, sizeof(u64
));
429 uaddr
= (uintptr_t)code
;
430 ret
= jit_emit_elf(filename
, sym
, addr
, (const void *)uaddr
, csize
, jd
->debug_data
, jd
->nr_debug_entries
,
431 jd
->unwinding_data
, jd
->eh_frame_hdr_size
, jd
->unwinding_size
);
433 if (jd
->debug_data
&& jd
->nr_debug_entries
) {
434 free(jd
->debug_data
);
435 jd
->debug_data
= NULL
;
436 jd
->nr_debug_entries
= 0;
439 if (jd
->unwinding_data
&& jd
->eh_frame_hdr_size
) {
440 free(jd
->unwinding_data
);
441 jd
->unwinding_data
= NULL
;
442 jd
->eh_frame_hdr_size
= 0;
443 jd
->unwinding_mapped_size
= 0;
444 jd
->unwinding_size
= 0;
451 if (stat(filename
, &st
))
452 memset(&st
, 0, sizeof(st
));
454 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
455 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
456 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
457 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
459 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
460 event
->mmap2
.start
= addr
;
461 event
->mmap2
.len
= usize
? ALIGN_8(csize
) + usize
: csize
;
462 event
->mmap2
.pid
= pid
;
463 event
->mmap2
.tid
= tid
;
464 event
->mmap2
.ino
= st
.st_ino
;
465 event
->mmap2
.maj
= major(st
.st_dev
);
466 event
->mmap2
.min
= minor(st
.st_dev
);
467 event
->mmap2
.prot
= st
.st_mode
;
468 event
->mmap2
.flags
= MAP_SHARED
;
469 event
->mmap2
.ino_generation
= 1;
471 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
472 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
476 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
477 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
480 * create pseudo sample to induce dso hit increment
481 * use first address as sample address
483 memset(&sample
, 0, sizeof(sample
));
484 sample
.cpumode
= PERF_RECORD_MISC_USER
;
487 sample
.time
= id
->time
;
490 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
494 ret
= jit_inject_event(jd
, event
);
496 * mark dso as use to generate buildid in the header
499 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
504 static int jit_repipe_code_move(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
506 struct perf_sample sample
;
507 union perf_event
*event
;
508 struct perf_tool
*tool
= jd
->session
->tool
;
523 usize
= jd
->unwinding_mapped_size
;
524 idr_size
= jd
->machine
->id_hdr_size
;
527 * +16 to account for sample_id_all (hack)
529 event
= calloc(1, sizeof(*event
) + 16);
533 filename
= event
->mmap2
.filename
;
534 size
= snprintf(filename
, PATH_MAX
, "%s/jitted-%d-%"PRIu64
,
537 jr
->move
.code_index
);
541 if (stat(filename
, &st
))
542 memset(&st
, 0, sizeof(st
));
544 size
= PERF_ALIGN(size
, sizeof(u64
));
546 event
->mmap2
.header
.type
= PERF_RECORD_MMAP2
;
547 event
->mmap2
.header
.misc
= PERF_RECORD_MISC_USER
;
548 event
->mmap2
.header
.size
= (sizeof(event
->mmap2
) -
549 (sizeof(event
->mmap2
.filename
) - size
) + idr_size
);
550 event
->mmap2
.pgoff
= GEN_ELF_TEXT_OFFSET
;
551 event
->mmap2
.start
= jr
->move
.new_code_addr
;
552 event
->mmap2
.len
= usize
? ALIGN_8(jr
->move
.code_size
) + usize
553 : jr
->move
.code_size
;
554 event
->mmap2
.pid
= pid
;
555 event
->mmap2
.tid
= tid
;
556 event
->mmap2
.ino
= st
.st_ino
;
557 event
->mmap2
.maj
= major(st
.st_dev
);
558 event
->mmap2
.min
= minor(st
.st_dev
);
559 event
->mmap2
.prot
= st
.st_mode
;
560 event
->mmap2
.flags
= MAP_SHARED
;
561 event
->mmap2
.ino_generation
= 1;
563 id
= (void *)((unsigned long)event
+ event
->mmap
.header
.size
- idr_size
);
564 if (jd
->sample_type
& PERF_SAMPLE_TID
) {
568 if (jd
->sample_type
& PERF_SAMPLE_TIME
)
569 id
->time
= convert_timestamp(jd
, jr
->load
.p
.timestamp
);
572 * create pseudo sample to induce dso hit increment
573 * use first address as sample address
575 memset(&sample
, 0, sizeof(sample
));
576 sample
.cpumode
= PERF_RECORD_MISC_USER
;
579 sample
.time
= id
->time
;
580 sample
.ip
= jr
->move
.new_code_addr
;
582 ret
= perf_event__process_mmap2(tool
, event
, &sample
, jd
->machine
);
586 ret
= jit_inject_event(jd
, event
);
588 build_id__mark_dso_hit(tool
, event
, &sample
, NULL
, jd
->machine
);
593 static int jit_repipe_debug_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
601 sz
= jr
->prefix
.total_size
- sizeof(jr
->info
);
606 memcpy(data
, &jr
->info
.entries
, sz
);
608 jd
->debug_data
= data
;
611 * we must use nr_entry instead of size here because
612 * we cannot distinguish actual entry from padding otherwise
614 jd
->nr_debug_entries
= jr
->info
.nr_entry
;
620 jit_repipe_unwinding_info(struct jit_buf_desc
*jd
, union jr_entry
*jr
)
622 void *unwinding_data
;
623 uint32_t unwinding_data_size
;
628 unwinding_data_size
= jr
->prefix
.total_size
- sizeof(jr
->unwinding
);
629 unwinding_data
= malloc(unwinding_data_size
);
633 memcpy(unwinding_data
, &jr
->unwinding
.unwinding_data
,
634 unwinding_data_size
);
636 jd
->eh_frame_hdr_size
= jr
->unwinding
.eh_frame_hdr_size
;
637 jd
->unwinding_size
= jr
->unwinding
.unwinding_size
;
638 jd
->unwinding_mapped_size
= jr
->unwinding
.mapped_size
;
639 jd
->unwinding_data
= unwinding_data
;
645 jit_process_dump(struct jit_buf_desc
*jd
)
650 while ((jr
= jit_get_next_entry(jd
))) {
651 switch(jr
->prefix
.id
) {
653 ret
= jit_repipe_code_load(jd
, jr
);
656 ret
= jit_repipe_code_move(jd
, jr
);
658 case JIT_CODE_DEBUG_INFO
:
659 ret
= jit_repipe_debug_info(jd
, jr
);
661 case JIT_CODE_UNWINDING_INFO
:
662 ret
= jit_repipe_unwinding_info(jd
, jr
);
673 jit_inject(struct jit_buf_desc
*jd
, char *path
)
678 fprintf(stderr
, "injecting: %s\n", path
);
680 ret
= jit_open(jd
, path
);
684 ret
= jit_process_dump(jd
);
689 fprintf(stderr
, "injected: %s (%d)\n", path
, ret
);
695 * File must be with pattern .../jit-XXXX.dump
696 * where XXXX is the PID of the process which did the mmap()
697 * as captured in the RECORD_MMAP record
700 jit_detect(char *mmap_name
, pid_t pid
)
707 fprintf(stderr
, "jit marker trying : %s\n", mmap_name
);
711 p
= strrchr(mmap_name
, '/');
718 if (strncmp(p
, "/jit-", 5))
727 * must be followed by a pid
732 pid2
= (int)strtol(p
, &end
, 10);
737 * pid does not match mmap pid
738 * pid==0 in system-wide mode (synthesized)
740 if (pid
&& pid2
!= pid
)
745 if (strcmp(end
, ".dump"))
749 fprintf(stderr
, "jit marker found: %s\n", mmap_name
);
755 jit_process(struct perf_session
*session
,
756 struct perf_data
*output
,
757 struct machine
*machine
,
762 struct perf_evsel
*first
;
763 struct jit_buf_desc jd
;
767 * first, detect marker mmap (i.e., the jitdump mmap)
769 if (jit_detect(filename
, pid
))
772 memset(&jd
, 0, sizeof(jd
));
774 jd
.session
= session
;
776 jd
.machine
= machine
;
779 * track sample_type to compute id_all layout
780 * perf sets the same sample type to all events as of now
782 first
= perf_evlist__first(session
->evlist
);
783 jd
.sample_type
= first
->attr
.sample_type
;
787 ret
= jit_inject(&jd
, filename
);
789 *nbytes
= jd
.bytes_written
;