1 //===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines a JITEventListener object that tells perf about JITted
10 // functions, including source line information.
12 // Documentation for perf jit integration is available at:
13 // https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jitdump-specification.txt
14 // https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt
16 //===----------------------------------------------------------------------===//
18 #include "llvm/ADT/Twine.h"
19 #include "llvm/Config/config.h"
20 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
21 #include "llvm/ExecutionEngine/JITEventListener.h"
22 #include "llvm/Object/ObjectFile.h"
23 #include "llvm/Object/SymbolSize.h"
24 #include "llvm/Support/Debug.h"
25 #include "llvm/Support/Errno.h"
26 #include "llvm/Support/FileSystem.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/Mutex.h"
29 #include "llvm/Support/Path.h"
30 #include "llvm/Support/Process.h"
31 #include "llvm/Support/Threading.h"
32 #include "llvm/Support/raw_ostream.h"
35 #include <sys/mman.h> // mmap()
36 #include <sys/types.h> // getpid()
37 #include <time.h> // clock_gettime(), time(), localtime_r() */
38 #include <unistd.h> // for getpid(), read(), close()
41 using namespace llvm::object
;
42 typedef DILineInfoSpecifier::FileLineInfoKind FileLineInfoKind
;
46 // language identifier (XXX: should we generate something better from debug
48 #define JIT_LANG "llvm-IR"
49 #define LLVM_PERF_JIT_MAGIC \
50 ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 | \
52 #define LLVM_PERF_JIT_VERSION 1
54 // bit 0: set if the jitdump file is using an architecture-specific timestamp
56 #define JITDUMP_FLAGS_ARCH_TIMESTAMP (1ULL << 0)
58 struct LLVMPerfJitHeader
;
60 class PerfJITEventListener
: public JITEventListener
{
62 PerfJITEventListener();
63 ~PerfJITEventListener() {
68 void notifyObjectLoaded(ObjectKey K
, const ObjectFile
&Obj
,
69 const RuntimeDyld::LoadedObjectInfo
&L
) override
;
70 void notifyFreeingObject(ObjectKey K
) override
;
73 bool InitDebuggingDir();
76 static bool FillMachine(LLVMPerfJitHeader
&hdr
);
78 void NotifyCode(Expected
<llvm::StringRef
> &Symbol
, uint64_t CodeAddr
,
80 void NotifyDebug(uint64_t CodeAddr
, DILineInfoTable Lines
);
85 // base directory for output data
88 // output data stream, closed via Dumpstream
92 std::unique_ptr
<raw_fd_ostream
> Dumpstream
;
94 // prevent concurrent dumps from messing up the output file
98 void *MarkerAddr
= NULL
;
100 // perf support ready
101 bool SuccessfullyInitialized
= false;
103 // identifier for functions, primarily to identify when moving them around
104 uint64_t CodeGeneration
= 1;
107 // The following are POD struct definitions from the perf jit specification
109 enum LLVMPerfJitRecordType
{
111 JIT_CODE_MOVE
= 1, // not emitted, code isn't moved
112 JIT_CODE_DEBUG_INFO
= 2,
113 JIT_CODE_CLOSE
= 3, // not emitted, unnecessary
114 JIT_CODE_UNWINDING_INFO
= 4, // not emitted
119 struct LLVMPerfJitHeader
{
120 uint32_t Magic
; // characters "JiTD"
121 uint32_t Version
; // header version
122 uint32_t TotalSize
; // total size of header
123 uint32_t ElfMach
; // elf mach target
124 uint32_t Pad1
; // reserved
126 uint64_t Timestamp
; // timestamp
127 uint64_t Flags
; // flags
130 // record prefix (mandatory in each record)
131 struct LLVMPerfJitRecordPrefix
{
132 uint32_t Id
; // record type identifier
137 struct LLVMPerfJitRecordCodeLoad
{
138 LLVMPerfJitRecordPrefix Prefix
;
148 struct LLVMPerfJitDebugEntry
{
150 int Lineno
; // source line number starting at 1
151 int Discrim
; // column discriminator, 0 is default
152 // followed by null terminated filename, \xff\0 if same as previous entry
155 struct LLVMPerfJitRecordDebugInfo
{
156 LLVMPerfJitRecordPrefix Prefix
;
160 // followed by NrEntry LLVMPerfJitDebugEntry records
163 static inline uint64_t timespec_to_ns(const struct timespec
*ts
) {
164 const uint64_t NanoSecPerSec
= 1000000000;
165 return ((uint64_t)ts
->tv_sec
* NanoSecPerSec
) + ts
->tv_nsec
;
168 static inline uint64_t perf_get_timestamp(void) {
172 ret
= clock_gettime(CLOCK_MONOTONIC
, &ts
);
176 return timespec_to_ns(&ts
);
179 PerfJITEventListener::PerfJITEventListener() : Pid(::getpid()) {
180 // check if clock-source is supported
181 if (!perf_get_timestamp()) {
182 errs() << "kernel does not support CLOCK_MONOTONIC\n";
186 if (!InitDebuggingDir()) {
187 errs() << "could not initialize debugging directory\n";
191 std::string Filename
;
192 raw_string_ostream
FilenameBuf(Filename
);
193 FilenameBuf
<< JitPath
<< "/jit-" << Pid
<< ".dump";
195 // Need to open ourselves, because we need to hand the FD to OpenMarker() and
196 // raw_fd_ostream doesn't expose the FD.
197 using sys::fs::openFileForWrite
;
199 openFileForReadWrite(FilenameBuf
.str(), DumpFd
,
200 sys::fs::CD_CreateNew
, sys::fs::OF_None
)) {
201 errs() << "could not open JIT dump file " << FilenameBuf
.str() << ": "
202 << EC
.message() << "\n";
206 Dumpstream
= std::make_unique
<raw_fd_ostream
>(DumpFd
, true);
208 LLVMPerfJitHeader Header
= {0};
209 if (!FillMachine(Header
))
212 // signal this process emits JIT information
216 // emit dumpstream header
217 Header
.Magic
= LLVM_PERF_JIT_MAGIC
;
218 Header
.Version
= LLVM_PERF_JIT_VERSION
;
219 Header
.TotalSize
= sizeof(Header
);
221 Header
.Timestamp
= perf_get_timestamp();
222 Dumpstream
->write(reinterpret_cast<const char *>(&Header
), sizeof(Header
));
224 // Everything initialized, can do profiling now.
225 if (!Dumpstream
->has_error())
226 SuccessfullyInitialized
= true;
229 void PerfJITEventListener::notifyObjectLoaded(
230 ObjectKey K
, const ObjectFile
&Obj
,
231 const RuntimeDyld::LoadedObjectInfo
&L
) {
233 if (!SuccessfullyInitialized
)
236 OwningBinary
<ObjectFile
> DebugObjOwner
= L
.getObjectForDebug(Obj
);
237 const ObjectFile
&DebugObj
= *DebugObjOwner
.getBinary();
239 // Get the address of the object image for use as a unique identifier
240 std::unique_ptr
<DIContext
> Context
= DWARFContext::create(DebugObj
);
242 // Use symbol info to iterate over functions in the object.
243 for (const std::pair
<SymbolRef
, uint64_t> &P
: computeSymbolSizes(DebugObj
)) {
244 SymbolRef Sym
= P
.first
;
245 std::string SourceFileName
;
247 Expected
<SymbolRef::Type
> SymTypeOrErr
= Sym
.getType();
249 // There's not much we can with errors here
250 consumeError(SymTypeOrErr
.takeError());
253 SymbolRef::Type SymType
= *SymTypeOrErr
;
254 if (SymType
!= SymbolRef::ST_Function
)
257 Expected
<StringRef
> Name
= Sym
.getName();
259 consumeError(Name
.takeError());
263 Expected
<uint64_t> AddrOrErr
= Sym
.getAddress();
265 consumeError(AddrOrErr
.takeError());
268 uint64_t Size
= P
.second
;
269 object::SectionedAddress Address
;
270 Address
.Address
= *AddrOrErr
;
272 uint64_t SectionIndex
= object::SectionedAddress::UndefSection
;
273 if (auto SectOrErr
= Sym
.getSection())
274 if (*SectOrErr
!= Obj
.section_end())
275 SectionIndex
= SectOrErr
.get()->getIndex();
277 // According to spec debugging info has to come before loading the
278 // corresonding code load.
279 DILineInfoTable Lines
= Context
->getLineInfoForAddressRange(
280 {*AddrOrErr
, SectionIndex
}, Size
, FileLineInfoKind::AbsoluteFilePath
);
282 NotifyDebug(*AddrOrErr
, Lines
);
283 NotifyCode(Name
, *AddrOrErr
, Size
);
289 void PerfJITEventListener::notifyFreeingObject(ObjectKey K
) {
290 // perf currently doesn't have an interface for unloading. But munmap()ing the
291 // code section does, so that's ok.
294 bool PerfJITEventListener::InitDebuggingDir() {
297 char TimeBuffer
[sizeof("YYYYMMDD")];
298 SmallString
<64> Path
;
300 // search for location to dump data to
301 if (const char *BaseDir
= getenv("JITDUMPDIR"))
302 Path
.append(BaseDir
);
303 else if (!sys::path::home_directory(Path
))
306 // create debug directory
307 Path
+= "/.debug/jit/";
308 if (auto EC
= sys::fs::create_directories(Path
)) {
309 errs() << "could not create jit cache directory " << Path
<< ": "
310 << EC
.message() << "\n";
314 // create unique directory for dump data related to this process
316 localtime_r(&Time
, &LocalTime
);
317 strftime(TimeBuffer
, sizeof(TimeBuffer
), "%Y%m%d", &LocalTime
);
318 Path
+= JIT_LANG
"-jit-";
321 SmallString
<128> UniqueDebugDir
;
323 using sys::fs::createUniqueDirectory
;
324 if (auto EC
= createUniqueDirectory(Path
, UniqueDebugDir
)) {
325 errs() << "could not create unique jit cache directory " << UniqueDebugDir
326 << ": " << EC
.message() << "\n";
330 JitPath
= UniqueDebugDir
.str();
335 bool PerfJITEventListener::OpenMarker() {
336 // We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap
337 // is captured either live (perf record running when we mmap) or in deferred
338 // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump
339 // file for more meta data info about the jitted code. Perf report/annotate
340 // detect this special filename and process the jitdump file.
342 // Mapping must be PROT_EXEC to ensure it is captured by perf record
343 // even when not using -d option.
344 MarkerAddr
= ::mmap(NULL
, sys::Process::getPageSizeEstimate(),
345 PROT_READ
| PROT_EXEC
, MAP_PRIVATE
, DumpFd
, 0);
347 if (MarkerAddr
== MAP_FAILED
) {
348 errs() << "could not mmap JIT marker\n";
354 void PerfJITEventListener::CloseMarker() {
358 munmap(MarkerAddr
, sys::Process::getPageSizeEstimate());
359 MarkerAddr
= nullptr;
362 bool PerfJITEventListener::FillMachine(LLVMPerfJitHeader
&hdr
) {
369 size_t RequiredMemory
= sizeof(id
) + sizeof(info
);
371 ErrorOr
<std::unique_ptr
<MemoryBuffer
>> MB
=
372 MemoryBuffer::getFileSlice("/proc/self/exe",
376 // This'll not guarantee that enough data was actually read from the
377 // underlying file. Instead the trailing part of the buffer would be
378 // zeroed. Given the ELF signature check below that seems ok though,
379 // it's unlikely that the file ends just after that, and the
380 // consequence would just be that perf wouldn't recognize the
382 if (auto EC
= MB
.getError()) {
383 errs() << "could not open /proc/self/exe: " << EC
.message() << "\n";
387 memcpy(&id
, (*MB
)->getBufferStart(), sizeof(id
));
388 memcpy(&info
, (*MB
)->getBufferStart() + sizeof(id
), sizeof(info
));
390 // check ELF signature
391 if (id
[0] != 0x7f || id
[1] != 'E' || id
[2] != 'L' || id
[3] != 'F') {
392 errs() << "invalid elf signature\n";
396 hdr
.ElfMach
= info
.e_machine
;
401 void PerfJITEventListener::NotifyCode(Expected
<llvm::StringRef
> &Symbol
,
402 uint64_t CodeAddr
, uint64_t CodeSize
) {
403 assert(SuccessfullyInitialized
);
405 // 0 length functions can't have samples.
409 LLVMPerfJitRecordCodeLoad rec
;
410 rec
.Prefix
.Id
= JIT_CODE_LOAD
;
411 rec
.Prefix
.TotalSize
= sizeof(rec
) + // debug record itself
412 Symbol
->size() + 1 + // symbol name
413 CodeSize
; // and code
414 rec
.Prefix
.Timestamp
= perf_get_timestamp();
416 rec
.CodeSize
= CodeSize
;
418 rec
.CodeAddr
= CodeAddr
;
420 rec
.Tid
= get_threadid();
422 // avoid interspersing output
423 std::lock_guard
<sys::Mutex
> Guard(Mutex
);
425 rec
.CodeIndex
= CodeGeneration
++; // under lock!
427 Dumpstream
->write(reinterpret_cast<const char *>(&rec
), sizeof(rec
));
428 Dumpstream
->write(Symbol
->data(), Symbol
->size() + 1);
429 Dumpstream
->write(reinterpret_cast<const char *>(CodeAddr
), CodeSize
);
432 void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr
,
433 DILineInfoTable Lines
) {
434 assert(SuccessfullyInitialized
);
436 // Didn't get useful debug info.
440 LLVMPerfJitRecordDebugInfo rec
;
441 rec
.Prefix
.Id
= JIT_CODE_DEBUG_INFO
;
442 rec
.Prefix
.TotalSize
= sizeof(rec
); // will be increased further
443 rec
.Prefix
.Timestamp
= perf_get_timestamp();
444 rec
.CodeAddr
= CodeAddr
;
445 rec
.NrEntry
= Lines
.size();
447 // compute total size size of record (variable due to filenames)
448 DILineInfoTable::iterator Begin
= Lines
.begin();
449 DILineInfoTable::iterator End
= Lines
.end();
450 for (DILineInfoTable::iterator It
= Begin
; It
!= End
; ++It
) {
451 DILineInfo
&line
= It
->second
;
452 rec
.Prefix
.TotalSize
+= sizeof(LLVMPerfJitDebugEntry
);
453 rec
.Prefix
.TotalSize
+= line
.FileName
.size() + 1;
456 // The debug_entry describes the source line information. It is defined as
458 // * uint64_t code_addr: address of function for which the debug information
460 // * uint32_t line : source file line number (starting at 1)
461 // * uint32_t discrim : column discriminator, 0 is default
462 // * char name[n] : source file name in ASCII, including null termination
464 // avoid interspersing output
465 std::lock_guard
<sys::Mutex
> Guard(Mutex
);
467 Dumpstream
->write(reinterpret_cast<const char *>(&rec
), sizeof(rec
));
469 for (DILineInfoTable::iterator It
= Begin
; It
!= End
; ++It
) {
470 LLVMPerfJitDebugEntry LineInfo
;
471 DILineInfo
&Line
= It
->second
;
473 LineInfo
.Addr
= It
->first
;
474 // The function re-created by perf is preceded by a elf
475 // header. Need to adjust for that, otherwise the results are
477 LineInfo
.Addr
+= 0x40;
478 LineInfo
.Lineno
= Line
.Line
;
479 LineInfo
.Discrim
= Line
.Discriminator
;
481 Dumpstream
->write(reinterpret_cast<const char *>(&LineInfo
),
483 Dumpstream
->write(Line
.FileName
.c_str(), Line
.FileName
.size() + 1);
487 // There should be only a single event listener per process, otherwise perf gets
489 llvm::ManagedStatic
<PerfJITEventListener
> PerfListener
;
491 } // end anonymous namespace
494 JITEventListener
*JITEventListener::createPerfJITEventListener() {
495 return &*PerfListener
;
500 LLVMJITEventListenerRef
LLVMCreatePerfJITEventListener(void)
502 return wrap(JITEventListener::createPerfJITEventListener());