1 //===-- ProfiledBinary.h - Binary decoder -----------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #ifndef LLVM_TOOLS_LLVM_PROFGEN_PROFILEDBINARY_H
10 #define LLVM_TOOLS_LLVM_PROFGEN_PROFILEDBINARY_H
12 #include "CallContext.h"
13 #include "ErrorHandling.h"
14 #include "llvm/ADT/DenseMap.h"
15 #include "llvm/ADT/StringRef.h"
16 #include "llvm/ADT/StringSet.h"
17 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
18 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
19 #include "llvm/MC/MCAsmInfo.h"
20 #include "llvm/MC/MCContext.h"
21 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
22 #include "llvm/MC/MCInst.h"
23 #include "llvm/MC/MCInstPrinter.h"
24 #include "llvm/MC/MCInstrAnalysis.h"
25 #include "llvm/MC/MCInstrInfo.h"
26 #include "llvm/MC/MCObjectFileInfo.h"
27 #include "llvm/MC/MCPseudoProbe.h"
28 #include "llvm/MC/MCRegisterInfo.h"
29 #include "llvm/MC/MCSubtargetInfo.h"
30 #include "llvm/MC/MCTargetOptions.h"
31 #include "llvm/Object/ELFObjectFile.h"
32 #include "llvm/ProfileData/SampleProf.h"
33 #include "llvm/Support/CommandLine.h"
34 #include "llvm/Support/Path.h"
35 #include "llvm/Transforms/IPO/SampleContextTracker.h"
40 #include <unordered_map>
41 #include <unordered_set>
45 extern cl::opt
<bool> EnableCSPreInliner
;
46 extern cl::opt
<bool> UseContextCostForPreInliner
;
50 using namespace sampleprof
;
51 using namespace llvm::object
;
54 namespace sampleprof
{
57 class MissingFrameInferrer
;
59 struct InstructionPointer
{
60 const ProfiledBinary
*Binary
;
61 // Address of the executable segment of the binary.
63 // Index to the sorted code address array of the binary.
65 InstructionPointer(const ProfiledBinary
*Binary
, uint64_t Address
,
66 bool RoundToNext
= false);
69 void update(uint64_t Addr
);
72 // The special frame addresses.
73 enum SpecialFrameAddr
{
74 // Dummy root of frame trie.
76 // Represent all the addresses outside of current binary.
77 // This's also used to indicate the call stack should be truncated since this
78 // isn't a real call context the compiler will see.
82 using RangesTy
= std::vector
<std::pair
<uint64_t, uint64_t>>;
84 struct BinaryFunction
{
86 // End of range is an exclusive bound.
89 uint64_t getFuncSize() {
91 for (auto &R
: Ranges
) {
92 Sum
+= R
.second
- R
.first
;
98 // Info about function range. A function can be split into multiple
99 // non-continuous ranges, each range corresponds to one FuncRange.
101 uint64_t StartAddress
;
102 // EndAddress is an exclusive bound.
104 // Function the range belongs to
105 BinaryFunction
*Func
;
106 // Whether the start address is the real entry of the function.
107 bool IsFuncEntry
= false;
109 StringRef
getFuncName() { return Func
->FuncName
; }
112 // PrologEpilog address tracker, used to filter out broken stack samples
113 // Currently we use a heuristic size (two) to infer prolog and epilog
114 // based on the start address and return address. In the future,
115 // we will switch to Dwarf CFI based tracker
116 struct PrologEpilogTracker
{
117 // A set of prolog and epilog addresses. Used by virtual unwinding.
118 std::unordered_set
<uint64_t> PrologEpilogSet
;
119 ProfiledBinary
*Binary
;
120 PrologEpilogTracker(ProfiledBinary
*Bin
) : Binary(Bin
){};
122 // Take the two addresses from the start of function as prolog
124 inferPrologAddresses(std::map
<uint64_t, FuncRange
> &FuncStartAddressMap
) {
125 for (auto I
: FuncStartAddressMap
) {
126 PrologEpilogSet
.insert(I
.first
);
127 InstructionPointer
IP(Binary
, I
.first
);
130 PrologEpilogSet
.insert(IP
.Address
);
134 // Take the last two addresses before the return address as epilog
135 void inferEpilogAddresses(std::unordered_set
<uint64_t> &RetAddrs
) {
136 for (auto Addr
: RetAddrs
) {
137 PrologEpilogSet
.insert(Addr
);
138 InstructionPointer
IP(Binary
, Addr
);
141 PrologEpilogSet
.insert(IP
.Address
);
146 // Track function byte size under different context (outlined version as well as
147 // various inlined versions). It also provides query support to get function
148 // size with the best matching context, which is used to help pre-inliner use
149 // accurate post-optimization size to make decisions.
150 // TODO: If an inlinee is completely optimized away, ideally we should have zero
151 // for its context size, currently we would misss such context since it doesn't
152 // have instructions. To fix this, we need to mark all inlinee with entry probe
153 // but without instructions as having zero size.
154 class BinarySizeContextTracker
{
156 // Add instruction with given size to a context
157 void addInstructionForContext(const SampleContextFrameVector
&Context
,
160 // Get function size with a specific context. When there's no exact match
161 // for the given context, try to retrieve the size of that function from
162 // closest matching context.
163 uint32_t getFuncSizeForContext(const ContextTrieNode
*Context
);
165 // For inlinees that are full optimized away, we can establish zero size using
166 // their remaining probes.
167 void trackInlineesOptimizedAway(MCPseudoProbeDecoder
&ProbeDecoder
);
169 using ProbeFrameStack
= SmallVector
<std::pair
<StringRef
, uint32_t>>;
170 void trackInlineesOptimizedAway(MCPseudoProbeDecoder
&ProbeDecoder
,
171 MCDecodedPseudoProbeInlineTree
&ProbeNode
,
172 ProbeFrameStack
&Context
);
174 void dump() { RootContext
.dumpTree(); }
177 // Root node for context trie tree, node that this is a reverse context trie
178 // with callee as parent and caller as child. This way we can traverse from
179 // root to find the best/longest matching context if an exact match does not
180 // exist. It gives us the best possible estimate for function's post-inline,
181 // post-optimization byte size.
182 ContextTrieNode RootContext
;
185 using AddressRange
= std::pair
<uint64_t, uint64_t>;
187 class ProfiledBinary
{
188 // Absolute path of the executable binary.
190 // Path of the debug info binary.
191 std::string DebugBinaryPath
;
192 // The target triple.
194 // Path of symbolizer path which should be pointed to binary with debug info.
195 StringRef SymbolizerPath
;
196 // Options used to configure the symbolizer
197 symbolize::LLVMSymbolizer::Options SymbolizerOpts
;
198 // The runtime base address that the first executable segment is loaded at.
199 uint64_t BaseAddress
= 0;
200 // The runtime base address that the first loadabe segment is loaded at.
201 uint64_t FirstLoadableAddress
= 0;
202 // The preferred load address of each executable segment.
203 std::vector
<uint64_t> PreferredTextSegmentAddresses
;
204 // The file offset of each executable segment.
205 std::vector
<uint64_t> TextSegmentOffsets
;
207 // Mutiple MC component info
208 std::unique_ptr
<const MCRegisterInfo
> MRI
;
209 std::unique_ptr
<const MCAsmInfo
> AsmInfo
;
210 std::unique_ptr
<const MCSubtargetInfo
> STI
;
211 std::unique_ptr
<const MCInstrInfo
> MII
;
212 std::unique_ptr
<MCDisassembler
> DisAsm
;
213 std::unique_ptr
<const MCInstrAnalysis
> MIA
;
214 std::unique_ptr
<MCInstPrinter
> IPrinter
;
215 // A list of text sections sorted by start RVA and size. Used to check
216 // if a given RVA is a valid code address.
217 std::set
<std::pair
<uint64_t, uint64_t>> TextSections
;
219 // A map of mapping function name to BinaryFunction info.
220 std::unordered_map
<std::string
, BinaryFunction
> BinaryFunctions
;
222 // Lookup BinaryFunctions using the function name's MD5 hash. Needed if the
223 // profile is using MD5.
224 std::unordered_map
<uint64_t, BinaryFunction
*> HashBinaryFunctions
;
226 // A list of binary functions that have samples.
227 std::unordered_set
<const BinaryFunction
*> ProfiledFunctions
;
229 // GUID to Elf symbol start address map
230 DenseMap
<uint64_t, uint64_t> SymbolStartAddrs
;
232 // These maps are for temporary use of warning diagnosis.
233 DenseSet
<int64_t> AddrsWithMultipleSymbols
;
234 DenseSet
<std::pair
<uint64_t, uint64_t>> AddrsWithInvalidInstruction
;
236 // Start address to Elf symbol GUID map
237 std::unordered_multimap
<uint64_t, uint64_t> StartAddrToSymMap
;
239 // An ordered map of mapping function's start address to function range
240 // relevant info. Currently to determine if the offset of ELF is the start of
241 // a real function, we leverage the function range info from DWARF.
242 std::map
<uint64_t, FuncRange
> StartAddrToFuncRangeMap
;
244 // Address to context location map. Used to expand the context.
245 std::unordered_map
<uint64_t, SampleContextFrameVector
> AddressToLocStackMap
;
247 // Address to instruction size map. Also used for quick Address lookup.
248 std::unordered_map
<uint64_t, uint64_t> AddressToInstSizeMap
;
250 // An array of Addresses of all instructions sorted in increasing order. The
251 // sorting is needed to fast advance to the next forward/backward instruction.
252 std::vector
<uint64_t> CodeAddressVec
;
253 // A set of call instruction addresses. Used by virtual unwinding.
254 std::unordered_set
<uint64_t> CallAddressSet
;
255 // A set of return instruction addresses. Used by virtual unwinding.
256 std::unordered_set
<uint64_t> RetAddressSet
;
257 // An ordered set of unconditional branch instruction addresses.
258 std::set
<uint64_t> UncondBranchAddrSet
;
259 // A set of branch instruction addresses.
260 std::unordered_set
<uint64_t> BranchAddressSet
;
262 // Estimate and track function prolog and epilog ranges.
263 PrologEpilogTracker ProEpilogTracker
;
265 // Infer missing frames due to compiler optimizations such as tail call
267 std::unique_ptr
<MissingFrameInferrer
> MissingContextInferrer
;
269 // Track function sizes under different context
270 BinarySizeContextTracker FuncSizeTracker
;
272 // The symbolizer used to get inline context for an instruction.
273 std::unique_ptr
<symbolize::LLVMSymbolizer
> Symbolizer
;
275 // String table owning function name strings created from the symbolizer.
276 std::unordered_set
<std::string
> NameStrings
;
278 // A collection of functions to print disassembly for.
279 StringSet
<> DisassembleFunctionSet
;
281 // Pseudo probe decoder
282 MCPseudoProbeDecoder ProbeDecoder
;
284 // Function name to probe frame map for top-level outlined functions.
285 StringMap
<MCDecodedPseudoProbeInlineTree
*> TopLevelProbeFrameMap
;
287 bool UsePseudoProbes
= false;
289 bool UseFSDiscriminator
= false;
291 // Whether we need to symbolize all instructions to get function context size.
292 bool TrackFuncContextSize
= false;
294 // Indicate if the base loading address is parsed from the mmap event or uses
295 // the preferred address
296 bool IsLoadedByMMap
= false;
297 // Use to avoid redundant warning.
298 bool MissingMMapWarned
= false;
300 void setPreferredTextSegmentAddresses(const ELFObjectFileBase
*O
);
302 template <class ELFT
>
303 void setPreferredTextSegmentAddresses(const ELFFile
<ELFT
> &Obj
,
306 void checkPseudoProbe(const ELFObjectFileBase
*Obj
);
308 void decodePseudoProbe(const ELFObjectFileBase
*Obj
);
311 checkUseFSDiscriminator(const ELFObjectFileBase
*Obj
,
312 std::map
<SectionRef
, SectionSymbolsTy
> &AllSymbols
);
314 // Set up disassembler and related components.
315 void setUpDisassembler(const ELFObjectFileBase
*Obj
);
316 symbolize::LLVMSymbolizer::Options
getSymbolizerOpts() const;
318 // Load debug info of subprograms from DWARF section.
319 void loadSymbolsFromDWARF(ObjectFile
&Obj
);
321 // Load debug info from DWARF unit.
322 void loadSymbolsFromDWARFUnit(DWARFUnit
&CompilationUnit
);
324 // Create elf symbol to its start address mapping.
325 void populateElfSymbolAddressList(const ELFObjectFileBase
*O
);
327 // A function may be spilt into multiple non-continuous address ranges. We use
328 // this to set whether start a function range is the real entry of the
329 // function and also set false to the non-function label.
330 void setIsFuncEntry(FuncRange
*FRange
, StringRef RangeSymName
);
332 // Warn if no entry range exists in the function.
333 void warnNoFuncEntry();
335 /// Dissassemble the text section and build various address maps.
336 void disassemble(const ELFObjectFileBase
*O
);
338 /// Helper function to dissassemble the symbol and extract info for unwinding
339 bool dissassembleSymbol(std::size_t SI
, ArrayRef
<uint8_t> Bytes
,
340 SectionSymbolsTy
&Symbols
, const SectionRef
&Section
);
341 /// Symbolize a given instruction pointer and return a full call context.
342 SampleContextFrameVector
symbolize(const InstructionPointer
&IP
,
343 bool UseCanonicalFnName
= false,
344 bool UseProbeDiscriminator
= false);
345 /// Decode the interesting parts of the binary and build internal data
346 /// structures. On high level, the parts of interest are:
347 /// 1. Text sections, including the main code section and the PLT
348 /// entries that will be used to handle cross-module call transitions.
349 /// 2. The .debug_line section, used by Dwarf-based profile generation.
350 /// 3. Pseudo probe related sections, used by probe-based profile
355 ProfiledBinary(const StringRef ExeBinPath
, const StringRef DebugBinPath
);
358 void decodePseudoProbe();
360 StringRef
getPath() const { return Path
; }
361 StringRef
getName() const { return llvm::sys::path::filename(Path
); }
362 uint64_t getBaseAddress() const { return BaseAddress
; }
363 void setBaseAddress(uint64_t Address
) { BaseAddress
= Address
; }
365 // Canonicalize to use preferred load address as base address.
366 uint64_t canonicalizeVirtualAddress(uint64_t Address
) {
367 return Address
- BaseAddress
+ getPreferredBaseAddress();
369 // Return the preferred load address for the first executable segment.
370 uint64_t getPreferredBaseAddress() const {
371 return PreferredTextSegmentAddresses
[0];
373 // Return the preferred load address for the first loadable segment.
374 uint64_t getFirstLoadableAddress() const { return FirstLoadableAddress
; }
375 // Return the file offset for the first executable segment.
376 uint64_t getTextSegmentOffset() const { return TextSegmentOffsets
[0]; }
377 const std::vector
<uint64_t> &getPreferredTextSegmentAddresses() const {
378 return PreferredTextSegmentAddresses
;
380 const std::vector
<uint64_t> &getTextSegmentOffsets() const {
381 return TextSegmentOffsets
;
384 uint64_t getInstSize(uint64_t Address
) const {
385 auto I
= AddressToInstSizeMap
.find(Address
);
386 if (I
== AddressToInstSizeMap
.end())
391 bool addressIsCode(uint64_t Address
) const {
392 return AddressToInstSizeMap
.find(Address
) != AddressToInstSizeMap
.end();
395 bool addressIsCall(uint64_t Address
) const {
396 return CallAddressSet
.count(Address
);
398 bool addressIsReturn(uint64_t Address
) const {
399 return RetAddressSet
.count(Address
);
401 bool addressInPrologEpilog(uint64_t Address
) const {
402 return ProEpilogTracker
.PrologEpilogSet
.count(Address
);
405 bool addressIsTransfer(uint64_t Address
) {
406 return BranchAddressSet
.count(Address
) || RetAddressSet
.count(Address
) ||
407 CallAddressSet
.count(Address
);
410 bool rangeCrossUncondBranch(uint64_t Start
, uint64_t End
) {
413 auto R
= UncondBranchAddrSet
.lower_bound(Start
);
414 return R
!= UncondBranchAddrSet
.end() && *R
< End
;
417 uint64_t getAddressforIndex(uint64_t Index
) const {
418 return CodeAddressVec
[Index
];
421 size_t getCodeAddrVecSize() const { return CodeAddressVec
.size(); }
423 bool usePseudoProbes() const { return UsePseudoProbes
; }
424 bool useFSDiscriminator() const { return UseFSDiscriminator
; }
425 // Get the index in CodeAddressVec for the address
426 // As we might get an address which is not the code
427 // here it would round to the next valid code address by
428 // using lower bound operation
429 uint32_t getIndexForAddr(uint64_t Address
) const {
430 auto Low
= llvm::lower_bound(CodeAddressVec
, Address
);
431 return Low
- CodeAddressVec
.begin();
434 uint64_t getCallAddrFromFrameAddr(uint64_t FrameAddr
) const {
435 if (FrameAddr
== ExternalAddr
)
437 auto I
= getIndexForAddr(FrameAddr
);
438 FrameAddr
= I
? getAddressforIndex(I
- 1) : 0;
439 if (FrameAddr
&& addressIsCall(FrameAddr
))
444 FuncRange
*findFuncRangeForStartAddr(uint64_t Address
) {
445 auto I
= StartAddrToFuncRangeMap
.find(Address
);
446 if (I
== StartAddrToFuncRangeMap
.end())
451 // Binary search the function range which includes the input address.
452 FuncRange
*findFuncRange(uint64_t Address
) {
453 auto I
= StartAddrToFuncRangeMap
.upper_bound(Address
);
454 if (I
== StartAddrToFuncRangeMap
.begin())
458 if (Address
>= I
->second
.EndAddress
)
464 // Get all ranges of one function.
465 RangesTy
getRanges(uint64_t Address
) {
466 auto *FRange
= findFuncRange(Address
);
467 // Ignore the range which falls into plt section or system lib.
471 return FRange
->Func
->Ranges
;
474 const std::unordered_map
<std::string
, BinaryFunction
> &
475 getAllBinaryFunctions() {
476 return BinaryFunctions
;
479 std::unordered_set
<const BinaryFunction
*> &getProfiledFunctions() {
480 return ProfiledFunctions
;
483 void setProfiledFunctions(std::unordered_set
<const BinaryFunction
*> &Funcs
) {
484 ProfiledFunctions
= Funcs
;
487 BinaryFunction
*getBinaryFunction(FunctionId FName
) {
488 if (FName
.isStringRef()) {
489 auto I
= BinaryFunctions
.find(FName
.str());
490 if (I
== BinaryFunctions
.end())
494 auto I
= HashBinaryFunctions
.find(FName
.getHashCode());
495 if (I
== HashBinaryFunctions
.end())
500 uint32_t getFuncSizeForContext(const ContextTrieNode
*ContextNode
) {
501 return FuncSizeTracker
.getFuncSizeForContext(ContextNode
);
504 void inferMissingFrames(const SmallVectorImpl
<uint64_t> &Context
,
505 SmallVectorImpl
<uint64_t> &NewContext
);
507 // Load the symbols from debug table and populate into symbol list.
508 void populateSymbolListFromDWARF(ProfileSymbolList
&SymbolList
);
510 SampleContextFrameVector
511 getFrameLocationStack(uint64_t Address
, bool UseProbeDiscriminator
= false) {
512 InstructionPointer
IP(this, Address
);
513 return symbolize(IP
, SymbolizerOpts
.UseSymbolTable
, UseProbeDiscriminator
);
516 const SampleContextFrameVector
&
517 getCachedFrameLocationStack(uint64_t Address
,
518 bool UseProbeDiscriminator
= false) {
519 auto I
= AddressToLocStackMap
.emplace(Address
, SampleContextFrameVector());
521 I
.first
->second
= getFrameLocationStack(Address
, UseProbeDiscriminator
);
523 return I
.first
->second
;
526 std::optional
<SampleContextFrame
> getInlineLeafFrameLoc(uint64_t Address
) {
527 const auto &Stack
= getCachedFrameLocationStack(Address
);
533 void flushSymbolizer() { Symbolizer
.reset(); }
535 MissingFrameInferrer
*getMissingContextInferrer() {
536 return MissingContextInferrer
.get();
539 // Compare two addresses' inline context
540 bool inlineContextEqual(uint64_t Add1
, uint64_t Add2
);
542 // Get the full context of the current stack with inline context filled in.
543 // It will search the disassembling info stored in AddressToLocStackMap. This
544 // is used as the key of function sample map
545 SampleContextFrameVector
546 getExpandedContext(const SmallVectorImpl
<uint64_t> &Stack
,
547 bool &WasLeafInlined
);
548 // Go through instructions among the given range and record its size for the
550 void computeInlinedContextSizeForRange(uint64_t StartAddress
,
551 uint64_t EndAddress
);
553 void computeInlinedContextSizeForFunc(const BinaryFunction
*Func
);
555 const MCDecodedPseudoProbe
*getCallProbeForAddr(uint64_t Address
) const {
556 return ProbeDecoder
.getCallProbeForAddr(Address
);
559 void getInlineContextForProbe(const MCDecodedPseudoProbe
*Probe
,
560 SampleContextFrameVector
&InlineContextStack
,
561 bool IncludeLeaf
= false) const {
562 SmallVector
<MCPseduoProbeFrameLocation
, 16> ProbeInlineContext
;
563 ProbeDecoder
.getInlineContextForProbe(Probe
, ProbeInlineContext
,
565 for (uint32_t I
= 0; I
< ProbeInlineContext
.size(); I
++) {
566 auto &Callsite
= ProbeInlineContext
[I
];
567 // Clear the current context for an unknown probe.
568 if (Callsite
.second
== 0 && I
!= ProbeInlineContext
.size() - 1) {
569 InlineContextStack
.clear();
572 InlineContextStack
.emplace_back(FunctionId(Callsite
.first
),
573 LineLocation(Callsite
.second
, 0));
576 const AddressProbesMap
&getAddress2ProbesMap() const {
577 return ProbeDecoder
.getAddress2ProbesMap();
579 const MCPseudoProbeFuncDesc
*getFuncDescForGUID(uint64_t GUID
) {
580 return ProbeDecoder
.getFuncDescForGUID(GUID
);
583 const MCPseudoProbeFuncDesc
*
584 getInlinerDescForProbe(const MCDecodedPseudoProbe
*Probe
) {
585 return ProbeDecoder
.getInlinerDescForProbe(Probe
);
588 bool getTrackFuncContextSize() { return TrackFuncContextSize
; }
590 bool getIsLoadedByMMap() { return IsLoadedByMMap
; }
592 void setIsLoadedByMMap(bool Value
) { IsLoadedByMMap
= Value
; }
594 bool getMissingMMapWarned() { return MissingMMapWarned
; }
596 void setMissingMMapWarned(bool Value
) { MissingMMapWarned
= Value
; }
599 } // end namespace sampleprof
600 } // end namespace llvm