1 //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "llvm/MC/MCPseudoProbe.h"
10 #include "llvm/MC/MCAsmInfo.h"
11 #include "llvm/MC/MCContext.h"
12 #include "llvm/MC/MCObjectFileInfo.h"
13 #include "llvm/MC/MCObjectStreamer.h"
14 #include "llvm/MC/MCStreamer.h"
15 #include "llvm/Support/Endian.h"
16 #include "llvm/Support/LEB128.h"
17 #include "llvm/Support/raw_ostream.h"
21 #define DEBUG_TYPE "mcpseudoprobe"
24 using namespace support
;
27 int MCPseudoProbeTable::DdgPrintIndent
= 0;
30 static const MCExpr
*buildSymbolDiff(MCObjectStreamer
*MCOS
, const MCSymbol
*A
,
32 MCContext
&Context
= MCOS
->getContext();
33 MCSymbolRefExpr::VariantKind Variant
= MCSymbolRefExpr::VK_None
;
34 const MCExpr
*ARef
= MCSymbolRefExpr::create(A
, Variant
, Context
);
35 const MCExpr
*BRef
= MCSymbolRefExpr::create(B
, Variant
, Context
);
36 const MCExpr
*AddrDelta
=
37 MCBinaryExpr::create(MCBinaryExpr::Sub
, ARef
, BRef
, Context
);
41 void MCPseudoProbe::emit(MCObjectStreamer
*MCOS
,
42 const MCPseudoProbe
*LastProbe
) const {
44 MCOS
->emitULEB128IntValue(Index
);
45 // Emit Type and the flag:
46 // Type (bit 0 to 3), with bit 4 to 6 for attributes.
47 // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
48 // the following field is a symbolic code address or an address delta.
49 assert(Type
<= 0xF && "Probe type too big to encode, exceeding 15");
50 assert(Attributes
<= 0x7 &&
51 "Probe attributes too big to encode, exceeding 7");
52 uint8_t PackedType
= Type
| (Attributes
<< 4);
53 uint8_t Flag
= LastProbe
? ((int8_t)MCPseudoProbeFlag::AddressDelta
<< 7) : 0;
54 MCOS
->emitInt8(Flag
| PackedType
);
57 // Emit the delta between the address label and LastProbe.
58 const MCExpr
*AddrDelta
=
59 buildSymbolDiff(MCOS
, Label
, LastProbe
->getLabel());
61 if (AddrDelta
->evaluateAsAbsolute(Delta
, MCOS
->getAssemblerPtr())) {
62 MCOS
->emitSLEB128IntValue(Delta
);
64 MCOS
->insert(new MCPseudoProbeAddrFragment(AddrDelta
));
67 // Emit label as a symbolic code address.
68 MCOS
->emitSymbolValue(
69 Label
, MCOS
->getContext().getAsmInfo()->getCodePointerSize());
73 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent
);
74 dbgs() << "Probe: " << Index
<< "\n";
78 void MCPseudoProbeInlineTree::addPseudoProbe(
79 const MCPseudoProbe
&Probe
, const MCPseudoProbeInlineStack
&InlineStack
) {
80 // The function should not be called on the root.
81 assert(isRoot() && "Should not be called on root");
83 // When it comes here, the input look like:
84 // Probe: GUID of C, ...
85 // InlineStack: [88, A], [66, B]
86 // which means, Function A inlines function B at call site with a probe id of
87 // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,
88 // A], [88, B], [66, C]} to locate the tree node where the probe should be
89 // added. Note that the edge [0, A] means A is the top-level function we are
90 // emitting probes for.
92 // Make a [0, A] edge.
93 // An empty inline stack means the function that the probe originates from
94 // is a top-level function.
96 if (InlineStack
.empty()) {
97 Top
= InlineSite(Probe
.getGuid(), 0);
99 Top
= InlineSite(std::get
<0>(InlineStack
.front()), 0);
102 auto *Cur
= getOrAddNode(Top
);
104 // Make interior edges by walking the inline stack. Once it's done, Cur should
105 // point to the node that the probe originates from.
106 if (!InlineStack
.empty()) {
107 auto Iter
= InlineStack
.begin();
108 auto Index
= std::get
<1>(*Iter
);
110 for (; Iter
!= InlineStack
.end(); Iter
++) {
111 // Make an edge by using the previous probe id and current GUID.
112 Cur
= Cur
->getOrAddNode(InlineSite(std::get
<0>(*Iter
), Index
));
113 Index
= std::get
<1>(*Iter
);
115 Cur
= Cur
->getOrAddNode(InlineSite(Probe
.getGuid(), Index
));
118 Cur
->Probes
.push_back(Probe
);
121 void MCPseudoProbeInlineTree::emit(MCObjectStreamer
*MCOS
,
122 const MCPseudoProbe
*&LastProbe
) {
124 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent
);
125 dbgs() << "Group [\n";
126 MCPseudoProbeTable::DdgPrintIndent
+= 2;
128 // Emit probes grouped by GUID.
131 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent
);
132 dbgs() << "GUID: " << Guid
<< "\n";
135 MCOS
->emitInt64(Guid
);
136 // Emit number of probes in this node
137 MCOS
->emitULEB128IntValue(Probes
.size());
138 // Emit number of direct inlinees
139 MCOS
->emitULEB128IntValue(Children
.size());
140 // Emit probes in this group
141 for (const auto &Probe
: Probes
) {
142 Probe
.emit(MCOS
, LastProbe
);
146 assert(Probes
.empty() && "Root should not have probes");
149 // Emit sorted descendant
150 // InlineSite is unique for each pair,
151 // so there will be no ordering of Inlinee based on MCPseudoProbeInlineTree*
152 std::map
<InlineSite
, MCPseudoProbeInlineTree
*> Inlinees
;
153 for (auto Child
= Children
.begin(); Child
!= Children
.end(); ++Child
)
154 Inlinees
[Child
->first
] = Child
->second
.get();
156 for (const auto &Inlinee
: Inlinees
) {
159 MCOS
->emitULEB128IntValue(std::get
<1>(Inlinee
.first
));
161 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent
);
162 dbgs() << "InlineSite: " << std::get
<1>(Inlinee
.first
) << "\n";
166 Inlinee
.second
->emit(MCOS
, LastProbe
);
170 MCPseudoProbeTable::DdgPrintIndent
-= 2;
171 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent
);
176 void MCPseudoProbeSection::emit(MCObjectStreamer
*MCOS
) {
177 MCContext
&Ctx
= MCOS
->getContext();
179 for (auto &ProbeSec
: MCProbeDivisions
) {
180 const MCPseudoProbe
*LastProbe
= nullptr;
182 Ctx
.getObjectFileInfo()->getPseudoProbeSection(ProbeSec
.first
)) {
183 // Switch to the .pseudoprobe section or a comdat group.
184 MCOS
->SwitchSection(S
);
185 // Emit probes grouped by GUID.
186 ProbeSec
.second
.emit(MCOS
, LastProbe
);
192 // This emits the pseudo probe tables.
194 void MCPseudoProbeTable::emit(MCObjectStreamer
*MCOS
) {
195 MCContext
&Ctx
= MCOS
->getContext();
196 auto &ProbeTable
= Ctx
.getMCPseudoProbeTable();
198 // Bail out early so we don't switch to the pseudo_probe section needlessly
199 // and in doing so create an unnecessary (if empty) section.
200 auto &ProbeSections
= ProbeTable
.getProbeSections();
201 if (ProbeSections
.empty())
204 LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent
= 0);
206 // Put out the probe.
207 ProbeSections
.emit(MCOS
);
210 static StringRef
getProbeFNameForGUID(const GUIDProbeFunctionMap
&GUID2FuncMAP
,
212 auto It
= GUID2FuncMAP
.find(GUID
);
213 assert(It
!= GUID2FuncMAP
.end() &&
214 "Probe function must exist for a valid GUID");
215 return It
->second
.FuncName
;
218 void MCPseudoProbeFuncDesc::print(raw_ostream
&OS
) {
219 OS
<< "GUID: " << FuncGUID
<< " Name: " << FuncName
<< "\n";
220 OS
<< "Hash: " << FuncHash
<< "\n";
223 void MCDecodedPseudoProbe::getInlineContext(
224 SmallVectorImpl
<std::string
> &ContextStack
,
225 const GUIDProbeFunctionMap
&GUID2FuncMAP
, bool ShowName
) const {
226 uint32_t Begin
= ContextStack
.size();
227 MCDecodedPseudoProbeInlineTree
*Cur
= InlineTree
;
228 // It will add the string of each node's inline site during iteration.
229 // Note that it won't include the probe's belonging function(leaf location)
230 while (Cur
->hasInlineSite()) {
231 std::string ContextStr
;
234 getProbeFNameForGUID(GUID2FuncMAP
, std::get
<0>(Cur
->ISite
));
235 ContextStr
+= FuncName
.str();
237 ContextStr
+= Twine(std::get
<0>(Cur
->ISite
)).str();
240 ContextStr
+= Twine(std::get
<1>(Cur
->ISite
)).str();
241 ContextStack
.emplace_back(ContextStr
);
242 Cur
= static_cast<MCDecodedPseudoProbeInlineTree
*>(Cur
->Parent
);
244 // Make the ContextStack in caller-callee order
245 std::reverse(ContextStack
.begin() + Begin
, ContextStack
.end());
248 std::string
MCDecodedPseudoProbe::getInlineContextStr(
249 const GUIDProbeFunctionMap
&GUID2FuncMAP
, bool ShowName
) const {
250 std::ostringstream OContextStr
;
251 SmallVector
<std::string
, 16> ContextStack
;
252 getInlineContext(ContextStack
, GUID2FuncMAP
, ShowName
);
253 for (auto &CxtStr
: ContextStack
) {
254 if (OContextStr
.str().size())
255 OContextStr
<< " @ ";
256 OContextStr
<< CxtStr
;
258 return OContextStr
.str();
261 static const char *PseudoProbeTypeStr
[3] = {"Block", "IndirectCall",
264 void MCDecodedPseudoProbe::print(raw_ostream
&OS
,
265 const GUIDProbeFunctionMap
&GUID2FuncMAP
,
266 bool ShowName
) const {
269 StringRef FuncName
= getProbeFNameForGUID(GUID2FuncMAP
, Guid
);
270 OS
<< FuncName
.str() << " ";
274 OS
<< "Index: " << Index
<< " ";
275 OS
<< "Type: " << PseudoProbeTypeStr
[static_cast<uint8_t>(Type
)] << " ";
276 std::string InlineContextStr
= getInlineContextStr(GUID2FuncMAP
, ShowName
);
277 if (InlineContextStr
.size()) {
279 OS
<< InlineContextStr
;
284 template <typename T
> ErrorOr
<T
> MCPseudoProbeDecoder::readUnencodedNumber() {
285 if (Data
+ sizeof(T
) > End
) {
286 return std::error_code();
288 T Val
= endian::readNext
<T
, little
, unaligned
>(Data
);
289 return ErrorOr
<T
>(Val
);
292 template <typename T
> ErrorOr
<T
> MCPseudoProbeDecoder::readUnsignedNumber() {
293 unsigned NumBytesRead
= 0;
294 uint64_t Val
= decodeULEB128(Data
, &NumBytesRead
);
295 if (Val
> std::numeric_limits
<T
>::max() || (Data
+ NumBytesRead
> End
)) {
296 return std::error_code();
298 Data
+= NumBytesRead
;
299 return ErrorOr
<T
>(static_cast<T
>(Val
));
302 template <typename T
> ErrorOr
<T
> MCPseudoProbeDecoder::readSignedNumber() {
303 unsigned NumBytesRead
= 0;
304 int64_t Val
= decodeSLEB128(Data
, &NumBytesRead
);
305 if (Val
> std::numeric_limits
<T
>::max() || (Data
+ NumBytesRead
> End
)) {
306 return std::error_code();
308 Data
+= NumBytesRead
;
309 return ErrorOr
<T
>(static_cast<T
>(Val
));
312 ErrorOr
<StringRef
> MCPseudoProbeDecoder::readString(uint32_t Size
) {
313 StringRef
Str(reinterpret_cast<const char *>(Data
), Size
);
314 if (Data
+ Size
> End
) {
315 return std::error_code();
318 return ErrorOr
<StringRef
>(Str
);
321 bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start
,
323 // The pseudo_probe_desc section has a format like:
324 // .section .pseudo_probe_desc,"",@progbits
325 // .quad -5182264717993193164 // GUID
326 // .quad 4294967295 // Hash
327 // .uleb 3 // Name size
328 // .ascii "foo" // Name
329 // .quad -2624081020897602054
330 // .quad 174696971957
338 auto ErrorOrGUID
= readUnencodedNumber
<uint64_t>();
342 auto ErrorOrHash
= readUnencodedNumber
<uint64_t>();
346 auto ErrorOrNameSize
= readUnsignedNumber
<uint32_t>();
347 if (!ErrorOrNameSize
)
349 uint32_t NameSize
= std::move(*ErrorOrNameSize
);
351 auto ErrorOrName
= readString(NameSize
);
355 uint64_t GUID
= std::move(*ErrorOrGUID
);
356 uint64_t Hash
= std::move(*ErrorOrHash
);
357 StringRef Name
= std::move(*ErrorOrName
);
359 // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap
360 GUID2FuncDescMap
.emplace(GUID
, MCPseudoProbeFuncDesc(GUID
, Hash
, Name
));
362 assert(Data
== End
&& "Have unprocessed data in pseudo_probe_desc section");
366 bool MCPseudoProbeDecoder::buildAddress2ProbeMap(const uint8_t *Start
,
368 // The pseudo_probe section encodes an inline forest and each tree has a
370 // FUNCTION BODY (one for each uninlined function present in the text
373 // GUID of the function
375 // Number of probes originating from this function.
376 // NUM_INLINED_FUNCTIONS (ULEB128)
377 // Number of callees inlined into this function, aka number of
378 // first-level inlinees
380 // A list of NPROBES entries. Each entry contains:
383 // 0 - block probe, 1 - indirect call, 2 - direct call
385 // 1 - tail call, 2 - dangling
386 // ADDRESS_TYPE (uint1)
387 // 0 - code address, 1 - address delta
388 // CODE_ADDRESS (uint64 or ULEB128)
389 // code address or address delta, depending on Flag
390 // INLINED FUNCTION RECORDS
391 // A list of NUM_INLINED_FUNCTIONS entries describing each of the
392 // inlined callees. Each record contains:
394 // Index of the callsite probe (ULEB128)
396 // A FUNCTION BODY entry describing the inlined function.
401 MCDecodedPseudoProbeInlineTree
*Root
= &DummyInlineRoot
;
402 MCDecodedPseudoProbeInlineTree
*Cur
= &DummyInlineRoot
;
403 uint64_t LastAddr
= 0;
405 // A DFS-based decoding
408 // Use a sequential id for top level inliner.
409 Index
= Root
->getChildren().size();
411 // Read inline site for inlinees
412 auto ErrorOrIndex
= readUnsignedNumber
<uint32_t>();
415 Index
= std::move(*ErrorOrIndex
);
417 // Switch/add to a new tree node(inlinee)
418 Cur
= Cur
->getOrAddNode(std::make_tuple(Cur
->Guid
, Index
));
420 auto ErrorOrCurGuid
= readUnencodedNumber
<uint64_t>();
423 Cur
->Guid
= std::move(*ErrorOrCurGuid
);
424 // Read number of probes in the current node.
425 auto ErrorOrNodeCount
= readUnsignedNumber
<uint32_t>();
426 if (!ErrorOrNodeCount
)
428 uint32_t NodeCount
= std::move(*ErrorOrNodeCount
);
429 // Read number of direct inlinees
430 auto ErrorOrCurChildrenToProcess
= readUnsignedNumber
<uint32_t>();
431 if (!ErrorOrCurChildrenToProcess
)
433 Cur
->ChildrenToProcess
= std::move(*ErrorOrCurChildrenToProcess
);
434 // Read all probes in this node
435 for (std::size_t I
= 0; I
< NodeCount
; I
++) {
437 auto ErrorOrIndex
= readUnsignedNumber
<uint32_t>();
440 uint32_t Index
= std::move(*ErrorOrIndex
);
442 auto ErrorOrValue
= readUnencodedNumber
<uint8_t>();
445 uint8_t Value
= std::move(*ErrorOrValue
);
446 uint8_t Kind
= Value
& 0xf;
447 uint8_t Attr
= (Value
& 0x70) >> 4;
451 auto ErrorOrOffset
= readSignedNumber
<int64_t>();
454 int64_t Offset
= std::move(*ErrorOrOffset
);
455 Addr
= LastAddr
+ Offset
;
457 auto ErrorOrAddr
= readUnencodedNumber
<int64_t>();
460 Addr
= std::move(*ErrorOrAddr
);
462 // Populate Address2ProbesMap
463 auto &Probes
= Address2ProbesMap
[Addr
];
464 Probes
.emplace_back(Addr
, Cur
->Guid
, Index
, PseudoProbeType(Kind
), Attr
,
466 Cur
->addProbes(&Probes
.back());
470 // Look for the parent for the next node by subtracting the current
471 // node count from tree counts along the parent chain. The first node
472 // in the chain that has a non-zero tree count is the target.
473 while (Cur
!= Root
) {
474 if (Cur
->ChildrenToProcess
== 0) {
475 Cur
= static_cast<MCDecodedPseudoProbeInlineTree
*>(Cur
->Parent
);
477 assert(Cur
->ChildrenToProcess
> 0 &&
478 "Should have some unprocessed nodes");
479 Cur
->ChildrenToProcess
-= 1;
487 assert(Data
== End
&& "Have unprocessed data in pseudo_probe section");
488 assert(Cur
== Root
&&
489 " Cur should point to root when the forest is fully built up");
493 void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream
&OS
) {
494 OS
<< "Pseudo Probe Desc:\n";
495 // Make the output deterministic
496 std::map
<uint64_t, MCPseudoProbeFuncDesc
> OrderedMap(GUID2FuncDescMap
.begin(),
497 GUID2FuncDescMap
.end());
498 for (auto &I
: OrderedMap
) {
503 void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream
&OS
,
505 auto It
= Address2ProbesMap
.find(Address
);
506 if (It
!= Address2ProbesMap
.end()) {
507 for (auto &Probe
: It
->second
) {
509 Probe
.print(OS
, GUID2FuncDescMap
, true);
514 void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream
&OS
) {
515 std::vector
<uint64_t> Addresses
;
516 for (auto Entry
: Address2ProbesMap
)
517 Addresses
.push_back(Entry
.first
);
518 std::sort(Addresses
.begin(), Addresses
.end());
519 for (auto K
: Addresses
) {
523 printProbeForAddress(OS
, K
);
527 const MCDecodedPseudoProbe
*
528 MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address
) const {
529 auto It
= Address2ProbesMap
.find(Address
);
530 if (It
== Address2ProbesMap
.end())
532 const auto &Probes
= It
->second
;
534 const MCDecodedPseudoProbe
*CallProbe
= nullptr;
535 for (const auto &Probe
: Probes
) {
536 if (Probe
.isCall()) {
538 "There should be only one call probe corresponding to address "
539 "which is a callsite.");
546 const MCPseudoProbeFuncDesc
*
547 MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID
) const {
548 auto It
= GUID2FuncDescMap
.find(GUID
);
549 assert(It
!= GUID2FuncDescMap
.end() && "Function descriptor doesn't exist");
553 void MCPseudoProbeDecoder::getInlineContextForProbe(
554 const MCDecodedPseudoProbe
*Probe
,
555 SmallVectorImpl
<std::string
> &InlineContextStack
, bool IncludeLeaf
) const {
556 Probe
->getInlineContext(InlineContextStack
, GUID2FuncDescMap
, true);
559 // Note that the context from probe doesn't include leaf frame,
560 // hence we need to retrieve and prepend leaf if requested.
561 const auto *FuncDesc
= getFuncDescForGUID(Probe
->getGuid());
562 InlineContextStack
.emplace_back(FuncDesc
->FuncName
+ ":" +
563 Twine(Probe
->getIndex()).str());
566 const MCPseudoProbeFuncDesc
*MCPseudoProbeDecoder::getInlinerDescForProbe(
567 const MCDecodedPseudoProbe
*Probe
) const {
568 MCDecodedPseudoProbeInlineTree
*InlinerNode
= Probe
->getInlineTreeNode();
569 if (!InlinerNode
->hasInlineSite())
571 return getFuncDescForGUID(std::get
<0>(InlinerNode
->ISite
));