[ORC] Add std::tuple support to SimplePackedSerialization.
[llvm-project.git] / llvm / lib / MC / MCPseudoProbe.cpp
blob965ffd260e1bfc9695cbda6a78999a81aa2ff479
1 //===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
9 #include "llvm/MC/MCPseudoProbe.h"
10 #include "llvm/MC/MCAsmInfo.h"
11 #include "llvm/MC/MCContext.h"
12 #include "llvm/MC/MCObjectFileInfo.h"
13 #include "llvm/MC/MCObjectStreamer.h"
14 #include "llvm/MC/MCStreamer.h"
15 #include "llvm/Support/Endian.h"
16 #include "llvm/Support/LEB128.h"
17 #include "llvm/Support/raw_ostream.h"
18 #include <limits>
19 #include <memory>
21 #define DEBUG_TYPE "mcpseudoprobe"
23 using namespace llvm;
24 using namespace support;
26 #ifndef NDEBUG
27 int MCPseudoProbeTable::DdgPrintIndent = 0;
28 #endif
30 static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,
31 const MCSymbol *B) {
32 MCContext &Context = MCOS->getContext();
33 MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
34 const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
35 const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
36 const MCExpr *AddrDelta =
37 MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);
38 return AddrDelta;
41 void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
42 const MCPseudoProbe *LastProbe) const {
43 // Emit Index
44 MCOS->emitULEB128IntValue(Index);
45 // Emit Type and the flag:
46 // Type (bit 0 to 3), with bit 4 to 6 for attributes.
47 // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
48 // the following field is a symbolic code address or an address delta.
49 assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
50 assert(Attributes <= 0x7 &&
51 "Probe attributes too big to encode, exceeding 7");
52 uint8_t PackedType = Type | (Attributes << 4);
53 uint8_t Flag = LastProbe ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
54 MCOS->emitInt8(Flag | PackedType);
56 if (LastProbe) {
57 // Emit the delta between the address label and LastProbe.
58 const MCExpr *AddrDelta =
59 buildSymbolDiff(MCOS, Label, LastProbe->getLabel());
60 int64_t Delta;
61 if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) {
62 MCOS->emitSLEB128IntValue(Delta);
63 } else {
64 MCOS->insert(new MCPseudoProbeAddrFragment(AddrDelta));
66 } else {
67 // Emit label as a symbolic code address.
68 MCOS->emitSymbolValue(
69 Label, MCOS->getContext().getAsmInfo()->getCodePointerSize());
72 LLVM_DEBUG({
73 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
74 dbgs() << "Probe: " << Index << "\n";
75 });
78 void MCPseudoProbeInlineTree::addPseudoProbe(
79 const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) {
80 // The function should not be called on the root.
81 assert(isRoot() && "Should not be called on root");
83 // When it comes here, the input look like:
84 // Probe: GUID of C, ...
85 // InlineStack: [88, A], [66, B]
86 // which means, Function A inlines function B at call site with a probe id of
87 // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,
88 // A], [88, B], [66, C]} to locate the tree node where the probe should be
89 // added. Note that the edge [0, A] means A is the top-level function we are
90 // emitting probes for.
92 // Make a [0, A] edge.
93 // An empty inline stack means the function that the probe originates from
94 // is a top-level function.
95 InlineSite Top;
96 if (InlineStack.empty()) {
97 Top = InlineSite(Probe.getGuid(), 0);
98 } else {
99 Top = InlineSite(std::get<0>(InlineStack.front()), 0);
102 auto *Cur = getOrAddNode(Top);
104 // Make interior edges by walking the inline stack. Once it's done, Cur should
105 // point to the node that the probe originates from.
106 if (!InlineStack.empty()) {
107 auto Iter = InlineStack.begin();
108 auto Index = std::get<1>(*Iter);
109 Iter++;
110 for (; Iter != InlineStack.end(); Iter++) {
111 // Make an edge by using the previous probe id and current GUID.
112 Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index));
113 Index = std::get<1>(*Iter);
115 Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index));
118 Cur->Probes.push_back(Probe);
121 void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,
122 const MCPseudoProbe *&LastProbe) {
123 LLVM_DEBUG({
124 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
125 dbgs() << "Group [\n";
126 MCPseudoProbeTable::DdgPrintIndent += 2;
128 // Emit probes grouped by GUID.
129 if (Guid != 0) {
130 LLVM_DEBUG({
131 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
132 dbgs() << "GUID: " << Guid << "\n";
134 // Emit Guid
135 MCOS->emitInt64(Guid);
136 // Emit number of probes in this node
137 MCOS->emitULEB128IntValue(Probes.size());
138 // Emit number of direct inlinees
139 MCOS->emitULEB128IntValue(Children.size());
140 // Emit probes in this group
141 for (const auto &Probe : Probes) {
142 Probe.emit(MCOS, LastProbe);
143 LastProbe = &Probe;
145 } else {
146 assert(Probes.empty() && "Root should not have probes");
149 // Emit sorted descendant
150 // InlineSite is unique for each pair,
151 // so there will be no ordering of Inlinee based on MCPseudoProbeInlineTree*
152 std::map<InlineSite, MCPseudoProbeInlineTree *> Inlinees;
153 for (auto Child = Children.begin(); Child != Children.end(); ++Child)
154 Inlinees[Child->first] = Child->second.get();
156 for (const auto &Inlinee : Inlinees) {
157 if (Guid) {
158 // Emit probe index
159 MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first));
160 LLVM_DEBUG({
161 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
162 dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n";
165 // Emit the group
166 Inlinee.second->emit(MCOS, LastProbe);
169 LLVM_DEBUG({
170 MCPseudoProbeTable::DdgPrintIndent -= 2;
171 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
172 dbgs() << "]\n";
176 void MCPseudoProbeSection::emit(MCObjectStreamer *MCOS) {
177 MCContext &Ctx = MCOS->getContext();
179 for (auto &ProbeSec : MCProbeDivisions) {
180 const MCPseudoProbe *LastProbe = nullptr;
181 if (auto *S =
182 Ctx.getObjectFileInfo()->getPseudoProbeSection(ProbeSec.first)) {
183 // Switch to the .pseudoprobe section or a comdat group.
184 MCOS->SwitchSection(S);
185 // Emit probes grouped by GUID.
186 ProbeSec.second.emit(MCOS, LastProbe);
192 // This emits the pseudo probe tables.
194 void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) {
195 MCContext &Ctx = MCOS->getContext();
196 auto &ProbeTable = Ctx.getMCPseudoProbeTable();
198 // Bail out early so we don't switch to the pseudo_probe section needlessly
199 // and in doing so create an unnecessary (if empty) section.
200 auto &ProbeSections = ProbeTable.getProbeSections();
201 if (ProbeSections.empty())
202 return;
204 LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0);
206 // Put out the probe.
207 ProbeSections.emit(MCOS);
210 static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP,
211 uint64_t GUID) {
212 auto It = GUID2FuncMAP.find(GUID);
213 assert(It != GUID2FuncMAP.end() &&
214 "Probe function must exist for a valid GUID");
215 return It->second.FuncName;
218 void MCPseudoProbeFuncDesc::print(raw_ostream &OS) {
219 OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n";
220 OS << "Hash: " << FuncHash << "\n";
223 void MCDecodedPseudoProbe::getInlineContext(
224 SmallVectorImpl<std::string> &ContextStack,
225 const GUIDProbeFunctionMap &GUID2FuncMAP, bool ShowName) const {
226 uint32_t Begin = ContextStack.size();
227 MCDecodedPseudoProbeInlineTree *Cur = InlineTree;
228 // It will add the string of each node's inline site during iteration.
229 // Note that it won't include the probe's belonging function(leaf location)
230 while (Cur->hasInlineSite()) {
231 std::string ContextStr;
232 if (ShowName) {
233 StringRef FuncName =
234 getProbeFNameForGUID(GUID2FuncMAP, std::get<0>(Cur->ISite));
235 ContextStr += FuncName.str();
236 } else {
237 ContextStr += Twine(std::get<0>(Cur->ISite)).str();
239 ContextStr += ":";
240 ContextStr += Twine(std::get<1>(Cur->ISite)).str();
241 ContextStack.emplace_back(ContextStr);
242 Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent);
244 // Make the ContextStack in caller-callee order
245 std::reverse(ContextStack.begin() + Begin, ContextStack.end());
248 std::string MCDecodedPseudoProbe::getInlineContextStr(
249 const GUIDProbeFunctionMap &GUID2FuncMAP, bool ShowName) const {
250 std::ostringstream OContextStr;
251 SmallVector<std::string, 16> ContextStack;
252 getInlineContext(ContextStack, GUID2FuncMAP, ShowName);
253 for (auto &CxtStr : ContextStack) {
254 if (OContextStr.str().size())
255 OContextStr << " @ ";
256 OContextStr << CxtStr;
258 return OContextStr.str();
261 static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall",
262 "DirectCall"};
264 void MCDecodedPseudoProbe::print(raw_ostream &OS,
265 const GUIDProbeFunctionMap &GUID2FuncMAP,
266 bool ShowName) const {
267 OS << "FUNC: ";
268 if (ShowName) {
269 StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Guid);
270 OS << FuncName.str() << " ";
271 } else {
272 OS << Guid << " ";
274 OS << "Index: " << Index << " ";
275 OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " ";
276 std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP, ShowName);
277 if (InlineContextStr.size()) {
278 OS << "Inlined: @ ";
279 OS << InlineContextStr;
281 OS << "\n";
284 template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() {
285 if (Data + sizeof(T) > End) {
286 return std::error_code();
288 T Val = endian::readNext<T, little, unaligned>(Data);
289 return ErrorOr<T>(Val);
292 template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() {
293 unsigned NumBytesRead = 0;
294 uint64_t Val = decodeULEB128(Data, &NumBytesRead);
295 if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
296 return std::error_code();
298 Data += NumBytesRead;
299 return ErrorOr<T>(static_cast<T>(Val));
302 template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() {
303 unsigned NumBytesRead = 0;
304 int64_t Val = decodeSLEB128(Data, &NumBytesRead);
305 if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {
306 return std::error_code();
308 Data += NumBytesRead;
309 return ErrorOr<T>(static_cast<T>(Val));
312 ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) {
313 StringRef Str(reinterpret_cast<const char *>(Data), Size);
314 if (Data + Size > End) {
315 return std::error_code();
317 Data += Size;
318 return ErrorOr<StringRef>(Str);
321 bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,
322 std::size_t Size) {
323 // The pseudo_probe_desc section has a format like:
324 // .section .pseudo_probe_desc,"",@progbits
325 // .quad -5182264717993193164 // GUID
326 // .quad 4294967295 // Hash
327 // .uleb 3 // Name size
328 // .ascii "foo" // Name
329 // .quad -2624081020897602054
330 // .quad 174696971957
331 // .uleb 34
332 // .ascii "main"
334 Data = Start;
335 End = Data + Size;
337 while (Data < End) {
338 auto ErrorOrGUID = readUnencodedNumber<uint64_t>();
339 if (!ErrorOrGUID)
340 return false;
342 auto ErrorOrHash = readUnencodedNumber<uint64_t>();
343 if (!ErrorOrHash)
344 return false;
346 auto ErrorOrNameSize = readUnsignedNumber<uint32_t>();
347 if (!ErrorOrNameSize)
348 return false;
349 uint32_t NameSize = std::move(*ErrorOrNameSize);
351 auto ErrorOrName = readString(NameSize);
352 if (!ErrorOrName)
353 return false;
355 uint64_t GUID = std::move(*ErrorOrGUID);
356 uint64_t Hash = std::move(*ErrorOrHash);
357 StringRef Name = std::move(*ErrorOrName);
359 // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap
360 GUID2FuncDescMap.emplace(GUID, MCPseudoProbeFuncDesc(GUID, Hash, Name));
362 assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");
363 return true;
366 bool MCPseudoProbeDecoder::buildAddress2ProbeMap(const uint8_t *Start,
367 std::size_t Size) {
368 // The pseudo_probe section encodes an inline forest and each tree has a
369 // format like:
370 // FUNCTION BODY (one for each uninlined function present in the text
371 // section)
372 // GUID (uint64)
373 // GUID of the function
374 // NPROBES (ULEB128)
375 // Number of probes originating from this function.
376 // NUM_INLINED_FUNCTIONS (ULEB128)
377 // Number of callees inlined into this function, aka number of
378 // first-level inlinees
379 // PROBE RECORDS
380 // A list of NPROBES entries. Each entry contains:
381 // INDEX (ULEB128)
382 // TYPE (uint4)
383 // 0 - block probe, 1 - indirect call, 2 - direct call
384 // ATTRIBUTE (uint3)
385 // 1 - tail call, 2 - dangling
386 // ADDRESS_TYPE (uint1)
387 // 0 - code address, 1 - address delta
388 // CODE_ADDRESS (uint64 or ULEB128)
389 // code address or address delta, depending on Flag
390 // INLINED FUNCTION RECORDS
391 // A list of NUM_INLINED_FUNCTIONS entries describing each of the
392 // inlined callees. Each record contains:
393 // INLINE SITE
394 // Index of the callsite probe (ULEB128)
395 // FUNCTION BODY
396 // A FUNCTION BODY entry describing the inlined function.
398 Data = Start;
399 End = Data + Size;
401 MCDecodedPseudoProbeInlineTree *Root = &DummyInlineRoot;
402 MCDecodedPseudoProbeInlineTree *Cur = &DummyInlineRoot;
403 uint64_t LastAddr = 0;
404 uint32_t Index = 0;
405 // A DFS-based decoding
406 while (Data < End) {
407 if (Root == Cur) {
408 // Use a sequential id for top level inliner.
409 Index = Root->getChildren().size();
410 } else {
411 // Read inline site for inlinees
412 auto ErrorOrIndex = readUnsignedNumber<uint32_t>();
413 if (!ErrorOrIndex)
414 return false;
415 Index = std::move(*ErrorOrIndex);
417 // Switch/add to a new tree node(inlinee)
418 Cur = Cur->getOrAddNode(std::make_tuple(Cur->Guid, Index));
419 // Read guid
420 auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>();
421 if (!ErrorOrCurGuid)
422 return false;
423 Cur->Guid = std::move(*ErrorOrCurGuid);
424 // Read number of probes in the current node.
425 auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>();
426 if (!ErrorOrNodeCount)
427 return false;
428 uint32_t NodeCount = std::move(*ErrorOrNodeCount);
429 // Read number of direct inlinees
430 auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>();
431 if (!ErrorOrCurChildrenToProcess)
432 return false;
433 Cur->ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess);
434 // Read all probes in this node
435 for (std::size_t I = 0; I < NodeCount; I++) {
436 // Read index
437 auto ErrorOrIndex = readUnsignedNumber<uint32_t>();
438 if (!ErrorOrIndex)
439 return false;
440 uint32_t Index = std::move(*ErrorOrIndex);
441 // Read type | flag.
442 auto ErrorOrValue = readUnencodedNumber<uint8_t>();
443 if (!ErrorOrValue)
444 return false;
445 uint8_t Value = std::move(*ErrorOrValue);
446 uint8_t Kind = Value & 0xf;
447 uint8_t Attr = (Value & 0x70) >> 4;
448 // Read address
449 uint64_t Addr = 0;
450 if (Value & 0x80) {
451 auto ErrorOrOffset = readSignedNumber<int64_t>();
452 if (!ErrorOrOffset)
453 return false;
454 int64_t Offset = std::move(*ErrorOrOffset);
455 Addr = LastAddr + Offset;
456 } else {
457 auto ErrorOrAddr = readUnencodedNumber<int64_t>();
458 if (!ErrorOrAddr)
459 return false;
460 Addr = std::move(*ErrorOrAddr);
462 // Populate Address2ProbesMap
463 auto &Probes = Address2ProbesMap[Addr];
464 Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr,
465 Cur);
466 Cur->addProbes(&Probes.back());
467 LastAddr = Addr;
470 // Look for the parent for the next node by subtracting the current
471 // node count from tree counts along the parent chain. The first node
472 // in the chain that has a non-zero tree count is the target.
473 while (Cur != Root) {
474 if (Cur->ChildrenToProcess == 0) {
475 Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent);
476 if (Cur != Root) {
477 assert(Cur->ChildrenToProcess > 0 &&
478 "Should have some unprocessed nodes");
479 Cur->ChildrenToProcess -= 1;
481 } else {
482 break;
487 assert(Data == End && "Have unprocessed data in pseudo_probe section");
488 assert(Cur == Root &&
489 " Cur should point to root when the forest is fully built up");
490 return true;
493 void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) {
494 OS << "Pseudo Probe Desc:\n";
495 // Make the output deterministic
496 std::map<uint64_t, MCPseudoProbeFuncDesc> OrderedMap(GUID2FuncDescMap.begin(),
497 GUID2FuncDescMap.end());
498 for (auto &I : OrderedMap) {
499 I.second.print(OS);
503 void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS,
504 uint64_t Address) {
505 auto It = Address2ProbesMap.find(Address);
506 if (It != Address2ProbesMap.end()) {
507 for (auto &Probe : It->second) {
508 OS << " [Probe]:\t";
509 Probe.print(OS, GUID2FuncDescMap, true);
514 void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) {
515 std::vector<uint64_t> Addresses;
516 for (auto Entry : Address2ProbesMap)
517 Addresses.push_back(Entry.first);
518 std::sort(Addresses.begin(), Addresses.end());
519 for (auto K : Addresses) {
520 OS << "Address:\t";
521 OS << K;
522 OS << "\n";
523 printProbeForAddress(OS, K);
527 const MCDecodedPseudoProbe *
528 MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const {
529 auto It = Address2ProbesMap.find(Address);
530 if (It == Address2ProbesMap.end())
531 return nullptr;
532 const auto &Probes = It->second;
534 const MCDecodedPseudoProbe *CallProbe = nullptr;
535 for (const auto &Probe : Probes) {
536 if (Probe.isCall()) {
537 assert(!CallProbe &&
538 "There should be only one call probe corresponding to address "
539 "which is a callsite.");
540 CallProbe = &Probe;
543 return CallProbe;
546 const MCPseudoProbeFuncDesc *
547 MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const {
548 auto It = GUID2FuncDescMap.find(GUID);
549 assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist");
550 return &It->second;
553 void MCPseudoProbeDecoder::getInlineContextForProbe(
554 const MCDecodedPseudoProbe *Probe,
555 SmallVectorImpl<std::string> &InlineContextStack, bool IncludeLeaf) const {
556 Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap, true);
557 if (!IncludeLeaf)
558 return;
559 // Note that the context from probe doesn't include leaf frame,
560 // hence we need to retrieve and prepend leaf if requested.
561 const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid());
562 InlineContextStack.emplace_back(FuncDesc->FuncName + ":" +
563 Twine(Probe->getIndex()).str());
566 const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe(
567 const MCDecodedPseudoProbe *Probe) const {
568 MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode();
569 if (!InlinerNode->hasInlineSite())
570 return nullptr;
571 return getFuncDescForGUID(std::get<0>(InlinerNode->ISite));