1 //===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains support for reading profiling data for clang's
10 // instrumentation based PGO and coverage.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/ProfileData/InstrProfReader.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/IR/ProfileSummary.h"
20 #include "llvm/ProfileData/InstrProf.h"
21 #include "llvm/ProfileData/ProfileCommon.h"
22 #include "llvm/Support/Endian.h"
23 #include "llvm/Support/Error.h"
24 #include "llvm/Support/ErrorOr.h"
25 #include "llvm/Support/MemoryBuffer.h"
26 #include "llvm/Support/SymbolRemappingReader.h"
27 #include "llvm/Support/SwapByteOrder.h"
34 #include <system_error>
40 static Expected
<std::unique_ptr
<MemoryBuffer
>>
41 setupMemoryBuffer(const Twine
&Path
) {
42 ErrorOr
<std::unique_ptr
<MemoryBuffer
>> BufferOrErr
=
43 MemoryBuffer::getFileOrSTDIN(Path
);
44 if (std::error_code EC
= BufferOrErr
.getError())
45 return errorCodeToError(EC
);
46 return std::move(BufferOrErr
.get());
49 static Error
initializeReader(InstrProfReader
&Reader
) {
50 return Reader
.readHeader();
53 Expected
<std::unique_ptr
<InstrProfReader
>>
54 InstrProfReader::create(const Twine
&Path
) {
55 // Set up the buffer to read.
56 auto BufferOrError
= setupMemoryBuffer(Path
);
57 if (Error E
= BufferOrError
.takeError())
59 return InstrProfReader::create(std::move(BufferOrError
.get()));
62 Expected
<std::unique_ptr
<InstrProfReader
>>
63 InstrProfReader::create(std::unique_ptr
<MemoryBuffer
> Buffer
) {
64 // Sanity check the buffer.
65 if (uint64_t(Buffer
->getBufferSize()) > std::numeric_limits
<uint64_t>::max())
66 return make_error
<InstrProfError
>(instrprof_error::too_large
);
68 if (Buffer
->getBufferSize() == 0)
69 return make_error
<InstrProfError
>(instrprof_error::empty_raw_profile
);
71 std::unique_ptr
<InstrProfReader
> Result
;
73 if (IndexedInstrProfReader::hasFormat(*Buffer
))
74 Result
.reset(new IndexedInstrProfReader(std::move(Buffer
)));
75 else if (RawInstrProfReader64::hasFormat(*Buffer
))
76 Result
.reset(new RawInstrProfReader64(std::move(Buffer
)));
77 else if (RawInstrProfReader32::hasFormat(*Buffer
))
78 Result
.reset(new RawInstrProfReader32(std::move(Buffer
)));
79 else if (TextInstrProfReader::hasFormat(*Buffer
))
80 Result
.reset(new TextInstrProfReader(std::move(Buffer
)));
82 return make_error
<InstrProfError
>(instrprof_error::unrecognized_format
);
84 // Initialize the reader and return the result.
85 if (Error E
= initializeReader(*Result
))
88 return std::move(Result
);
91 Expected
<std::unique_ptr
<IndexedInstrProfReader
>>
92 IndexedInstrProfReader::create(const Twine
&Path
, const Twine
&RemappingPath
) {
93 // Set up the buffer to read.
94 auto BufferOrError
= setupMemoryBuffer(Path
);
95 if (Error E
= BufferOrError
.takeError())
98 // Set up the remapping buffer if requested.
99 std::unique_ptr
<MemoryBuffer
> RemappingBuffer
;
100 std::string RemappingPathStr
= RemappingPath
.str();
101 if (!RemappingPathStr
.empty()) {
102 auto RemappingBufferOrError
= setupMemoryBuffer(RemappingPathStr
);
103 if (Error E
= RemappingBufferOrError
.takeError())
105 RemappingBuffer
= std::move(RemappingBufferOrError
.get());
108 return IndexedInstrProfReader::create(std::move(BufferOrError
.get()),
109 std::move(RemappingBuffer
));
112 Expected
<std::unique_ptr
<IndexedInstrProfReader
>>
113 IndexedInstrProfReader::create(std::unique_ptr
<MemoryBuffer
> Buffer
,
114 std::unique_ptr
<MemoryBuffer
> RemappingBuffer
) {
115 // Sanity check the buffer.
116 if (uint64_t(Buffer
->getBufferSize()) > std::numeric_limits
<uint64_t>::max())
117 return make_error
<InstrProfError
>(instrprof_error::too_large
);
119 // Create the reader.
120 if (!IndexedInstrProfReader::hasFormat(*Buffer
))
121 return make_error
<InstrProfError
>(instrprof_error::bad_magic
);
122 auto Result
= std::make_unique
<IndexedInstrProfReader
>(
123 std::move(Buffer
), std::move(RemappingBuffer
));
125 // Initialize the reader and return the result.
126 if (Error E
= initializeReader(*Result
))
129 return std::move(Result
);
132 void InstrProfIterator::Increment() {
133 if (auto E
= Reader
->readNextRecord(Record
)) {
134 // Handle errors in the reader.
135 InstrProfError::take(std::move(E
));
136 *this = InstrProfIterator();
140 bool TextInstrProfReader::hasFormat(const MemoryBuffer
&Buffer
) {
141 // Verify that this really looks like plain ASCII text by checking a
142 // 'reasonable' number of characters (up to profile magic size).
143 size_t count
= std::min(Buffer
.getBufferSize(), sizeof(uint64_t));
144 StringRef buffer
= Buffer
.getBufferStart();
146 std::all_of(buffer
.begin(), buffer
.begin() + count
,
147 [](char c
) { return isPrint(c
) || ::isspace(c
); });
150 // Read the profile variant flag from the header: ":FE" means this is a FE
151 // generated profile. ":IR" means this is an IR level profile. Other strings
152 // with a leading ':' will be reported an error format.
153 Error
TextInstrProfReader::readHeader() {
154 Symtab
.reset(new InstrProfSymtab());
155 bool IsIRInstr
= false;
156 if (!Line
->startswith(":")) {
157 IsIRLevelProfile
= false;
160 StringRef Str
= (Line
)->substr(1);
161 if (Str
.equals_lower("ir"))
163 else if (Str
.equals_lower("fe"))
165 else if (Str
.equals_lower("csir")) {
167 HasCSIRLevelProfile
= true;
169 return error(instrprof_error::bad_header
);
172 IsIRLevelProfile
= IsIRInstr
;
177 TextInstrProfReader::readValueProfileData(InstrProfRecord
&Record
) {
179 #define CHECK_LINE_END(Line) \
180 if (Line.is_at_end()) \
181 return error(instrprof_error::truncated);
182 #define READ_NUM(Str, Dst) \
183 if ((Str).getAsInteger(10, (Dst))) \
184 return error(instrprof_error::malformed);
185 #define VP_READ_ADVANCE(Val) \
186 CHECK_LINE_END(Line); \
188 READ_NUM((*Line), (Val)); \
191 if (Line
.is_at_end())
194 uint32_t NumValueKinds
;
195 if (Line
->getAsInteger(10, NumValueKinds
)) {
196 // No value profile data
199 if (NumValueKinds
== 0 || NumValueKinds
> IPVK_Last
+ 1)
200 return error(instrprof_error::malformed
);
203 for (uint32_t VK
= 0; VK
< NumValueKinds
; VK
++) {
204 VP_READ_ADVANCE(ValueKind
);
205 if (ValueKind
> IPVK_Last
)
206 return error(instrprof_error::malformed
);
207 VP_READ_ADVANCE(NumValueSites
);
211 Record
.reserveSites(VK
, NumValueSites
);
212 for (uint32_t S
= 0; S
< NumValueSites
; S
++) {
213 VP_READ_ADVANCE(NumValueData
);
215 std::vector
<InstrProfValueData
> CurrentValues
;
216 for (uint32_t V
= 0; V
< NumValueData
; V
++) {
217 CHECK_LINE_END(Line
);
218 std::pair
<StringRef
, StringRef
> VD
= Line
->rsplit(':');
219 uint64_t TakenCount
, Value
;
220 if (ValueKind
== IPVK_IndirectCallTarget
) {
221 if (InstrProfSymtab::isExternalSymbol(VD
.first
)) {
224 if (Error E
= Symtab
->addFuncName(VD
.first
))
226 Value
= IndexedInstrProf::ComputeHash(VD
.first
);
229 READ_NUM(VD
.first
, Value
);
231 READ_NUM(VD
.second
, TakenCount
);
232 CurrentValues
.push_back({Value
, TakenCount
});
235 Record
.addValueData(ValueKind
, S
, CurrentValues
.data(), NumValueData
,
241 #undef CHECK_LINE_END
243 #undef VP_READ_ADVANCE
246 Error
TextInstrProfReader::readNextRecord(NamedInstrProfRecord
&Record
) {
247 // Skip empty lines and comments.
248 while (!Line
.is_at_end() && (Line
->empty() || Line
->startswith("#")))
250 // If we hit EOF while looking for a name, we're done.
251 if (Line
.is_at_end()) {
252 return error(instrprof_error::eof
);
255 // Read the function name.
256 Record
.Name
= *Line
++;
257 if (Error E
= Symtab
->addFuncName(Record
.Name
))
258 return error(std::move(E
));
260 // Read the function hash.
261 if (Line
.is_at_end())
262 return error(instrprof_error::truncated
);
263 if ((Line
++)->getAsInteger(0, Record
.Hash
))
264 return error(instrprof_error::malformed
);
266 // Read the number of counters.
267 uint64_t NumCounters
;
268 if (Line
.is_at_end())
269 return error(instrprof_error::truncated
);
270 if ((Line
++)->getAsInteger(10, NumCounters
))
271 return error(instrprof_error::malformed
);
272 if (NumCounters
== 0)
273 return error(instrprof_error::malformed
);
275 // Read each counter and fill our internal storage with the values.
277 Record
.Counts
.reserve(NumCounters
);
278 for (uint64_t I
= 0; I
< NumCounters
; ++I
) {
279 if (Line
.is_at_end())
280 return error(instrprof_error::truncated
);
282 if ((Line
++)->getAsInteger(10, Count
))
283 return error(instrprof_error::malformed
);
284 Record
.Counts
.push_back(Count
);
287 // Check if value profile data exists and read it if so.
288 if (Error E
= readValueProfileData(Record
))
289 return error(std::move(E
));
294 template <class IntPtrT
>
295 bool RawInstrProfReader
<IntPtrT
>::hasFormat(const MemoryBuffer
&DataBuffer
) {
296 if (DataBuffer
.getBufferSize() < sizeof(uint64_t))
299 *reinterpret_cast<const uint64_t *>(DataBuffer
.getBufferStart());
300 return RawInstrProf::getMagic
<IntPtrT
>() == Magic
||
301 sys::getSwappedBytes(RawInstrProf::getMagic
<IntPtrT
>()) == Magic
;
304 template <class IntPtrT
>
305 Error RawInstrProfReader
<IntPtrT
>::readHeader() {
306 if (!hasFormat(*DataBuffer
))
307 return error(instrprof_error::bad_magic
);
308 if (DataBuffer
->getBufferSize() < sizeof(RawInstrProf::Header
))
309 return error(instrprof_error::bad_header
);
310 auto *Header
= reinterpret_cast<const RawInstrProf::Header
*>(
311 DataBuffer
->getBufferStart());
312 ShouldSwapBytes
= Header
->Magic
!= RawInstrProf::getMagic
<IntPtrT
>();
313 return readHeader(*Header
);
316 template <class IntPtrT
>
317 Error RawInstrProfReader
<IntPtrT
>::readNextHeader(const char *CurrentPos
) {
318 const char *End
= DataBuffer
->getBufferEnd();
319 // Skip zero padding between profiles.
320 while (CurrentPos
!= End
&& *CurrentPos
== 0)
322 // If there's nothing left, we're done.
323 if (CurrentPos
== End
)
324 return make_error
<InstrProfError
>(instrprof_error::eof
);
325 // If there isn't enough space for another header, this is probably just
326 // garbage at the end of the file.
327 if (CurrentPos
+ sizeof(RawInstrProf::Header
) > End
)
328 return make_error
<InstrProfError
>(instrprof_error::malformed
);
329 // The writer ensures each profile is padded to start at an aligned address.
330 if (reinterpret_cast<size_t>(CurrentPos
) % alignof(uint64_t))
331 return make_error
<InstrProfError
>(instrprof_error::malformed
);
332 // The magic should have the same byte order as in the previous header.
333 uint64_t Magic
= *reinterpret_cast<const uint64_t *>(CurrentPos
);
334 if (Magic
!= swap(RawInstrProf::getMagic
<IntPtrT
>()))
335 return make_error
<InstrProfError
>(instrprof_error::bad_magic
);
337 // There's another profile to read, so we need to process the header.
338 auto *Header
= reinterpret_cast<const RawInstrProf::Header
*>(CurrentPos
);
339 return readHeader(*Header
);
342 template <class IntPtrT
>
343 Error RawInstrProfReader
<IntPtrT
>::createSymtab(InstrProfSymtab
&Symtab
) {
344 if (Error E
= Symtab
.create(StringRef(NamesStart
, NamesSize
)))
345 return error(std::move(E
));
346 for (const RawInstrProf::ProfileData
<IntPtrT
> *I
= Data
; I
!= DataEnd
; ++I
) {
347 const IntPtrT FPtr
= swap(I
->FunctionPointer
);
350 Symtab
.mapAddress(FPtr
, I
->NameRef
);
355 template <class IntPtrT
>
356 Error RawInstrProfReader
<IntPtrT
>::readHeader(
357 const RawInstrProf::Header
&Header
) {
358 Version
= swap(Header
.Version
);
359 if (GET_VERSION(Version
) != RawInstrProf::Version
)
360 return error(instrprof_error::unsupported_version
);
362 CountersDelta
= swap(Header
.CountersDelta
);
363 NamesDelta
= swap(Header
.NamesDelta
);
364 auto DataSize
= swap(Header
.DataSize
);
365 auto CountersSize
= swap(Header
.CountersSize
);
366 NamesSize
= swap(Header
.NamesSize
);
367 ValueKindLast
= swap(Header
.ValueKindLast
);
369 auto DataSizeInBytes
= DataSize
* sizeof(RawInstrProf::ProfileData
<IntPtrT
>);
370 auto PaddingSize
= getNumPaddingBytes(NamesSize
);
372 ptrdiff_t DataOffset
= sizeof(RawInstrProf::Header
);
373 ptrdiff_t CountersOffset
= DataOffset
+ DataSizeInBytes
;
374 ptrdiff_t NamesOffset
= CountersOffset
+ sizeof(uint64_t) * CountersSize
;
375 ptrdiff_t ValueDataOffset
= NamesOffset
+ NamesSize
+ PaddingSize
;
377 auto *Start
= reinterpret_cast<const char *>(&Header
);
378 if (Start
+ ValueDataOffset
> DataBuffer
->getBufferEnd())
379 return error(instrprof_error::bad_header
);
381 Data
= reinterpret_cast<const RawInstrProf::ProfileData
<IntPtrT
> *>(
383 DataEnd
= Data
+ DataSize
;
384 CountersStart
= reinterpret_cast<const uint64_t *>(Start
+ CountersOffset
);
385 NamesStart
= Start
+ NamesOffset
;
386 ValueDataStart
= reinterpret_cast<const uint8_t *>(Start
+ ValueDataOffset
);
388 std::unique_ptr
<InstrProfSymtab
> NewSymtab
= std::make_unique
<InstrProfSymtab
>();
389 if (Error E
= createSymtab(*NewSymtab
.get()))
392 Symtab
= std::move(NewSymtab
);
396 template <class IntPtrT
>
397 Error RawInstrProfReader
<IntPtrT
>::readName(NamedInstrProfRecord
&Record
) {
398 Record
.Name
= getName(Data
->NameRef
);
402 template <class IntPtrT
>
403 Error RawInstrProfReader
<IntPtrT
>::readFuncHash(NamedInstrProfRecord
&Record
) {
404 Record
.Hash
= swap(Data
->FuncHash
);
408 template <class IntPtrT
>
409 Error RawInstrProfReader
<IntPtrT
>::readRawCounts(
410 InstrProfRecord
&Record
) {
411 uint32_t NumCounters
= swap(Data
->NumCounters
);
412 IntPtrT CounterPtr
= Data
->CounterPtr
;
413 if (NumCounters
== 0)
414 return error(instrprof_error::malformed
);
416 auto *NamesStartAsCounter
= reinterpret_cast<const uint64_t *>(NamesStart
);
417 ptrdiff_t MaxNumCounters
= NamesStartAsCounter
- CountersStart
;
419 // Check bounds. Note that the counter pointer embedded in the data record
420 // may itself be corrupt.
421 if (NumCounters
> MaxNumCounters
)
422 return error(instrprof_error::malformed
);
423 ptrdiff_t CounterOffset
= getCounterOffset(CounterPtr
);
424 if (CounterOffset
< 0 || CounterOffset
> MaxNumCounters
||
425 (CounterOffset
+ NumCounters
) > MaxNumCounters
)
426 return error(instrprof_error::malformed
);
428 auto RawCounts
= makeArrayRef(getCounter(CounterOffset
), NumCounters
);
430 if (ShouldSwapBytes
) {
431 Record
.Counts
.clear();
432 Record
.Counts
.reserve(RawCounts
.size());
433 for (uint64_t Count
: RawCounts
)
434 Record
.Counts
.push_back(swap(Count
));
436 Record
.Counts
= RawCounts
;
441 template <class IntPtrT
>
442 Error RawInstrProfReader
<IntPtrT
>::readValueProfilingData(
443 InstrProfRecord
&Record
) {
444 Record
.clearValueData();
445 CurValueDataSize
= 0;
446 // Need to match the logic in value profile dumper code in compiler-rt:
447 uint32_t NumValueKinds
= 0;
448 for (uint32_t I
= 0; I
< IPVK_Last
+ 1; I
++)
449 NumValueKinds
+= (Data
->NumValueSites
[I
] != 0);
454 Expected
<std::unique_ptr
<ValueProfData
>> VDataPtrOrErr
=
455 ValueProfData::getValueProfData(
456 ValueDataStart
, (const unsigned char *)DataBuffer
->getBufferEnd(),
457 getDataEndianness());
459 if (Error E
= VDataPtrOrErr
.takeError())
462 // Note that besides deserialization, this also performs the conversion for
463 // indirect call targets. The function pointers from the raw profile are
464 // remapped into function name hashes.
465 VDataPtrOrErr
.get()->deserializeTo(Record
, Symtab
.get());
466 CurValueDataSize
= VDataPtrOrErr
.get()->getSize();
470 template <class IntPtrT
>
471 Error RawInstrProfReader
<IntPtrT
>::readNextRecord(NamedInstrProfRecord
&Record
) {
473 // At this point, ValueDataStart field points to the next header.
474 if (Error E
= readNextHeader(getNextHeaderPos()))
475 return error(std::move(E
));
477 // Read name ad set it in Record.
478 if (Error E
= readName(Record
))
479 return error(std::move(E
));
481 // Read FuncHash and set it in Record.
482 if (Error E
= readFuncHash(Record
))
483 return error(std::move(E
));
485 // Read raw counts and set Record.
486 if (Error E
= readRawCounts(Record
))
487 return error(std::move(E
));
489 // Read value data and set Record.
490 if (Error E
= readValueProfilingData(Record
))
491 return error(std::move(E
));
500 template class RawInstrProfReader
<uint32_t>;
501 template class RawInstrProfReader
<uint64_t>;
503 } // end namespace llvm
505 InstrProfLookupTrait::hash_value_type
506 InstrProfLookupTrait::ComputeHash(StringRef K
) {
507 return IndexedInstrProf::ComputeHash(HashType
, K
);
510 using data_type
= InstrProfLookupTrait::data_type
;
511 using offset_type
= InstrProfLookupTrait::offset_type
;
513 bool InstrProfLookupTrait::readValueProfilingData(
514 const unsigned char *&D
, const unsigned char *const End
) {
515 Expected
<std::unique_ptr
<ValueProfData
>> VDataPtrOrErr
=
516 ValueProfData::getValueProfData(D
, End
, ValueProfDataEndianness
);
518 if (VDataPtrOrErr
.takeError())
521 VDataPtrOrErr
.get()->deserializeTo(DataBuffer
.back(), nullptr);
522 D
+= VDataPtrOrErr
.get()->TotalSize
;
527 data_type
InstrProfLookupTrait::ReadData(StringRef K
, const unsigned char *D
,
529 using namespace support
;
531 // Check if the data is corrupt. If so, don't try to read it.
532 if (N
% sizeof(uint64_t))
536 std::vector
<uint64_t> CounterBuffer
;
538 const unsigned char *End
= D
+ N
;
541 if (D
+ sizeof(uint64_t) >= End
)
543 uint64_t Hash
= endian::readNext
<uint64_t, little
, unaligned
>(D
);
545 // Initialize number of counters for GET_VERSION(FormatVersion) == 1.
546 uint64_t CountsSize
= N
/ sizeof(uint64_t) - 1;
547 // If format version is different then read the number of counters.
548 if (GET_VERSION(FormatVersion
) != IndexedInstrProf::ProfVersion::Version1
) {
549 if (D
+ sizeof(uint64_t) > End
)
551 CountsSize
= endian::readNext
<uint64_t, little
, unaligned
>(D
);
553 // Read counter values.
554 if (D
+ CountsSize
* sizeof(uint64_t) > End
)
557 CounterBuffer
.clear();
558 CounterBuffer
.reserve(CountsSize
);
559 for (uint64_t J
= 0; J
< CountsSize
; ++J
)
560 CounterBuffer
.push_back(endian::readNext
<uint64_t, little
, unaligned
>(D
));
562 DataBuffer
.emplace_back(K
, Hash
, std::move(CounterBuffer
));
564 // Read value profiling data.
565 if (GET_VERSION(FormatVersion
) > IndexedInstrProf::ProfVersion::Version2
&&
566 !readValueProfilingData(D
, End
)) {
574 template <typename HashTableImpl
>
575 Error InstrProfReaderIndex
<HashTableImpl
>::getRecords(
576 StringRef FuncName
, ArrayRef
<NamedInstrProfRecord
> &Data
) {
577 auto Iter
= HashTable
->find(FuncName
);
578 if (Iter
== HashTable
->end())
579 return make_error
<InstrProfError
>(instrprof_error::unknown_function
);
583 return make_error
<InstrProfError
>(instrprof_error::malformed
);
585 return Error::success();
588 template <typename HashTableImpl
>
589 Error InstrProfReaderIndex
<HashTableImpl
>::getRecords(
590 ArrayRef
<NamedInstrProfRecord
> &Data
) {
592 return make_error
<InstrProfError
>(instrprof_error::eof
);
594 Data
= *RecordIterator
;
597 return make_error
<InstrProfError
>(instrprof_error::malformed
);
599 return Error::success();
602 template <typename HashTableImpl
>
603 InstrProfReaderIndex
<HashTableImpl
>::InstrProfReaderIndex(
604 const unsigned char *Buckets
, const unsigned char *const Payload
,
605 const unsigned char *const Base
, IndexedInstrProf::HashT HashType
,
607 FormatVersion
= Version
;
608 HashTable
.reset(HashTableImpl::Create(
609 Buckets
, Payload
, Base
,
610 typename
HashTableImpl::InfoType(HashType
, Version
)));
611 RecordIterator
= HashTable
->data_begin();
615 /// A remapper that does not apply any remappings.
616 class InstrProfReaderNullRemapper
: public InstrProfReaderRemapper
{
617 InstrProfReaderIndexBase
&Underlying
;
620 InstrProfReaderNullRemapper(InstrProfReaderIndexBase
&Underlying
)
621 : Underlying(Underlying
) {}
623 Error
getRecords(StringRef FuncName
,
624 ArrayRef
<NamedInstrProfRecord
> &Data
) override
{
625 return Underlying
.getRecords(FuncName
, Data
);
630 /// A remapper that applies remappings based on a symbol remapping file.
631 template <typename HashTableImpl
>
632 class llvm::InstrProfReaderItaniumRemapper
633 : public InstrProfReaderRemapper
{
635 InstrProfReaderItaniumRemapper(
636 std::unique_ptr
<MemoryBuffer
> RemapBuffer
,
637 InstrProfReaderIndex
<HashTableImpl
> &Underlying
)
638 : RemapBuffer(std::move(RemapBuffer
)), Underlying(Underlying
) {
641 /// Extract the original function name from a PGO function name.
642 static StringRef
extractName(StringRef Name
) {
643 // We can have multiple :-separated pieces; there can be pieces both
644 // before and after the mangled name. Find the first part that starts
645 // with '_Z'; we'll assume that's the mangled name we want.
646 std::pair
<StringRef
, StringRef
> Parts
= {StringRef(), Name
};
648 Parts
= Parts
.second
.split(':');
649 if (Parts
.first
.startswith("_Z"))
651 if (Parts
.second
.empty())
656 /// Given a mangled name extracted from a PGO function name, and a new
657 /// form for that mangled name, reconstitute the name.
658 static void reconstituteName(StringRef OrigName
, StringRef ExtractedName
,
659 StringRef Replacement
,
660 SmallVectorImpl
<char> &Out
) {
661 Out
.reserve(OrigName
.size() + Replacement
.size() - ExtractedName
.size());
662 Out
.insert(Out
.end(), OrigName
.begin(), ExtractedName
.begin());
663 Out
.insert(Out
.end(), Replacement
.begin(), Replacement
.end());
664 Out
.insert(Out
.end(), ExtractedName
.end(), OrigName
.end());
667 Error
populateRemappings() override
{
668 if (Error E
= Remappings
.read(*RemapBuffer
))
670 for (StringRef Name
: Underlying
.HashTable
->keys()) {
671 StringRef RealName
= extractName(Name
);
672 if (auto Key
= Remappings
.insert(RealName
)) {
673 // FIXME: We could theoretically map the same equivalence class to
674 // multiple names in the profile data. If that happens, we should
675 // return NamedInstrProfRecords from all of them.
676 MappedNames
.insert({Key
, RealName
});
679 return Error::success();
682 Error
getRecords(StringRef FuncName
,
683 ArrayRef
<NamedInstrProfRecord
> &Data
) override
{
684 StringRef RealName
= extractName(FuncName
);
685 if (auto Key
= Remappings
.lookup(RealName
)) {
686 StringRef Remapped
= MappedNames
.lookup(Key
);
687 if (!Remapped
.empty()) {
688 if (RealName
.begin() == FuncName
.begin() &&
689 RealName
.end() == FuncName
.end())
692 // Try rebuilding the name from the given remapping.
693 SmallString
<256> Reconstituted
;
694 reconstituteName(FuncName
, RealName
, Remapped
, Reconstituted
);
695 Error E
= Underlying
.getRecords(Reconstituted
, Data
);
699 // If we failed because the name doesn't exist, fall back to asking
700 // about the original name.
701 if (Error Unhandled
= handleErrors(
702 std::move(E
), [](std::unique_ptr
<InstrProfError
> Err
) {
703 return Err
->get() == instrprof_error::unknown_function
705 : Error(std::move(Err
));
711 return Underlying
.getRecords(FuncName
, Data
);
715 /// The memory buffer containing the remapping configuration. Remappings
716 /// holds pointers into this buffer.
717 std::unique_ptr
<MemoryBuffer
> RemapBuffer
;
719 /// The mangling remapper.
720 SymbolRemappingReader Remappings
;
722 /// Mapping from mangled name keys to the name used for the key in the
724 /// FIXME: Can we store a location within the on-disk hash table instead of
726 DenseMap
<SymbolRemappingReader::Key
, StringRef
> MappedNames
;
728 /// The real profile data reader.
729 InstrProfReaderIndex
<HashTableImpl
> &Underlying
;
732 bool IndexedInstrProfReader::hasFormat(const MemoryBuffer
&DataBuffer
) {
733 using namespace support
;
735 if (DataBuffer
.getBufferSize() < 8)
738 endian::read
<uint64_t, little
, aligned
>(DataBuffer
.getBufferStart());
739 // Verify that it's magical.
740 return Magic
== IndexedInstrProf::Magic
;
743 const unsigned char *
744 IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version
,
745 const unsigned char *Cur
, bool UseCS
) {
746 using namespace IndexedInstrProf
;
747 using namespace support
;
749 if (Version
>= IndexedInstrProf::Version4
) {
750 const IndexedInstrProf::Summary
*SummaryInLE
=
751 reinterpret_cast<const IndexedInstrProf::Summary
*>(Cur
);
753 endian::byte_swap
<uint64_t, little
>(SummaryInLE
->NumSummaryFields
);
755 endian::byte_swap
<uint64_t, little
>(SummaryInLE
->NumCutoffEntries
);
756 uint32_t SummarySize
=
757 IndexedInstrProf::Summary::getSize(NFields
, NEntries
);
758 std::unique_ptr
<IndexedInstrProf::Summary
> SummaryData
=
759 IndexedInstrProf::allocSummary(SummarySize
);
761 const uint64_t *Src
= reinterpret_cast<const uint64_t *>(SummaryInLE
);
762 uint64_t *Dst
= reinterpret_cast<uint64_t *>(SummaryData
.get());
763 for (unsigned I
= 0; I
< SummarySize
/ sizeof(uint64_t); I
++)
764 Dst
[I
] = endian::byte_swap
<uint64_t, little
>(Src
[I
]);
766 SummaryEntryVector DetailedSummary
;
767 for (unsigned I
= 0; I
< SummaryData
->NumCutoffEntries
; I
++) {
768 const IndexedInstrProf::Summary::Entry
&Ent
= SummaryData
->getEntry(I
);
769 DetailedSummary
.emplace_back((uint32_t)Ent
.Cutoff
, Ent
.MinBlockCount
,
772 std::unique_ptr
<llvm::ProfileSummary
> &Summary
=
773 UseCS
? this->CS_Summary
: this->Summary
;
775 // initialize InstrProfSummary using the SummaryData from disk.
776 Summary
= std::make_unique
<ProfileSummary
>(
777 UseCS
? ProfileSummary::PSK_CSInstr
: ProfileSummary::PSK_Instr
,
778 DetailedSummary
, SummaryData
->get(Summary::TotalBlockCount
),
779 SummaryData
->get(Summary::MaxBlockCount
),
780 SummaryData
->get(Summary::MaxInternalBlockCount
),
781 SummaryData
->get(Summary::MaxFunctionCount
),
782 SummaryData
->get(Summary::TotalNumBlocks
),
783 SummaryData
->get(Summary::TotalNumFunctions
));
784 return Cur
+ SummarySize
;
786 // The older versions do not support a profile summary. This just computes
787 // an empty summary, which will not result in accurate hot/cold detection.
788 // We would need to call addRecord for all NamedInstrProfRecords to get the
789 // correct summary. However, this version is old (prior to early 2016) and
790 // has not been supporting an accurate summary for several years.
791 InstrProfSummaryBuilder
Builder(ProfileSummaryBuilder::DefaultCutoffs
);
792 Summary
= Builder
.getSummary();
797 Error
IndexedInstrProfReader::readHeader() {
798 using namespace support
;
800 const unsigned char *Start
=
801 (const unsigned char *)DataBuffer
->getBufferStart();
802 const unsigned char *Cur
= Start
;
803 if ((const unsigned char *)DataBuffer
->getBufferEnd() - Cur
< 24)
804 return error(instrprof_error::truncated
);
806 auto *Header
= reinterpret_cast<const IndexedInstrProf::Header
*>(Cur
);
807 Cur
+= sizeof(IndexedInstrProf::Header
);
809 // Check the magic number.
810 uint64_t Magic
= endian::byte_swap
<uint64_t, little
>(Header
->Magic
);
811 if (Magic
!= IndexedInstrProf::Magic
)
812 return error(instrprof_error::bad_magic
);
815 uint64_t FormatVersion
= endian::byte_swap
<uint64_t, little
>(Header
->Version
);
816 if (GET_VERSION(FormatVersion
) >
817 IndexedInstrProf::ProfVersion::CurrentVersion
)
818 return error(instrprof_error::unsupported_version
);
820 Cur
= readSummary((IndexedInstrProf::ProfVersion
)FormatVersion
, Cur
,
822 if (FormatVersion
& VARIANT_MASK_CSIR_PROF
)
823 Cur
= readSummary((IndexedInstrProf::ProfVersion
)FormatVersion
, Cur
,
826 // Read the hash type and start offset.
827 IndexedInstrProf::HashT HashType
= static_cast<IndexedInstrProf::HashT
>(
828 endian::byte_swap
<uint64_t, little
>(Header
->HashType
));
829 if (HashType
> IndexedInstrProf::HashT::Last
)
830 return error(instrprof_error::unsupported_hash_type
);
832 uint64_t HashOffset
= endian::byte_swap
<uint64_t, little
>(Header
->HashOffset
);
834 // The rest of the file is an on disk hash table.
836 std::make_unique
<InstrProfReaderIndex
<OnDiskHashTableImplV3
>>(
837 Start
+ HashOffset
, Cur
, Start
, HashType
, FormatVersion
);
839 // Load the remapping table now if requested.
840 if (RemappingBuffer
) {
841 Remapper
= std::make_unique
<
842 InstrProfReaderItaniumRemapper
<OnDiskHashTableImplV3
>>(
843 std::move(RemappingBuffer
), *IndexPtr
);
844 if (Error E
= Remapper
->populateRemappings())
847 Remapper
= std::make_unique
<InstrProfReaderNullRemapper
>(*IndexPtr
);
849 Index
= std::move(IndexPtr
);
854 InstrProfSymtab
&IndexedInstrProfReader::getSymtab() {
856 return *Symtab
.get();
858 std::unique_ptr
<InstrProfSymtab
> NewSymtab
= std::make_unique
<InstrProfSymtab
>();
859 if (Error E
= Index
->populateSymtab(*NewSymtab
.get())) {
860 consumeError(error(InstrProfError::take(std::move(E
))));
863 Symtab
= std::move(NewSymtab
);
864 return *Symtab
.get();
867 Expected
<InstrProfRecord
>
868 IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName
,
870 ArrayRef
<NamedInstrProfRecord
> Data
;
871 Error Err
= Remapper
->getRecords(FuncName
, Data
);
873 return std::move(Err
);
874 // Found it. Look for counters with the right hash.
875 for (unsigned I
= 0, E
= Data
.size(); I
< E
; ++I
) {
876 // Check for a match and fill the vector if there is one.
877 if (Data
[I
].Hash
== FuncHash
) {
878 return std::move(Data
[I
]);
881 return error(instrprof_error::hash_mismatch
);
884 Error
IndexedInstrProfReader::getFunctionCounts(StringRef FuncName
,
886 std::vector
<uint64_t> &Counts
) {
887 Expected
<InstrProfRecord
> Record
= getInstrProfRecord(FuncName
, FuncHash
);
888 if (Error E
= Record
.takeError())
889 return error(std::move(E
));
891 Counts
= Record
.get().Counts
;
895 Error
IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord
&Record
) {
896 ArrayRef
<NamedInstrProfRecord
> Data
;
898 Error E
= Index
->getRecords(Data
);
900 return error(std::move(E
));
902 Record
= Data
[RecordIndex
++];
903 if (RecordIndex
>= Data
.size()) {
904 Index
->advanceToNextKey();
910 void InstrProfReader::accumulateCounts(CountSumOrPercent
&Sum
, bool IsCS
) {
911 uint64_t NumFuncs
= 0;
912 for (const auto &Func
: *this) {
913 if (isIRLevelProfile()) {
914 bool FuncIsCS
= NamedInstrProfRecord::hasCSFlagInHash(Func
.Hash
);
915 if (FuncIsCS
!= IsCS
)
918 Func
.accumulateCounts(Sum
);
921 Sum
.NumEntries
= NumFuncs
;