1 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the class that writes LLVM sample profiles. It
10 // supports two file formats: text and binary. The textual representation
11 // is useful for debugging and testing purposes. The binary representation
12 // is more compact, resulting in smaller file sizes. However, they can
13 // both be used interchangeably.
15 // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
18 //===----------------------------------------------------------------------===//
20 #include "llvm/ProfileData/SampleProfWriter.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ProfileData/ProfileCommon.h"
23 #include "llvm/ProfileData/SampleProf.h"
24 #include "llvm/Support/Compression.h"
25 #include "llvm/Support/Endian.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/ErrorOr.h"
28 #include "llvm/Support/FileSystem.h"
29 #include "llvm/Support/LEB128.h"
30 #include "llvm/Support/MD5.h"
31 #include "llvm/Support/raw_ostream.h"
36 #include <system_error>
41 using namespace sampleprof
;
43 std::error_code
SampleProfileWriter::writeFuncProfiles(
44 const StringMap
<FunctionSamples
> &ProfileMap
) {
45 // Sort the ProfileMap by total samples.
46 typedef std::pair
<StringRef
, const FunctionSamples
*> NameFunctionSamples
;
47 std::vector
<NameFunctionSamples
> V
;
48 for (const auto &I
: ProfileMap
)
49 V
.push_back(std::make_pair(I
.getKey(), &I
.second
));
52 V
, [](const NameFunctionSamples
&A
, const NameFunctionSamples
&B
) {
53 if (A
.second
->getTotalSamples() == B
.second
->getTotalSamples())
54 return A
.first
> B
.first
;
55 return A
.second
->getTotalSamples() > B
.second
->getTotalSamples();
58 for (const auto &I
: V
) {
59 if (std::error_code EC
= writeSample(*I
.second
))
62 return sampleprof_error::success
;
66 SampleProfileWriter::write(const StringMap
<FunctionSamples
> &ProfileMap
) {
67 if (std::error_code EC
= writeHeader(ProfileMap
))
70 if (std::error_code EC
= writeFuncProfiles(ProfileMap
))
73 return sampleprof_error::success
;
77 SampleProfileWriterExtBinaryBase::getEntryInLayout(SecType Type
) {
78 auto SecIt
= std::find_if(
79 SectionHdrLayout
.begin(), SectionHdrLayout
.end(),
80 [=](const auto &Entry
) -> bool { return Entry
.Type
== Type
; });
84 /// Return the current position and prepare to use it as the start
85 /// position of a section.
86 uint64_t SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type
) {
87 uint64_t SectionStart
= OutputStream
->tell();
88 auto &Entry
= getEntryInLayout(Type
);
89 // Use LocalBuf as a temporary output for writting data.
90 if (hasSecFlag(Entry
, SecFlagCompress
))
91 LocalBufStream
.swap(OutputStream
);
95 std::error_code
SampleProfileWriterExtBinaryBase::compressAndOutput() {
96 if (!llvm::zlib::isAvailable())
97 return sampleprof_error::zlib_unavailable
;
98 std::string
&UncompressedStrings
=
99 static_cast<raw_string_ostream
*>(LocalBufStream
.get())->str();
100 if (UncompressedStrings
.size() == 0)
101 return sampleprof_error::success
;
102 auto &OS
= *OutputStream
;
103 SmallString
<128> CompressedStrings
;
104 llvm::Error E
= zlib::compress(UncompressedStrings
, CompressedStrings
,
105 zlib::BestSizeCompression
);
107 return sampleprof_error::compress_failed
;
108 encodeULEB128(UncompressedStrings
.size(), OS
);
109 encodeULEB128(CompressedStrings
.size(), OS
);
110 OS
<< CompressedStrings
.str();
111 UncompressedStrings
.clear();
112 return sampleprof_error::success
;
115 /// Add a new section into section header table.
117 SampleProfileWriterExtBinaryBase::addNewSection(SecType Type
,
118 uint64_t SectionStart
) {
119 auto Entry
= getEntryInLayout(Type
);
120 if (hasSecFlag(Entry
, SecFlagCompress
)) {
121 LocalBufStream
.swap(OutputStream
);
122 if (std::error_code EC
= compressAndOutput())
125 SecHdrTable
.push_back({Type
, Entry
.Flags
, SectionStart
- FileStart
,
126 OutputStream
->tell() - SectionStart
});
127 return sampleprof_error::success
;
130 std::error_code
SampleProfileWriterExtBinaryBase::write(
131 const StringMap
<FunctionSamples
> &ProfileMap
) {
132 if (std::error_code EC
= writeHeader(ProfileMap
))
135 std::string LocalBuf
;
136 LocalBufStream
= std::make_unique
<raw_string_ostream
>(LocalBuf
);
137 if (std::error_code EC
= writeSections(ProfileMap
))
140 if (std::error_code EC
= writeSecHdrTable())
143 return sampleprof_error::success
;
147 SampleProfileWriterExtBinary::writeSample(const FunctionSamples
&S
) {
148 uint64_t Offset
= OutputStream
->tell();
149 StringRef Name
= S
.getName();
150 FuncOffsetTable
[Name
] = Offset
- SecLBRProfileStart
;
151 encodeULEB128(S
.getHeadSamples(), *OutputStream
);
155 std::error_code
SampleProfileWriterExtBinary::writeFuncOffsetTable() {
156 auto &OS
= *OutputStream
;
158 // Write out the table size.
159 encodeULEB128(FuncOffsetTable
.size(), OS
);
161 // Write out FuncOffsetTable.
162 for (auto entry
: FuncOffsetTable
) {
163 writeNameIdx(entry
.first
);
164 encodeULEB128(entry
.second
, OS
);
166 return sampleprof_error::success
;
169 std::error_code
SampleProfileWriterExtBinary::writeSections(
170 const StringMap
<FunctionSamples
> &ProfileMap
) {
171 uint64_t SectionStart
= markSectionStart(SecProfSummary
);
172 computeSummary(ProfileMap
);
173 if (auto EC
= writeSummary())
175 if (std::error_code EC
= addNewSection(SecProfSummary
, SectionStart
))
178 // Generate the name table for all the functions referenced in the profile.
179 SectionStart
= markSectionStart(SecNameTable
);
180 for (const auto &I
: ProfileMap
) {
185 if (std::error_code EC
= addNewSection(SecNameTable
, SectionStart
))
188 SectionStart
= markSectionStart(SecLBRProfile
);
189 SecLBRProfileStart
= OutputStream
->tell();
190 if (std::error_code EC
= writeFuncProfiles(ProfileMap
))
192 if (std::error_code EC
= addNewSection(SecLBRProfile
, SectionStart
))
195 if (ProfSymList
&& ProfSymList
->toCompress())
196 setToCompressSection(SecProfileSymbolList
);
198 SectionStart
= markSectionStart(SecProfileSymbolList
);
199 if (ProfSymList
&& ProfSymList
->size() > 0)
200 if (std::error_code EC
= ProfSymList
->write(*OutputStream
))
202 if (std::error_code EC
= addNewSection(SecProfileSymbolList
, SectionStart
))
205 SectionStart
= markSectionStart(SecFuncOffsetTable
);
206 if (std::error_code EC
= writeFuncOffsetTable())
208 if (std::error_code EC
= addNewSection(SecFuncOffsetTable
, SectionStart
))
211 return sampleprof_error::success
;
214 std::error_code
SampleProfileWriterCompactBinary::write(
215 const StringMap
<FunctionSamples
> &ProfileMap
) {
216 if (std::error_code EC
= SampleProfileWriter::write(ProfileMap
))
218 if (std::error_code EC
= writeFuncOffsetTable())
220 return sampleprof_error::success
;
223 /// Write samples to a text file.
225 /// Note: it may be tempting to implement this in terms of
226 /// FunctionSamples::print(). Please don't. The dump functionality is intended
227 /// for debugging and has no specified form.
229 /// The format used here is more structured and deliberate because
230 /// it needs to be parsed by the SampleProfileReaderText class.
231 std::error_code
SampleProfileWriterText::writeSample(const FunctionSamples
&S
) {
232 auto &OS
= *OutputStream
;
233 OS
<< S
.getName() << ":" << S
.getTotalSamples();
235 OS
<< ":" << S
.getHeadSamples();
238 SampleSorter
<LineLocation
, SampleRecord
> SortedSamples(S
.getBodySamples());
239 for (const auto &I
: SortedSamples
.get()) {
240 LineLocation Loc
= I
->first
;
241 const SampleRecord
&Sample
= I
->second
;
242 OS
.indent(Indent
+ 1);
243 if (Loc
.Discriminator
== 0)
244 OS
<< Loc
.LineOffset
<< ": ";
246 OS
<< Loc
.LineOffset
<< "." << Loc
.Discriminator
<< ": ";
248 OS
<< Sample
.getSamples();
250 for (const auto &J
: Sample
.getSortedCallTargets())
251 OS
<< " " << J
.first
<< ":" << J
.second
;
255 SampleSorter
<LineLocation
, FunctionSamplesMap
> SortedCallsiteSamples(
256 S
.getCallsiteSamples());
258 for (const auto &I
: SortedCallsiteSamples
.get())
259 for (const auto &FS
: I
->second
) {
260 LineLocation Loc
= I
->first
;
261 const FunctionSamples
&CalleeSamples
= FS
.second
;
263 if (Loc
.Discriminator
== 0)
264 OS
<< Loc
.LineOffset
<< ": ";
266 OS
<< Loc
.LineOffset
<< "." << Loc
.Discriminator
<< ": ";
267 if (std::error_code EC
= writeSample(CalleeSamples
))
272 return sampleprof_error::success
;
275 std::error_code
SampleProfileWriterBinary::writeNameIdx(StringRef FName
) {
276 const auto &ret
= NameTable
.find(FName
);
277 if (ret
== NameTable
.end())
278 return sampleprof_error::truncated_name_table
;
279 encodeULEB128(ret
->second
, *OutputStream
);
280 return sampleprof_error::success
;
283 void SampleProfileWriterBinary::addName(StringRef FName
) {
284 NameTable
.insert(std::make_pair(FName
, 0));
287 void SampleProfileWriterBinary::addNames(const FunctionSamples
&S
) {
288 // Add all the names in indirect call targets.
289 for (const auto &I
: S
.getBodySamples()) {
290 const SampleRecord
&Sample
= I
.second
;
291 for (const auto &J
: Sample
.getCallTargets())
295 // Recursively add all the names for inlined callsites.
296 for (const auto &J
: S
.getCallsiteSamples())
297 for (const auto &FS
: J
.second
) {
298 const FunctionSamples
&CalleeSamples
= FS
.second
;
299 addName(CalleeSamples
.getName());
300 addNames(CalleeSamples
);
304 void SampleProfileWriterBinary::stablizeNameTable(std::set
<StringRef
> &V
) {
305 // Sort the names to make NameTable deterministic.
306 for (const auto &I
: NameTable
)
309 for (const StringRef
&N
: V
)
313 std::error_code
SampleProfileWriterBinary::writeNameTable() {
314 auto &OS
= *OutputStream
;
315 std::set
<StringRef
> V
;
316 stablizeNameTable(V
);
318 // Write out the name table.
319 encodeULEB128(NameTable
.size(), OS
);
322 encodeULEB128(0, OS
);
324 return sampleprof_error::success
;
327 std::error_code
SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
328 auto &OS
= *OutputStream
;
330 // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
331 auto &OFS
= static_cast<raw_fd_ostream
&>(OS
);
332 uint64_t FuncOffsetTableStart
= OS
.tell();
333 if (OFS
.seek(TableOffset
) == (uint64_t)-1)
334 return sampleprof_error::ostream_seek_unsupported
;
335 support::endian::Writer
Writer(*OutputStream
, support::little
);
336 Writer
.write(FuncOffsetTableStart
);
337 if (OFS
.seek(FuncOffsetTableStart
) == (uint64_t)-1)
338 return sampleprof_error::ostream_seek_unsupported
;
340 // Write out the table size.
341 encodeULEB128(FuncOffsetTable
.size(), OS
);
343 // Write out FuncOffsetTable.
344 for (auto entry
: FuncOffsetTable
) {
345 writeNameIdx(entry
.first
);
346 encodeULEB128(entry
.second
, OS
);
348 return sampleprof_error::success
;
351 std::error_code
SampleProfileWriterCompactBinary::writeNameTable() {
352 auto &OS
= *OutputStream
;
353 std::set
<StringRef
> V
;
354 stablizeNameTable(V
);
356 // Write out the name table.
357 encodeULEB128(NameTable
.size(), OS
);
359 encodeULEB128(MD5Hash(N
), OS
);
361 return sampleprof_error::success
;
365 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format
) {
366 auto &OS
= *OutputStream
;
367 // Write file magic identifier.
368 encodeULEB128(SPMagic(Format
), OS
);
369 encodeULEB128(SPVersion(), OS
);
370 return sampleprof_error::success
;
373 std::error_code
SampleProfileWriterBinary::writeHeader(
374 const StringMap
<FunctionSamples
> &ProfileMap
) {
375 writeMagicIdent(Format
);
377 computeSummary(ProfileMap
);
378 if (auto EC
= writeSummary())
381 // Generate the name table for all the functions referenced in the profile.
382 for (const auto &I
: ProfileMap
) {
388 return sampleprof_error::success
;
391 void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
392 for (auto &Entry
: SectionHdrLayout
)
393 addSecFlags(Entry
, SecFlagCompress
);
396 void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type
) {
397 addSectionFlags(Type
, SecFlagCompress
);
400 void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type
,
402 for (auto &Entry
: SectionHdrLayout
) {
403 if (Entry
.Type
== Type
)
404 addSecFlags(Entry
, Flags
);
408 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
409 support::endian::Writer
Writer(*OutputStream
, support::little
);
411 Writer
.write(static_cast<uint64_t>(SectionHdrLayout
.size()));
412 SecHdrTableOffset
= OutputStream
->tell();
413 for (uint32_t i
= 0; i
< SectionHdrLayout
.size(); i
++) {
414 Writer
.write(static_cast<uint64_t>(-1));
415 Writer
.write(static_cast<uint64_t>(-1));
416 Writer
.write(static_cast<uint64_t>(-1));
417 Writer
.write(static_cast<uint64_t>(-1));
421 std::error_code
SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
422 auto &OFS
= static_cast<raw_fd_ostream
&>(*OutputStream
);
423 uint64_t Saved
= OutputStream
->tell();
425 // Set OutputStream to the location saved in SecHdrTableOffset.
426 if (OFS
.seek(SecHdrTableOffset
) == (uint64_t)-1)
427 return sampleprof_error::ostream_seek_unsupported
;
428 support::endian::Writer
Writer(*OutputStream
, support::little
);
430 DenseMap
<uint32_t, uint32_t> IndexMap
;
431 for (uint32_t i
= 0; i
< SecHdrTable
.size(); i
++) {
432 IndexMap
.insert({static_cast<uint32_t>(SecHdrTable
[i
].Type
), i
});
435 // Write the section header table in the order specified in
436 // SectionHdrLayout. That is the sections order Reader will see.
437 // Note that the sections order in which Reader expects to read
438 // may be different from the order in which Writer is able to
439 // write, so we need to adjust the order in SecHdrTable to be
440 // consistent with SectionHdrLayout when we write SecHdrTable
442 for (uint32_t i
= 0; i
< SectionHdrLayout
.size(); i
++) {
443 uint32_t idx
= IndexMap
[static_cast<uint32_t>(SectionHdrLayout
[i
].Type
)];
444 Writer
.write(static_cast<uint64_t>(SecHdrTable
[idx
].Type
));
445 Writer
.write(static_cast<uint64_t>(SecHdrTable
[idx
].Flags
));
446 Writer
.write(static_cast<uint64_t>(SecHdrTable
[idx
].Offset
));
447 Writer
.write(static_cast<uint64_t>(SecHdrTable
[idx
].Size
));
450 // Reset OutputStream.
451 if (OFS
.seek(Saved
) == (uint64_t)-1)
452 return sampleprof_error::ostream_seek_unsupported
;
454 return sampleprof_error::success
;
457 std::error_code
SampleProfileWriterExtBinaryBase::writeHeader(
458 const StringMap
<FunctionSamples
> &ProfileMap
) {
459 auto &OS
= *OutputStream
;
460 FileStart
= OS
.tell();
461 writeMagicIdent(Format
);
464 return sampleprof_error::success
;
467 std::error_code
SampleProfileWriterCompactBinary::writeHeader(
468 const StringMap
<FunctionSamples
> &ProfileMap
) {
469 support::endian::Writer
Writer(*OutputStream
, support::little
);
470 if (auto EC
= SampleProfileWriterBinary::writeHeader(ProfileMap
))
473 // Reserve a slot for the offset of function offset table. The slot will
474 // be populated with the offset of FuncOffsetTable later.
475 TableOffset
= OutputStream
->tell();
476 Writer
.write(static_cast<uint64_t>(-2));
477 return sampleprof_error::success
;
480 std::error_code
SampleProfileWriterBinary::writeSummary() {
481 auto &OS
= *OutputStream
;
482 encodeULEB128(Summary
->getTotalCount(), OS
);
483 encodeULEB128(Summary
->getMaxCount(), OS
);
484 encodeULEB128(Summary
->getMaxFunctionCount(), OS
);
485 encodeULEB128(Summary
->getNumCounts(), OS
);
486 encodeULEB128(Summary
->getNumFunctions(), OS
);
487 std::vector
<ProfileSummaryEntry
> &Entries
= Summary
->getDetailedSummary();
488 encodeULEB128(Entries
.size(), OS
);
489 for (auto Entry
: Entries
) {
490 encodeULEB128(Entry
.Cutoff
, OS
);
491 encodeULEB128(Entry
.MinCount
, OS
);
492 encodeULEB128(Entry
.NumCounts
, OS
);
494 return sampleprof_error::success
;
496 std::error_code
SampleProfileWriterBinary::writeBody(const FunctionSamples
&S
) {
497 auto &OS
= *OutputStream
;
499 if (std::error_code EC
= writeNameIdx(S
.getName()))
502 encodeULEB128(S
.getTotalSamples(), OS
);
504 // Emit all the body samples.
505 encodeULEB128(S
.getBodySamples().size(), OS
);
506 for (const auto &I
: S
.getBodySamples()) {
507 LineLocation Loc
= I
.first
;
508 const SampleRecord
&Sample
= I
.second
;
509 encodeULEB128(Loc
.LineOffset
, OS
);
510 encodeULEB128(Loc
.Discriminator
, OS
);
511 encodeULEB128(Sample
.getSamples(), OS
);
512 encodeULEB128(Sample
.getCallTargets().size(), OS
);
513 for (const auto &J
: Sample
.getSortedCallTargets()) {
514 StringRef Callee
= J
.first
;
515 uint64_t CalleeSamples
= J
.second
;
516 if (std::error_code EC
= writeNameIdx(Callee
))
518 encodeULEB128(CalleeSamples
, OS
);
522 // Recursively emit all the callsite samples.
523 uint64_t NumCallsites
= 0;
524 for (const auto &J
: S
.getCallsiteSamples())
525 NumCallsites
+= J
.second
.size();
526 encodeULEB128(NumCallsites
, OS
);
527 for (const auto &J
: S
.getCallsiteSamples())
528 for (const auto &FS
: J
.second
) {
529 LineLocation Loc
= J
.first
;
530 const FunctionSamples
&CalleeSamples
= FS
.second
;
531 encodeULEB128(Loc
.LineOffset
, OS
);
532 encodeULEB128(Loc
.Discriminator
, OS
);
533 if (std::error_code EC
= writeBody(CalleeSamples
))
537 return sampleprof_error::success
;
540 /// Write samples of a top-level function to a binary file.
542 /// \returns true if the samples were written successfully, false otherwise.
544 SampleProfileWriterBinary::writeSample(const FunctionSamples
&S
) {
545 encodeULEB128(S
.getHeadSamples(), *OutputStream
);
550 SampleProfileWriterCompactBinary::writeSample(const FunctionSamples
&S
) {
551 uint64_t Offset
= OutputStream
->tell();
552 StringRef Name
= S
.getName();
553 FuncOffsetTable
[Name
] = Offset
;
554 encodeULEB128(S
.getHeadSamples(), *OutputStream
);
558 /// Create a sample profile file writer based on the specified format.
560 /// \param Filename The file to create.
562 /// \param Format Encoding format for the profile file.
564 /// \returns an error code indicating the status of the created writer.
565 ErrorOr
<std::unique_ptr
<SampleProfileWriter
>>
566 SampleProfileWriter::create(StringRef Filename
, SampleProfileFormat Format
) {
568 std::unique_ptr
<raw_ostream
> OS
;
569 if (Format
== SPF_Binary
|| Format
== SPF_Ext_Binary
||
570 Format
== SPF_Compact_Binary
)
571 OS
.reset(new raw_fd_ostream(Filename
, EC
, sys::fs::OF_None
));
573 OS
.reset(new raw_fd_ostream(Filename
, EC
, sys::fs::OF_Text
));
577 return create(OS
, Format
);
580 /// Create a sample profile stream writer based on the specified format.
582 /// \param OS The output stream to store the profile data to.
584 /// \param Format Encoding format for the profile file.
586 /// \returns an error code indicating the status of the created writer.
587 ErrorOr
<std::unique_ptr
<SampleProfileWriter
>>
588 SampleProfileWriter::create(std::unique_ptr
<raw_ostream
> &OS
,
589 SampleProfileFormat Format
) {
591 std::unique_ptr
<SampleProfileWriter
> Writer
;
593 if (Format
== SPF_Binary
)
594 Writer
.reset(new SampleProfileWriterRawBinary(OS
));
595 else if (Format
== SPF_Ext_Binary
)
596 Writer
.reset(new SampleProfileWriterExtBinary(OS
));
597 else if (Format
== SPF_Compact_Binary
)
598 Writer
.reset(new SampleProfileWriterCompactBinary(OS
));
599 else if (Format
== SPF_Text
)
600 Writer
.reset(new SampleProfileWriterText(OS
));
601 else if (Format
== SPF_GCC
)
602 EC
= sampleprof_error::unsupported_writing_format
;
604 EC
= sampleprof_error::unrecognized_format
;
609 Writer
->Format
= Format
;
610 return std::move(Writer
);
613 void SampleProfileWriter::computeSummary(
614 const StringMap
<FunctionSamples
> &ProfileMap
) {
615 SampleProfileSummaryBuilder
Builder(ProfileSummaryBuilder::DefaultCutoffs
);
616 for (const auto &I
: ProfileMap
) {
617 const FunctionSamples
&Profile
= I
.second
;
618 Builder
.addRecord(Profile
);
620 Summary
= Builder
.getSummary();