[InstCombine] Signed saturation tests. NFC
[llvm-complete.git] / lib / ProfileData / SampleProfWriter.cpp
blob8d09af31f94bf30a5d5233657240aeee867bc47b
1 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class that writes LLVM sample profiles. It
10 // supports two file formats: text and binary. The textual representation
11 // is useful for debugging and testing purposes. The binary representation
12 // is more compact, resulting in smaller file sizes. However, they can
13 // both be used interchangeably.
15 // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16 // supported formats.
18 //===----------------------------------------------------------------------===//
20 #include "llvm/ProfileData/SampleProfWriter.h"
21 #include "llvm/ADT/StringRef.h"
22 #include "llvm/ProfileData/ProfileCommon.h"
23 #include "llvm/ProfileData/SampleProf.h"
24 #include "llvm/Support/Compression.h"
25 #include "llvm/Support/Endian.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/ErrorOr.h"
28 #include "llvm/Support/FileSystem.h"
29 #include "llvm/Support/LEB128.h"
30 #include "llvm/Support/MD5.h"
31 #include "llvm/Support/raw_ostream.h"
32 #include <algorithm>
33 #include <cstdint>
34 #include <memory>
35 #include <set>
36 #include <system_error>
37 #include <utility>
38 #include <vector>
40 using namespace llvm;
41 using namespace sampleprof;
43 std::error_code SampleProfileWriter::writeFuncProfiles(
44 const StringMap<FunctionSamples> &ProfileMap) {
45 // Sort the ProfileMap by total samples.
46 typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples;
47 std::vector<NameFunctionSamples> V;
48 for (const auto &I : ProfileMap)
49 V.push_back(std::make_pair(I.getKey(), &I.second));
51 llvm::stable_sort(
52 V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) {
53 if (A.second->getTotalSamples() == B.second->getTotalSamples())
54 return A.first > B.first;
55 return A.second->getTotalSamples() > B.second->getTotalSamples();
56 });
58 for (const auto &I : V) {
59 if (std::error_code EC = writeSample(*I.second))
60 return EC;
62 return sampleprof_error::success;
65 std::error_code
66 SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
67 if (std::error_code EC = writeHeader(ProfileMap))
68 return EC;
70 if (std::error_code EC = writeFuncProfiles(ProfileMap))
71 return EC;
73 return sampleprof_error::success;
76 SecHdrTableEntry &
77 SampleProfileWriterExtBinaryBase::getEntryInLayout(SecType Type) {
78 auto SecIt = std::find_if(
79 SectionHdrLayout.begin(), SectionHdrLayout.end(),
80 [=](const auto &Entry) -> bool { return Entry.Type == Type; });
81 return *SecIt;
84 /// Return the current position and prepare to use it as the start
85 /// position of a section.
86 uint64_t SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type) {
87 uint64_t SectionStart = OutputStream->tell();
88 auto &Entry = getEntryInLayout(Type);
89 // Use LocalBuf as a temporary output for writting data.
90 if (hasSecFlag(Entry, SecFlagCompress))
91 LocalBufStream.swap(OutputStream);
92 return SectionStart;
95 std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
96 if (!llvm::zlib::isAvailable())
97 return sampleprof_error::zlib_unavailable;
98 std::string &UncompressedStrings =
99 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
100 if (UncompressedStrings.size() == 0)
101 return sampleprof_error::success;
102 auto &OS = *OutputStream;
103 SmallString<128> CompressedStrings;
104 llvm::Error E = zlib::compress(UncompressedStrings, CompressedStrings,
105 zlib::BestSizeCompression);
106 if (E)
107 return sampleprof_error::compress_failed;
108 encodeULEB128(UncompressedStrings.size(), OS);
109 encodeULEB128(CompressedStrings.size(), OS);
110 OS << CompressedStrings.str();
111 UncompressedStrings.clear();
112 return sampleprof_error::success;
115 /// Add a new section into section header table.
116 std::error_code
117 SampleProfileWriterExtBinaryBase::addNewSection(SecType Type,
118 uint64_t SectionStart) {
119 auto Entry = getEntryInLayout(Type);
120 if (hasSecFlag(Entry, SecFlagCompress)) {
121 LocalBufStream.swap(OutputStream);
122 if (std::error_code EC = compressAndOutput())
123 return EC;
125 SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
126 OutputStream->tell() - SectionStart});
127 return sampleprof_error::success;
130 std::error_code SampleProfileWriterExtBinaryBase::write(
131 const StringMap<FunctionSamples> &ProfileMap) {
132 if (std::error_code EC = writeHeader(ProfileMap))
133 return EC;
135 std::string LocalBuf;
136 LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
137 if (std::error_code EC = writeSections(ProfileMap))
138 return EC;
140 if (std::error_code EC = writeSecHdrTable())
141 return EC;
143 return sampleprof_error::success;
146 std::error_code
147 SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) {
148 uint64_t Offset = OutputStream->tell();
149 StringRef Name = S.getName();
150 FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
151 encodeULEB128(S.getHeadSamples(), *OutputStream);
152 return writeBody(S);
155 std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() {
156 auto &OS = *OutputStream;
158 // Write out the table size.
159 encodeULEB128(FuncOffsetTable.size(), OS);
161 // Write out FuncOffsetTable.
162 for (auto entry : FuncOffsetTable) {
163 writeNameIdx(entry.first);
164 encodeULEB128(entry.second, OS);
166 return sampleprof_error::success;
169 std::error_code SampleProfileWriterExtBinary::writeSections(
170 const StringMap<FunctionSamples> &ProfileMap) {
171 uint64_t SectionStart = markSectionStart(SecProfSummary);
172 computeSummary(ProfileMap);
173 if (auto EC = writeSummary())
174 return EC;
175 if (std::error_code EC = addNewSection(SecProfSummary, SectionStart))
176 return EC;
178 // Generate the name table for all the functions referenced in the profile.
179 SectionStart = markSectionStart(SecNameTable);
180 for (const auto &I : ProfileMap) {
181 addName(I.first());
182 addNames(I.second);
184 writeNameTable();
185 if (std::error_code EC = addNewSection(SecNameTable, SectionStart))
186 return EC;
188 SectionStart = markSectionStart(SecLBRProfile);
189 SecLBRProfileStart = OutputStream->tell();
190 if (std::error_code EC = writeFuncProfiles(ProfileMap))
191 return EC;
192 if (std::error_code EC = addNewSection(SecLBRProfile, SectionStart))
193 return EC;
195 if (ProfSymList && ProfSymList->toCompress())
196 setToCompressSection(SecProfileSymbolList);
198 SectionStart = markSectionStart(SecProfileSymbolList);
199 if (ProfSymList && ProfSymList->size() > 0)
200 if (std::error_code EC = ProfSymList->write(*OutputStream))
201 return EC;
202 if (std::error_code EC = addNewSection(SecProfileSymbolList, SectionStart))
203 return EC;
205 SectionStart = markSectionStart(SecFuncOffsetTable);
206 if (std::error_code EC = writeFuncOffsetTable())
207 return EC;
208 if (std::error_code EC = addNewSection(SecFuncOffsetTable, SectionStart))
209 return EC;
211 return sampleprof_error::success;
214 std::error_code SampleProfileWriterCompactBinary::write(
215 const StringMap<FunctionSamples> &ProfileMap) {
216 if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
217 return EC;
218 if (std::error_code EC = writeFuncOffsetTable())
219 return EC;
220 return sampleprof_error::success;
223 /// Write samples to a text file.
225 /// Note: it may be tempting to implement this in terms of
226 /// FunctionSamples::print(). Please don't. The dump functionality is intended
227 /// for debugging and has no specified form.
229 /// The format used here is more structured and deliberate because
230 /// it needs to be parsed by the SampleProfileReaderText class.
231 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
232 auto &OS = *OutputStream;
233 OS << S.getName() << ":" << S.getTotalSamples();
234 if (Indent == 0)
235 OS << ":" << S.getHeadSamples();
236 OS << "\n";
238 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
239 for (const auto &I : SortedSamples.get()) {
240 LineLocation Loc = I->first;
241 const SampleRecord &Sample = I->second;
242 OS.indent(Indent + 1);
243 if (Loc.Discriminator == 0)
244 OS << Loc.LineOffset << ": ";
245 else
246 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
248 OS << Sample.getSamples();
250 for (const auto &J : Sample.getSortedCallTargets())
251 OS << " " << J.first << ":" << J.second;
252 OS << "\n";
255 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
256 S.getCallsiteSamples());
257 Indent += 1;
258 for (const auto &I : SortedCallsiteSamples.get())
259 for (const auto &FS : I->second) {
260 LineLocation Loc = I->first;
261 const FunctionSamples &CalleeSamples = FS.second;
262 OS.indent(Indent);
263 if (Loc.Discriminator == 0)
264 OS << Loc.LineOffset << ": ";
265 else
266 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
267 if (std::error_code EC = writeSample(CalleeSamples))
268 return EC;
270 Indent -= 1;
272 return sampleprof_error::success;
275 std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
276 const auto &ret = NameTable.find(FName);
277 if (ret == NameTable.end())
278 return sampleprof_error::truncated_name_table;
279 encodeULEB128(ret->second, *OutputStream);
280 return sampleprof_error::success;
283 void SampleProfileWriterBinary::addName(StringRef FName) {
284 NameTable.insert(std::make_pair(FName, 0));
287 void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
288 // Add all the names in indirect call targets.
289 for (const auto &I : S.getBodySamples()) {
290 const SampleRecord &Sample = I.second;
291 for (const auto &J : Sample.getCallTargets())
292 addName(J.first());
295 // Recursively add all the names for inlined callsites.
296 for (const auto &J : S.getCallsiteSamples())
297 for (const auto &FS : J.second) {
298 const FunctionSamples &CalleeSamples = FS.second;
299 addName(CalleeSamples.getName());
300 addNames(CalleeSamples);
304 void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) {
305 // Sort the names to make NameTable deterministic.
306 for (const auto &I : NameTable)
307 V.insert(I.first);
308 int i = 0;
309 for (const StringRef &N : V)
310 NameTable[N] = i++;
313 std::error_code SampleProfileWriterBinary::writeNameTable() {
314 auto &OS = *OutputStream;
315 std::set<StringRef> V;
316 stablizeNameTable(V);
318 // Write out the name table.
319 encodeULEB128(NameTable.size(), OS);
320 for (auto N : V) {
321 OS << N;
322 encodeULEB128(0, OS);
324 return sampleprof_error::success;
327 std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
328 auto &OS = *OutputStream;
330 // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
331 auto &OFS = static_cast<raw_fd_ostream &>(OS);
332 uint64_t FuncOffsetTableStart = OS.tell();
333 if (OFS.seek(TableOffset) == (uint64_t)-1)
334 return sampleprof_error::ostream_seek_unsupported;
335 support::endian::Writer Writer(*OutputStream, support::little);
336 Writer.write(FuncOffsetTableStart);
337 if (OFS.seek(FuncOffsetTableStart) == (uint64_t)-1)
338 return sampleprof_error::ostream_seek_unsupported;
340 // Write out the table size.
341 encodeULEB128(FuncOffsetTable.size(), OS);
343 // Write out FuncOffsetTable.
344 for (auto entry : FuncOffsetTable) {
345 writeNameIdx(entry.first);
346 encodeULEB128(entry.second, OS);
348 return sampleprof_error::success;
351 std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
352 auto &OS = *OutputStream;
353 std::set<StringRef> V;
354 stablizeNameTable(V);
356 // Write out the name table.
357 encodeULEB128(NameTable.size(), OS);
358 for (auto N : V) {
359 encodeULEB128(MD5Hash(N), OS);
361 return sampleprof_error::success;
364 std::error_code
365 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
366 auto &OS = *OutputStream;
367 // Write file magic identifier.
368 encodeULEB128(SPMagic(Format), OS);
369 encodeULEB128(SPVersion(), OS);
370 return sampleprof_error::success;
373 std::error_code SampleProfileWriterBinary::writeHeader(
374 const StringMap<FunctionSamples> &ProfileMap) {
375 writeMagicIdent(Format);
377 computeSummary(ProfileMap);
378 if (auto EC = writeSummary())
379 return EC;
381 // Generate the name table for all the functions referenced in the profile.
382 for (const auto &I : ProfileMap) {
383 addName(I.first());
384 addNames(I.second);
387 writeNameTable();
388 return sampleprof_error::success;
391 void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
392 for (auto &Entry : SectionHdrLayout)
393 addSecFlags(Entry, SecFlagCompress);
396 void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {
397 addSectionFlags(Type, SecFlagCompress);
400 void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type,
401 SecFlags Flags) {
402 for (auto &Entry : SectionHdrLayout) {
403 if (Entry.Type == Type)
404 addSecFlags(Entry, Flags);
408 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
409 support::endian::Writer Writer(*OutputStream, support::little);
411 Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
412 SecHdrTableOffset = OutputStream->tell();
413 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
414 Writer.write(static_cast<uint64_t>(-1));
415 Writer.write(static_cast<uint64_t>(-1));
416 Writer.write(static_cast<uint64_t>(-1));
417 Writer.write(static_cast<uint64_t>(-1));
421 std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
422 auto &OFS = static_cast<raw_fd_ostream &>(*OutputStream);
423 uint64_t Saved = OutputStream->tell();
425 // Set OutputStream to the location saved in SecHdrTableOffset.
426 if (OFS.seek(SecHdrTableOffset) == (uint64_t)-1)
427 return sampleprof_error::ostream_seek_unsupported;
428 support::endian::Writer Writer(*OutputStream, support::little);
430 DenseMap<uint32_t, uint32_t> IndexMap;
431 for (uint32_t i = 0; i < SecHdrTable.size(); i++) {
432 IndexMap.insert({static_cast<uint32_t>(SecHdrTable[i].Type), i});
435 // Write the section header table in the order specified in
436 // SectionHdrLayout. That is the sections order Reader will see.
437 // Note that the sections order in which Reader expects to read
438 // may be different from the order in which Writer is able to
439 // write, so we need to adjust the order in SecHdrTable to be
440 // consistent with SectionHdrLayout when we write SecHdrTable
441 // to the memory.
442 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
443 uint32_t idx = IndexMap[static_cast<uint32_t>(SectionHdrLayout[i].Type)];
444 Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Type));
445 Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Flags));
446 Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Offset));
447 Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Size));
450 // Reset OutputStream.
451 if (OFS.seek(Saved) == (uint64_t)-1)
452 return sampleprof_error::ostream_seek_unsupported;
454 return sampleprof_error::success;
457 std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
458 const StringMap<FunctionSamples> &ProfileMap) {
459 auto &OS = *OutputStream;
460 FileStart = OS.tell();
461 writeMagicIdent(Format);
463 allocSecHdrTable();
464 return sampleprof_error::success;
467 std::error_code SampleProfileWriterCompactBinary::writeHeader(
468 const StringMap<FunctionSamples> &ProfileMap) {
469 support::endian::Writer Writer(*OutputStream, support::little);
470 if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
471 return EC;
473 // Reserve a slot for the offset of function offset table. The slot will
474 // be populated with the offset of FuncOffsetTable later.
475 TableOffset = OutputStream->tell();
476 Writer.write(static_cast<uint64_t>(-2));
477 return sampleprof_error::success;
480 std::error_code SampleProfileWriterBinary::writeSummary() {
481 auto &OS = *OutputStream;
482 encodeULEB128(Summary->getTotalCount(), OS);
483 encodeULEB128(Summary->getMaxCount(), OS);
484 encodeULEB128(Summary->getMaxFunctionCount(), OS);
485 encodeULEB128(Summary->getNumCounts(), OS);
486 encodeULEB128(Summary->getNumFunctions(), OS);
487 std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary();
488 encodeULEB128(Entries.size(), OS);
489 for (auto Entry : Entries) {
490 encodeULEB128(Entry.Cutoff, OS);
491 encodeULEB128(Entry.MinCount, OS);
492 encodeULEB128(Entry.NumCounts, OS);
494 return sampleprof_error::success;
496 std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
497 auto &OS = *OutputStream;
499 if (std::error_code EC = writeNameIdx(S.getName()))
500 return EC;
502 encodeULEB128(S.getTotalSamples(), OS);
504 // Emit all the body samples.
505 encodeULEB128(S.getBodySamples().size(), OS);
506 for (const auto &I : S.getBodySamples()) {
507 LineLocation Loc = I.first;
508 const SampleRecord &Sample = I.second;
509 encodeULEB128(Loc.LineOffset, OS);
510 encodeULEB128(Loc.Discriminator, OS);
511 encodeULEB128(Sample.getSamples(), OS);
512 encodeULEB128(Sample.getCallTargets().size(), OS);
513 for (const auto &J : Sample.getSortedCallTargets()) {
514 StringRef Callee = J.first;
515 uint64_t CalleeSamples = J.second;
516 if (std::error_code EC = writeNameIdx(Callee))
517 return EC;
518 encodeULEB128(CalleeSamples, OS);
522 // Recursively emit all the callsite samples.
523 uint64_t NumCallsites = 0;
524 for (const auto &J : S.getCallsiteSamples())
525 NumCallsites += J.second.size();
526 encodeULEB128(NumCallsites, OS);
527 for (const auto &J : S.getCallsiteSamples())
528 for (const auto &FS : J.second) {
529 LineLocation Loc = J.first;
530 const FunctionSamples &CalleeSamples = FS.second;
531 encodeULEB128(Loc.LineOffset, OS);
532 encodeULEB128(Loc.Discriminator, OS);
533 if (std::error_code EC = writeBody(CalleeSamples))
534 return EC;
537 return sampleprof_error::success;
540 /// Write samples of a top-level function to a binary file.
542 /// \returns true if the samples were written successfully, false otherwise.
543 std::error_code
544 SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
545 encodeULEB128(S.getHeadSamples(), *OutputStream);
546 return writeBody(S);
549 std::error_code
550 SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {
551 uint64_t Offset = OutputStream->tell();
552 StringRef Name = S.getName();
553 FuncOffsetTable[Name] = Offset;
554 encodeULEB128(S.getHeadSamples(), *OutputStream);
555 return writeBody(S);
558 /// Create a sample profile file writer based on the specified format.
560 /// \param Filename The file to create.
562 /// \param Format Encoding format for the profile file.
564 /// \returns an error code indicating the status of the created writer.
565 ErrorOr<std::unique_ptr<SampleProfileWriter>>
566 SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
567 std::error_code EC;
568 std::unique_ptr<raw_ostream> OS;
569 if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
570 Format == SPF_Compact_Binary)
571 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
572 else
573 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_Text));
574 if (EC)
575 return EC;
577 return create(OS, Format);
580 /// Create a sample profile stream writer based on the specified format.
582 /// \param OS The output stream to store the profile data to.
584 /// \param Format Encoding format for the profile file.
586 /// \returns an error code indicating the status of the created writer.
587 ErrorOr<std::unique_ptr<SampleProfileWriter>>
588 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
589 SampleProfileFormat Format) {
590 std::error_code EC;
591 std::unique_ptr<SampleProfileWriter> Writer;
593 if (Format == SPF_Binary)
594 Writer.reset(new SampleProfileWriterRawBinary(OS));
595 else if (Format == SPF_Ext_Binary)
596 Writer.reset(new SampleProfileWriterExtBinary(OS));
597 else if (Format == SPF_Compact_Binary)
598 Writer.reset(new SampleProfileWriterCompactBinary(OS));
599 else if (Format == SPF_Text)
600 Writer.reset(new SampleProfileWriterText(OS));
601 else if (Format == SPF_GCC)
602 EC = sampleprof_error::unsupported_writing_format;
603 else
604 EC = sampleprof_error::unrecognized_format;
606 if (EC)
607 return EC;
609 Writer->Format = Format;
610 return std::move(Writer);
613 void SampleProfileWriter::computeSummary(
614 const StringMap<FunctionSamples> &ProfileMap) {
615 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
616 for (const auto &I : ProfileMap) {
617 const FunctionSamples &Profile = I.second;
618 Builder.addRecord(Profile);
620 Summary = Builder.getSummary();