lib/ProfileData/SampleProfWriter.cpp

   1 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the class that writes LLVM sample profiles. It
  10 // supports two file formats: text and binary. The textual representation
  11 // is useful for debugging and testing purposes. The binary representation
  12 // is more compact, resulting in smaller file sizes. However, they can
  13 // both be used interchangeably.
  14 //
  15 // See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
  16 // supported formats.
  17 //
  18 //===----------------------------------------------------------------------===//
  19
  20 #include "llvm/ProfileData/SampleProfWriter.h"
  21 #include "llvm/ADT/StringRef.h"
  22 #include "llvm/ProfileData/ProfileCommon.h"
  23 #include "llvm/ProfileData/SampleProf.h"
  24 #include "llvm/Support/Endian.h"
  25 #include "llvm/Support/EndianStream.h"
  26 #include "llvm/Support/ErrorOr.h"
  27 #include "llvm/Support/FileSystem.h"
  28 #include "llvm/Support/LEB128.h"
  29 #include "llvm/Support/MD5.h"
  30 #include "llvm/Support/raw_ostream.h"
  31 #include <algorithm>
  32 #include <cstdint>
  33 #include <memory>
  34 #include <set>
  35 #include <system_error>
  36 #include <utility>
  37 #include <vector>
  38
  39 using namespace llvm;
  40 using namespace sampleprof;
  41
  42 std::error_code SampleProfileWriter::writeFuncProfiles(
  43     const StringMap<FunctionSamples> &ProfileMap) {
  44   // Sort the ProfileMap by total samples.
  45   typedef std::pair<StringRef, const FunctionSamples *> NameFunctionSamples;
  46   std::vector<NameFunctionSamples> V;
  47   for (const auto &I : ProfileMap)
  48     V.push_back(std::make_pair(I.getKey(), &I.second));
  49
  50   llvm::stable_sort(
  51       V, [](const NameFunctionSamples &A, const NameFunctionSamples &B) {
  52         if (A.second->getTotalSamples() == B.second->getTotalSamples())
  53           return A.first > B.first;
  54         return A.second->getTotalSamples() > B.second->getTotalSamples();
  55       });
  56
  57   for (const auto &I : V) {
  58     if (std::error_code EC = writeSample(*I.second))
  59       return EC;
  60   }
  61   return sampleprof_error::success;
  62 }
  63
  64 std::error_code
  65 SampleProfileWriter::write(const StringMap<FunctionSamples> &ProfileMap) {
  66   if (std::error_code EC = writeHeader(ProfileMap))
  67     return EC;
  68
  69   if (std::error_code EC = writeFuncProfiles(ProfileMap))
  70     return EC;
  71
  72   return sampleprof_error::success;
  73 }
  74
  75 /// Return the current position and prepare to use it as the start
  76 /// position of a section.
  77 uint64_t SampleProfileWriterExtBinaryBase::markSectionStart() {
  78   return OutputStream->tell();
  79 }
  80
  81 /// Add a new section into section header table. Return the position
  82 /// of SectionEnd.
  83 uint64_t
  84 SampleProfileWriterExtBinaryBase::addNewSection(SecType Sec,
  85                                                 uint64_t SectionStart) {
  86   uint64_t SectionEnd = OutputStream->tell();
  87   SecHdrTable.push_back(
  88       {Sec, 0, SectionStart - FileStart, SectionEnd - SectionStart});
  89   return SectionEnd;
  90 }
  91
  92 std::error_code SampleProfileWriterExtBinaryBase::write(
  93     const StringMap<FunctionSamples> &ProfileMap) {
  94   if (std::error_code EC = writeHeader(ProfileMap))
  95     return EC;
  96
  97   if (std::error_code EC = writeSections(ProfileMap))
  98     return EC;
  99
 100   if (std::error_code EC = writeSecHdrTable())
 101     return EC;
 102
 103   return sampleprof_error::success;
 104 }
 105
 106 std::error_code SampleProfileWriterExtBinary::writeSections(
 107     const StringMap<FunctionSamples> &ProfileMap) {
 108   uint64_t SectionStart = markSectionStart();
 109   computeSummary(ProfileMap);
 110   if (auto EC = writeSummary())
 111     return EC;
 112   SectionStart = addNewSection(SecProfSummary, SectionStart);
 113
 114   // Generate the name table for all the functions referenced in the profile.
 115   for (const auto &I : ProfileMap) {
 116     addName(I.first());
 117     addNames(I.second);
 118   }
 119   writeNameTable();
 120   SectionStart = addNewSection(SecNameTable, SectionStart);
 121
 122   if (std::error_code EC = writeFuncProfiles(ProfileMap))
 123     return EC;
 124   SectionStart = addNewSection(SecLBRProfile, SectionStart);
 125
 126   if (ProfSymList && ProfSymList->size() > 0)
 127     if (std::error_code EC = ProfSymList->write(*OutputStream))
 128       return EC;
 129   addNewSection(SecProfileSymbolList, SectionStart);
 130
 131   return sampleprof_error::success;
 132 }
 133
 134 std::error_code SampleProfileWriterCompactBinary::write(
 135     const StringMap<FunctionSamples> &ProfileMap) {
 136   if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
 137     return EC;
 138   if (std::error_code EC = writeFuncOffsetTable())
 139     return EC;
 140   return sampleprof_error::success;
 141 }
 142
 143 /// Write samples to a text file.
 144 ///
 145 /// Note: it may be tempting to implement this in terms of
 146 /// FunctionSamples::print().  Please don't.  The dump functionality is intended
 147 /// for debugging and has no specified form.
 148 ///
 149 /// The format used here is more structured and deliberate because
 150 /// it needs to be parsed by the SampleProfileReaderText class.
 151 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
 152   auto &OS = *OutputStream;
 153   OS << S.getName() << ":" << S.getTotalSamples();
 154   if (Indent == 0)
 155     OS << ":" << S.getHeadSamples();
 156   OS << "\n";
 157
 158   SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());
 159   for (const auto &I : SortedSamples.get()) {
 160     LineLocation Loc = I->first;
 161     const SampleRecord &Sample = I->second;
 162     OS.indent(Indent + 1);
 163     if (Loc.Discriminator == 0)
 164       OS << Loc.LineOffset << ": ";
 165     else
 166       OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
 167
 168     OS << Sample.getSamples();
 169
 170     for (const auto &J : Sample.getSortedCallTargets())
 171       OS << " " << J.first << ":" << J.second;
 172     OS << "\n";
 173   }
 174
 175   SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(
 176       S.getCallsiteSamples());
 177   Indent += 1;
 178   for (const auto &I : SortedCallsiteSamples.get())
 179     for (const auto &FS : I->second) {
 180       LineLocation Loc = I->first;
 181       const FunctionSamples &CalleeSamples = FS.second;
 182       OS.indent(Indent);
 183       if (Loc.Discriminator == 0)
 184         OS << Loc.LineOffset << ": ";
 185       else
 186         OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
 187       if (std::error_code EC = writeSample(CalleeSamples))
 188         return EC;
 189     }
 190   Indent -= 1;
 191
 192   return sampleprof_error::success;
 193 }
 194
 195 std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
 196   const auto &ret = NameTable.find(FName);
 197   if (ret == NameTable.end())
 198     return sampleprof_error::truncated_name_table;
 199   encodeULEB128(ret->second, *OutputStream);
 200   return sampleprof_error::success;
 201 }
 202
 203 void SampleProfileWriterBinary::addName(StringRef FName) {
 204   NameTable.insert(std::make_pair(FName, 0));
 205 }
 206
 207 void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
 208   // Add all the names in indirect call targets.
 209   for (const auto &I : S.getBodySamples()) {
 210     const SampleRecord &Sample = I.second;
 211     for (const auto &J : Sample.getCallTargets())
 212       addName(J.first());
 213   }
 214
 215   // Recursively add all the names for inlined callsites.
 216   for (const auto &J : S.getCallsiteSamples())
 217     for (const auto &FS : J.second) {
 218       const FunctionSamples &CalleeSamples = FS.second;
 219       addName(CalleeSamples.getName());
 220       addNames(CalleeSamples);
 221     }
 222 }
 223
 224 void SampleProfileWriterBinary::stablizeNameTable(std::set<StringRef> &V) {
 225   // Sort the names to make NameTable deterministic.
 226   for (const auto &I : NameTable)
 227     V.insert(I.first);
 228   int i = 0;
 229   for (const StringRef &N : V)
 230     NameTable[N] = i++;
 231 }
 232
 233 std::error_code SampleProfileWriterBinary::writeNameTable() {
 234   auto &OS = *OutputStream;
 235   std::set<StringRef> V;
 236   stablizeNameTable(V);
 237
 238   // Write out the name table.
 239   encodeULEB128(NameTable.size(), OS);
 240   for (auto N : V) {
 241     OS << N;
 242     encodeULEB128(0, OS);
 243   }
 244   return sampleprof_error::success;
 245 }
 246
 247 std::error_code SampleProfileWriterCompactBinary::writeFuncOffsetTable() {
 248   auto &OS = *OutputStream;
 249
 250   // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
 251   auto &OFS = static_cast<raw_fd_ostream &>(OS);
 252   uint64_t FuncOffsetTableStart = OS.tell();
 253   if (OFS.seek(TableOffset) == (uint64_t)-1)
 254     return sampleprof_error::ostream_seek_unsupported;
 255   support::endian::Writer Writer(*OutputStream, support::little);
 256   Writer.write(FuncOffsetTableStart);
 257   if (OFS.seek(FuncOffsetTableStart) == (uint64_t)-1)
 258     return sampleprof_error::ostream_seek_unsupported;
 259
 260   // Write out the table size.
 261   encodeULEB128(FuncOffsetTable.size(), OS);
 262
 263   // Write out FuncOffsetTable.
 264   for (auto entry : FuncOffsetTable) {
 265     writeNameIdx(entry.first);
 266     encodeULEB128(entry.second, OS);
 267   }
 268   return sampleprof_error::success;
 269 }
 270
 271 std::error_code SampleProfileWriterCompactBinary::writeNameTable() {
 272   auto &OS = *OutputStream;
 273   std::set<StringRef> V;
 274   stablizeNameTable(V);
 275
 276   // Write out the name table.
 277   encodeULEB128(NameTable.size(), OS);
 278   for (auto N : V) {
 279     encodeULEB128(MD5Hash(N), OS);
 280   }
 281   return sampleprof_error::success;
 282 }
 283
 284 std::error_code
 285 SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {
 286   auto &OS = *OutputStream;
 287   // Write file magic identifier.
 288   encodeULEB128(SPMagic(Format), OS);
 289   encodeULEB128(SPVersion(), OS);
 290   return sampleprof_error::success;
 291 }
 292
 293 std::error_code SampleProfileWriterBinary::writeHeader(
 294     const StringMap<FunctionSamples> &ProfileMap) {
 295   writeMagicIdent(Format);
 296
 297   computeSummary(ProfileMap);
 298   if (auto EC = writeSummary())
 299     return EC;
 300
 301   // Generate the name table for all the functions referenced in the profile.
 302   for (const auto &I : ProfileMap) {
 303     addName(I.first());
 304     addNames(I.second);
 305   }
 306
 307   writeNameTable();
 308   return sampleprof_error::success;
 309 }
 310
 311 void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
 312   support::endian::Writer Writer(*OutputStream, support::little);
 313
 314   Writer.write(static_cast<uint64_t>(SectionLayout.size()));
 315   SecHdrTableOffset = OutputStream->tell();
 316   for (uint32_t i = 0; i < SectionLayout.size(); i++) {
 317     Writer.write(static_cast<uint64_t>(-1));
 318     Writer.write(static_cast<uint64_t>(-1));
 319     Writer.write(static_cast<uint64_t>(-1));
 320     Writer.write(static_cast<uint64_t>(-1));
 321   }
 322 }
 323
 324 std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
 325   auto &OFS = static_cast<raw_fd_ostream &>(*OutputStream);
 326   uint64_t Saved = OutputStream->tell();
 327
 328   // Set OutputStream to the location saved in SecHdrTableOffset.
 329   if (OFS.seek(SecHdrTableOffset) == (uint64_t)-1)
 330     return sampleprof_error::ostream_seek_unsupported;
 331   support::endian::Writer Writer(*OutputStream, support::little);
 332
 333   DenseMap<uint32_t, uint32_t> IndexMap;
 334   for (uint32_t i = 0; i < SecHdrTable.size(); i++) {
 335     IndexMap.insert({static_cast<uint32_t>(SecHdrTable[i].Type), i});
 336   }
 337
 338   // Write the sections in the order specified in SectionLayout.
 339   // That is the sections order Reader will see. Note that the
 340   // sections order in which Reader expects to read may be different
 341   // from the order in which Writer is able to write, so we need
 342   // to adjust the order in SecHdrTable to be consistent with
 343   // SectionLayout when we write SecHdrTable to the memory.
 344   for (uint32_t i = 0; i < SectionLayout.size(); i++) {
 345     uint32_t idx = IndexMap[static_cast<uint32_t>(SectionLayout[i])];
 346     Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Type));
 347     Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Flag));
 348     Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Offset));
 349     Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Size));
 350   }
 351
 352   // Reset OutputStream.
 353   if (OFS.seek(Saved) == (uint64_t)-1)
 354     return sampleprof_error::ostream_seek_unsupported;
 355
 356   return sampleprof_error::success;
 357 }
 358
 359 std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
 360     const StringMap<FunctionSamples> &ProfileMap) {
 361   auto &OS = *OutputStream;
 362   FileStart = OS.tell();
 363   writeMagicIdent(Format);
 364
 365   initSectionLayout();
 366   allocSecHdrTable();
 367   return sampleprof_error::success;
 368 }
 369
 370 std::error_code SampleProfileWriterCompactBinary::writeHeader(
 371     const StringMap<FunctionSamples> &ProfileMap) {
 372   support::endian::Writer Writer(*OutputStream, support::little);
 373   if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
 374     return EC;
 375
 376   // Reserve a slot for the offset of function offset table. The slot will
 377   // be populated with the offset of FuncOffsetTable later.
 378   TableOffset = OutputStream->tell();
 379   Writer.write(static_cast<uint64_t>(-2));
 380   return sampleprof_error::success;
 381 }
 382
 383 std::error_code SampleProfileWriterBinary::writeSummary() {
 384   auto &OS = *OutputStream;
 385   encodeULEB128(Summary->getTotalCount(), OS);
 386   encodeULEB128(Summary->getMaxCount(), OS);
 387   encodeULEB128(Summary->getMaxFunctionCount(), OS);
 388   encodeULEB128(Summary->getNumCounts(), OS);
 389   encodeULEB128(Summary->getNumFunctions(), OS);
 390   std::vector<ProfileSummaryEntry> &Entries = Summary->getDetailedSummary();
 391   encodeULEB128(Entries.size(), OS);
 392   for (auto Entry : Entries) {
 393     encodeULEB128(Entry.Cutoff, OS);
 394     encodeULEB128(Entry.MinCount, OS);
 395     encodeULEB128(Entry.NumCounts, OS);
 396   }
 397   return sampleprof_error::success;
 398 }
 399 std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
 400   auto &OS = *OutputStream;
 401
 402   if (std::error_code EC = writeNameIdx(S.getName()))
 403     return EC;
 404
 405   encodeULEB128(S.getTotalSamples(), OS);
 406
 407   // Emit all the body samples.
 408   encodeULEB128(S.getBodySamples().size(), OS);
 409   for (const auto &I : S.getBodySamples()) {
 410     LineLocation Loc = I.first;
 411     const SampleRecord &Sample = I.second;
 412     encodeULEB128(Loc.LineOffset, OS);
 413     encodeULEB128(Loc.Discriminator, OS);
 414     encodeULEB128(Sample.getSamples(), OS);
 415     encodeULEB128(Sample.getCallTargets().size(), OS);
 416     for (const auto &J : Sample.getSortedCallTargets()) {
 417       StringRef Callee = J.first;
 418       uint64_t CalleeSamples = J.second;
 419       if (std::error_code EC = writeNameIdx(Callee))
 420         return EC;
 421       encodeULEB128(CalleeSamples, OS);
 422     }
 423   }
 424
 425   // Recursively emit all the callsite samples.
 426   uint64_t NumCallsites = 0;
 427   for (const auto &J : S.getCallsiteSamples())
 428     NumCallsites += J.second.size();
 429   encodeULEB128(NumCallsites, OS);
 430   for (const auto &J : S.getCallsiteSamples())
 431     for (const auto &FS : J.second) {
 432       LineLocation Loc = J.first;
 433       const FunctionSamples &CalleeSamples = FS.second;
 434       encodeULEB128(Loc.LineOffset, OS);
 435       encodeULEB128(Loc.Discriminator, OS);
 436       if (std::error_code EC = writeBody(CalleeSamples))
 437         return EC;
 438     }
 439
 440   return sampleprof_error::success;
 441 }
 442
 443 /// Write samples of a top-level function to a binary file.
 444 ///
 445 /// \returns true if the samples were written successfully, false otherwise.
 446 std::error_code
 447 SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {
 448   encodeULEB128(S.getHeadSamples(), *OutputStream);
 449   return writeBody(S);
 450 }
 451
 452 std::error_code
 453 SampleProfileWriterCompactBinary::writeSample(const FunctionSamples &S) {
 454   uint64_t Offset = OutputStream->tell();
 455   StringRef Name = S.getName();
 456   FuncOffsetTable[Name] = Offset;
 457   encodeULEB128(S.getHeadSamples(), *OutputStream);
 458   return writeBody(S);
 459 }
 460
 461 /// Create a sample profile file writer based on the specified format.
 462 ///
 463 /// \param Filename The file to create.
 464 ///
 465 /// \param Format Encoding format for the profile file.
 466 ///
 467 /// \returns an error code indicating the status of the created writer.
 468 ErrorOr<std::unique_ptr<SampleProfileWriter>>
 469 SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {
 470   std::error_code EC;
 471   std::unique_ptr<raw_ostream> OS;
 472   if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
 473       Format == SPF_Compact_Binary)
 474     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
 475   else
 476     OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_Text));
 477   if (EC)
 478     return EC;
 479
 480   return create(OS, Format);
 481 }
 482
 483 /// Create a sample profile stream writer based on the specified format.
 484 ///
 485 /// \param OS The output stream to store the profile data to.
 486 ///
 487 /// \param Format Encoding format for the profile file.
 488 ///
 489 /// \returns an error code indicating the status of the created writer.
 490 ErrorOr<std::unique_ptr<SampleProfileWriter>>
 491 SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
 492                             SampleProfileFormat Format) {
 493   std::error_code EC;
 494   std::unique_ptr<SampleProfileWriter> Writer;
 495
 496   if (Format == SPF_Binary)
 497     Writer.reset(new SampleProfileWriterRawBinary(OS));
 498   else if (Format == SPF_Ext_Binary)
 499     Writer.reset(new SampleProfileWriterExtBinary(OS));
 500   else if (Format == SPF_Compact_Binary)
 501     Writer.reset(new SampleProfileWriterCompactBinary(OS));
 502   else if (Format == SPF_Text)
 503     Writer.reset(new SampleProfileWriterText(OS));
 504   else if (Format == SPF_GCC)
 505     EC = sampleprof_error::unsupported_writing_format;
 506   else
 507     EC = sampleprof_error::unrecognized_format;
 508
 509   if (EC)
 510     return EC;
 511
 512   Writer->Format = Format;
 513   return std::move(Writer);
 514 }
 515
 516 void SampleProfileWriter::computeSummary(
 517     const StringMap<FunctionSamples> &ProfileMap) {
 518   SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
 519   for (const auto &I : ProfileMap) {
 520     const FunctionSamples &Profile = I.second;
 521     Builder.addRecord(Profile);
 522   }
 523   Summary = Builder.getSummary();
 524 }