llvm/lib/Transforms/Utils/SampleProfileLoaderBaseUtil.cpp

   1 //===- SampleProfileLoaderBaseUtil.cpp - Profile loader Util func ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the SampleProfileLoader base utility functions.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
  14
  15 namespace llvm {
  16
  17 cl::opt<unsigned> SampleProfileMaxPropagateIterations(
  18     "sample-profile-max-propagate-iterations", cl::init(100),
  19     cl::desc("Maximum number of iterations to go through when propagating "
  20              "sample block/edge weights through the CFG."));
  21
  22 cl::opt<unsigned> SampleProfileRecordCoverage(
  23     "sample-profile-check-record-coverage", cl::init(0), cl::value_desc("N"),
  24     cl::desc("Emit a warning if less than N% of records in the input profile "
  25              "are matched to the IR."));
  26
  27 cl::opt<unsigned> SampleProfileSampleCoverage(
  28     "sample-profile-check-sample-coverage", cl::init(0), cl::value_desc("N"),
  29     cl::desc("Emit a warning if less than N% of samples in the input profile "
  30              "are matched to the IR."));
  31
  32 cl::opt<bool> NoWarnSampleUnused(
  33     "no-warn-sample-unused", cl::init(false), cl::Hidden,
  34     cl::desc("Use this option to turn off/on warnings about function with "
  35              "samples but without debug information to use those samples. "));
  36
  37 namespace sampleprofutil {
  38
  39 /// Return true if the given callsite is hot wrt to hot cutoff threshold.
  40 ///
  41 /// Functions that were inlined in the original binary will be represented
  42 /// in the inline stack in the sample profile. If the profile shows that
  43 /// the original inline decision was "good" (i.e., the callsite is executed
  44 /// frequently), then we will recreate the inline decision and apply the
  45 /// profile from the inlined callsite.
  46 ///
  47 /// To decide whether an inlined callsite is hot, we compare the callsite
  48 /// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
  49 /// regarded as hot if the count is above the cutoff value.
  50 ///
  51 /// When ProfileAccurateForSymsInList is enabled and profile symbol list
  52 /// is present, functions in the profile symbol list but without profile will
  53 /// be regarded as cold and much less inlining will happen in CGSCC inlining
  54 /// pass, so we tend to lower the hot criteria here to allow more early
  55 /// inlining to happen for warm callsites and it is helpful for performance.
  56 bool callsiteIsHot(const FunctionSamples *CallsiteFS, ProfileSummaryInfo *PSI,
  57                    bool ProfAccForSymsInList) {
  58   if (!CallsiteFS)
  59     return false; // The callsite was not inlined in the original binary.
  60
  61   assert(PSI && "PSI is expected to be non null");
  62   uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
  63   if (ProfAccForSymsInList)
  64     return !PSI->isColdCount(CallsiteTotalSamples);
  65   else
  66     return PSI->isHotCount(CallsiteTotalSamples);
  67 }
  68
  69 /// Mark as used the sample record for the given function samples at
  70 /// (LineOffset, Discriminator).
  71 ///
  72 /// \returns true if this is the first time we mark the given record.
  73 bool SampleCoverageTracker::markSamplesUsed(const FunctionSamples *FS,
  74                                             uint32_t LineOffset,
  75                                             uint32_t Discriminator,
  76                                             uint64_t Samples) {
  77   LineLocation Loc(LineOffset, Discriminator);
  78   unsigned &Count = SampleCoverage[FS][Loc];
  79   bool FirstTime = (++Count == 1);
  80   if (FirstTime)
  81     TotalUsedSamples += Samples;
  82   return FirstTime;
  83 }
  84
  85 /// Return the number of sample records that were applied from this profile.
  86 ///
  87 /// This count does not include records from cold inlined callsites.
  88 unsigned
  89 SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
  90                                         ProfileSummaryInfo *PSI) const {
  91   auto I = SampleCoverage.find(FS);
  92
  93   // The size of the coverage map for FS represents the number of records
  94   // that were marked used at least once.
  95   unsigned Count = (I != SampleCoverage.end()) ? I->second.size() : 0;
  96
  97   // If there are inlined callsites in this function, count the samples found
  98   // in the respective bodies. However, do not bother counting callees with 0
  99   // total samples, these are callees that were never invoked at runtime.
 100   for (const auto &I : FS->getCallsiteSamples())
 101     for (const auto &J : I.second) {
 102       const FunctionSamples *CalleeSamples = &J.second;
 103       if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
 104         Count += countUsedRecords(CalleeSamples, PSI);
 105     }
 106
 107   return Count;
 108 }
 109
 110 /// Return the number of sample records in the body of this profile.
 111 ///
 112 /// This count does not include records from cold inlined callsites.
 113 unsigned
 114 SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
 115                                         ProfileSummaryInfo *PSI) const {
 116   unsigned Count = FS->getBodySamples().size();
 117
 118   // Only count records in hot callsites.
 119   for (const auto &I : FS->getCallsiteSamples())
 120     for (const auto &J : I.second) {
 121       const FunctionSamples *CalleeSamples = &J.second;
 122       if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
 123         Count += countBodyRecords(CalleeSamples, PSI);
 124     }
 125
 126   return Count;
 127 }
 128
 129 /// Return the number of samples collected in the body of this profile.
 130 ///
 131 /// This count does not include samples from cold inlined callsites.
 132 uint64_t
 133 SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
 134                                         ProfileSummaryInfo *PSI) const {
 135   uint64_t Total = 0;
 136   for (const auto &I : FS->getBodySamples())
 137     Total += I.second.getSamples();
 138
 139   // Only count samples in hot callsites.
 140   for (const auto &I : FS->getCallsiteSamples())
 141     for (const auto &J : I.second) {
 142       const FunctionSamples *CalleeSamples = &J.second;
 143       if (callsiteIsHot(CalleeSamples, PSI, ProfAccForSymsInList))
 144         Total += countBodySamples(CalleeSamples, PSI);
 145     }
 146
 147   return Total;
 148 }
 149
 150 /// Return the fraction of sample records used in this profile.
 151 ///
 152 /// The returned value is an unsigned integer in the range 0-100 indicating
 153 /// the percentage of sample records that were used while applying this
 154 /// profile to the associated function.
 155 unsigned SampleCoverageTracker::computeCoverage(unsigned Used,
 156                                                 unsigned Total) const {
 157   assert(Used <= Total &&
 158          "number of used records cannot exceed the total number of records");
 159   return Total > 0 ? Used * 100 / Total : 100;
 160 }
 161
 162 /// Create a global variable to flag FSDiscriminators are used.
 163 void createFSDiscriminatorVariable(Module *M) {
 164   const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";
 165   if (M->getGlobalVariable(FSDiscriminatorVar))
 166     return;
 167
 168   auto &Context = M->getContext();
 169   // Place this variable to llvm.used so it won't be GC'ed.
 170   appendToUsed(*M, {new GlobalVariable(*M, Type::getInt1Ty(Context), true,
 171                                        GlobalValue::WeakODRLinkage,
 172                                        ConstantInt::getTrue(Context),
 173                                        FSDiscriminatorVar)});
 174 }
 175
 176 } // end of namespace sampleprofutil
 177 } // end of namespace llvm