llvm/lib/Analysis/ProfileSummaryInfo.cpp

   1 //===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a pass that provides access to the global profile summary
  10 // information.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Analysis/ProfileSummaryInfo.h"
  15 #include "llvm/Analysis/BlockFrequencyInfo.h"
  16 #include "llvm/IR/BasicBlock.h"
  17 #include "llvm/IR/Instructions.h"
  18 #include "llvm/IR/Module.h"
  19 #include "llvm/IR/ProfileSummary.h"
  20 #include "llvm/InitializePasses.h"
  21 #include "llvm/ProfileData/ProfileCommon.h"
  22 #include "llvm/Support/CommandLine.h"
  23 #include <optional>
  24 using namespace llvm;
  25
  26 // Knobs for profile summary based thresholds.
  27 namespace llvm {
  28 extern cl::opt<int> ProfileSummaryCutoffHot;
  29 extern cl::opt<int> ProfileSummaryCutoffCold;
  30 extern cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold;
  31 extern cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold;
  32 extern cl::opt<int> ProfileSummaryHotCount;
  33 extern cl::opt<int> ProfileSummaryColdCount;
  34 } // namespace llvm
  35
  36 static cl::opt<bool> PartialProfile(
  37     "partial-profile", cl::Hidden, cl::init(false),
  38     cl::desc("Specify the current profile is used as a partial profile."));
  39
  40 cl::opt<bool> ScalePartialSampleProfileWorkingSetSize(
  41     "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true),
  42     cl::desc(
  43         "If true, scale the working set size of the partial sample profile "
  44         "by the partial profile ratio to reflect the size of the program "
  45         "being compiled."));
  46
  47 static cl::opt<double> PartialSampleProfileWorkingSetSizeScaleFactor(
  48     "partial-sample-profile-working-set-size-scale-factor", cl::Hidden,
  49     cl::init(0.008),
  50     cl::desc("The scale factor used to scale the working set size of the "
  51              "partial sample profile along with the partial profile ratio. "
  52              "This includes the factor of the profile counter per block "
  53              "and the factor to scale the working set size to use the same "
  54              "shared thresholds as PGO."));
  55
  56 // The profile summary metadata may be attached either by the frontend or by
  57 // any backend passes (IR level instrumentation, for example). This method
  58 // checks if the Summary is null and if so checks if the summary metadata is now
  59 // available in the module and parses it to get the Summary object.
  60 void ProfileSummaryInfo::refresh() {
  61   if (hasProfileSummary())
  62     return;
  63   // First try to get context sensitive ProfileSummary.
  64   auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);
  65   if (SummaryMD)
  66     Summary.reset(ProfileSummary::getFromMD(SummaryMD));
  67
  68   if (!hasProfileSummary()) {
  69     // This will actually return PSK_Instr or PSK_Sample summary.
  70     SummaryMD = M->getProfileSummary(/* IsCS */ false);
  71     if (SummaryMD)
  72       Summary.reset(ProfileSummary::getFromMD(SummaryMD));
  73   }
  74   if (!hasProfileSummary())
  75     return;
  76   computeThresholds();
  77 }
  78
  79 std::optional<uint64_t> ProfileSummaryInfo::getProfileCount(
  80     const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const {
  81   assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) &&
  82          "We can only get profile count for call/invoke instruction.");
  83   if (hasSampleProfile()) {
  84     // In sample PGO mode, check if there is a profile metadata on the
  85     // instruction. If it is present, determine hotness solely based on that,
  86     // since the sampled entry count may not be accurate. If there is no
  87     // annotated on the instruction, return std::nullopt.
  88     uint64_t TotalCount;
  89     if (Call.extractProfTotalWeight(TotalCount))
  90       return TotalCount;
  91     return std::nullopt;
  92   }
  93   if (BFI)
  94     return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic);
  95   return std::nullopt;
  96 }
  97
  98 bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const {
  99   assert(hasPartialSampleProfile() && "Expect partial sample profile");
 100   return !F.getEntryCount();
 101 }
 102
 103 /// Returns true if the function's entry is a cold. If it returns false, it
 104 /// either means it is not cold or it is unknown whether it is cold or not (for
 105 /// example, no profile data is available).
 106 bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const {
 107   if (!F)
 108     return false;
 109   if (F->hasFnAttribute(Attribute::Cold))
 110     return true;
 111   if (!hasProfileSummary())
 112     return false;
 113   auto FunctionCount = F->getEntryCount();
 114   // FIXME: The heuristic used below for determining coldness is based on
 115   // preliminary SPEC tuning for inliner. This will eventually be a
 116   // convenience method that calls isHotCount.
 117   return FunctionCount && isColdCount(FunctionCount->getCount());
 118 }
 119
 120 /// Compute the hot and cold thresholds.
 121 void ProfileSummaryInfo::computeThresholds() {
 122   auto &DetailedSummary = Summary->getDetailedSummary();
 123   auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile(
 124       DetailedSummary, ProfileSummaryCutoffHot);
 125   HotCountThreshold =
 126       ProfileSummaryBuilder::getHotCountThreshold(DetailedSummary);
 127   ColdCountThreshold =
 128       ProfileSummaryBuilder::getColdCountThreshold(DetailedSummary);
 129   assert(ColdCountThreshold <= HotCountThreshold &&
 130          "Cold count threshold cannot exceed hot count threshold!");
 131   if (!hasPartialSampleProfile() || !ScalePartialSampleProfileWorkingSetSize) {
 132     HasHugeWorkingSetSize =
 133         HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
 134     HasLargeWorkingSetSize =
 135         HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
 136   } else {
 137     // Scale the working set size of the partial sample profile to reflect the
 138     // size of the program being compiled.
 139     double PartialProfileRatio = Summary->getPartialProfileRatio();
 140     uint64_t ScaledHotEntryNumCounts =
 141         static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio *
 142                               PartialSampleProfileWorkingSetSizeScaleFactor);
 143     HasHugeWorkingSetSize =
 144         ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
 145     HasLargeWorkingSetSize =
 146         ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
 147   }
 148 }
 149
 150 std::optional<uint64_t>
 151 ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {
 152   if (!hasProfileSummary())
 153     return std::nullopt;
 154   auto iter = ThresholdCache.find(PercentileCutoff);
 155   if (iter != ThresholdCache.end()) {
 156     return iter->second;
 157   }
 158   auto &DetailedSummary = Summary->getDetailedSummary();
 159   auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,
 160                                                              PercentileCutoff);
 161   uint64_t CountThreshold = Entry.MinCount;
 162   ThresholdCache[PercentileCutoff] = CountThreshold;
 163   return CountThreshold;
 164 }
 165
 166 bool ProfileSummaryInfo::hasHugeWorkingSetSize() const {
 167   return HasHugeWorkingSetSize && *HasHugeWorkingSetSize;
 168 }
 169
 170 bool ProfileSummaryInfo::hasLargeWorkingSetSize() const {
 171   return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;
 172 }
 173
 174 bool ProfileSummaryInfo::isHotCount(uint64_t C) const {
 175   return HotCountThreshold && C >= *HotCountThreshold;
 176 }
 177
 178 bool ProfileSummaryInfo::isColdCount(uint64_t C) const {
 179   return ColdCountThreshold && C <= *ColdCountThreshold;
 180 }
 181
 182 template <bool isHot>
 183 bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff,
 184                                                        uint64_t C) const {
 185   auto CountThreshold = computeThreshold(PercentileCutoff);
 186   if (isHot)
 187     return CountThreshold && C >= *CountThreshold;
 188   else
 189     return CountThreshold && C <= *CountThreshold;
 190 }
 191
 192 bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff,
 193                                                  uint64_t C) const {
 194   return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C);
 195 }
 196
 197 bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff,
 198                                                   uint64_t C) const {
 199   return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C);
 200 }
 201
 202 uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const {
 203   return HotCountThreshold.value_or(UINT64_MAX);
 204 }
 205
 206 uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const {
 207   return ColdCountThreshold.value_or(0);
 208 }
 209
 210 bool ProfileSummaryInfo::isHotCallSite(const CallBase &CB,
 211                                        BlockFrequencyInfo *BFI) const {
 212   auto C = getProfileCount(CB, BFI);
 213   return C && isHotCount(*C);
 214 }
 215
 216 bool ProfileSummaryInfo::isColdCallSite(const CallBase &CB,
 217                                         BlockFrequencyInfo *BFI) const {
 218   auto C = getProfileCount(CB, BFI);
 219   if (C)
 220     return isColdCount(*C);
 221
 222   // In SamplePGO, if the caller has been sampled, and there is no profile
 223   // annotated on the callsite, we consider the callsite as cold.
 224   return hasSampleProfile() && CB.getCaller()->hasProfileData();
 225 }
 226
 227 bool ProfileSummaryInfo::hasPartialSampleProfile() const {
 228   return hasProfileSummary() &&
 229          Summary->getKind() == ProfileSummary::PSK_Sample &&
 230          (PartialProfile || Summary->isPartialProfile());
 231 }
 232
 233 INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",
 234                 "Profile summary info", false, true)
 235
 236 ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()
 237     : ImmutablePass(ID) {
 238   initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
 239 }
 240
 241 bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {
 242   PSI.reset(new ProfileSummaryInfo(M));
 243   return false;
 244 }
 245
 246 bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) {
 247   PSI.reset();
 248   return false;
 249 }
 250
 251 AnalysisKey ProfileSummaryAnalysis::Key;
 252 ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M,
 253                                                ModuleAnalysisManager &) {
 254   return ProfileSummaryInfo(M);
 255 }
 256
 257 PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M,
 258                                                  ModuleAnalysisManager &AM) {
 259   ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
 260
 261   OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
 262   for (auto &F : M) {
 263     OS << F.getName();
 264     if (PSI.isFunctionEntryHot(&F))
 265       OS << " :hot entry ";
 266     else if (PSI.isFunctionEntryCold(&F))
 267       OS << " :cold entry ";
 268     OS << "\n";
 269   }
 270   return PreservedAnalyses::all();
 271 }
 272
 273 char ProfileSummaryInfoWrapperPass::ID = 0;