llvm/lib/Analysis/ProfileSummaryInfo.cpp

   1 //===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a pass that provides access to the global profile summary
  10 // information.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Analysis/ProfileSummaryInfo.h"
  15 #include "llvm/Analysis/BlockFrequencyInfo.h"
  16 #include "llvm/IR/BasicBlock.h"
  17 #include "llvm/IR/Instructions.h"
  18 #include "llvm/IR/Module.h"
  19 #include "llvm/IR/ProfileSummary.h"
  20 #include "llvm/InitializePasses.h"
  21 #include "llvm/ProfileData/ProfileCommon.h"
  22 #include "llvm/Support/CommandLine.h"
  23 #include <optional>
  24 using namespace llvm;
  25
  26 static cl::opt<bool> PartialProfile(
  27     "partial-profile", cl::Hidden, cl::init(false),
  28     cl::desc("Specify the current profile is used as a partial profile."));
  29
  30 cl::opt<bool> ScalePartialSampleProfileWorkingSetSize(
  31     "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true),
  32     cl::desc(
  33         "If true, scale the working set size of the partial sample profile "
  34         "by the partial profile ratio to reflect the size of the program "
  35         "being compiled."));
  36
  37 static cl::opt<double> PartialSampleProfileWorkingSetSizeScaleFactor(
  38     "partial-sample-profile-working-set-size-scale-factor", cl::Hidden,
  39     cl::init(0.008),
  40     cl::desc("The scale factor used to scale the working set size of the "
  41              "partial sample profile along with the partial profile ratio. "
  42              "This includes the factor of the profile counter per block "
  43              "and the factor to scale the working set size to use the same "
  44              "shared thresholds as PGO."));
  45
  46 // The profile summary metadata may be attached either by the frontend or by
  47 // any backend passes (IR level instrumentation, for example). This method
  48 // checks if the Summary is null and if so checks if the summary metadata is now
  49 // available in the module and parses it to get the Summary object.
  50 void ProfileSummaryInfo::refresh() {
  51   if (hasProfileSummary())
  52     return;
  53   // First try to get context sensitive ProfileSummary.
  54   auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);
  55   if (SummaryMD)
  56     Summary.reset(ProfileSummary::getFromMD(SummaryMD));
  57
  58   if (!hasProfileSummary()) {
  59     // This will actually return PSK_Instr or PSK_Sample summary.
  60     SummaryMD = M->getProfileSummary(/* IsCS */ false);
  61     if (SummaryMD)
  62       Summary.reset(ProfileSummary::getFromMD(SummaryMD));
  63   }
  64   if (!hasProfileSummary())
  65     return;
  66   computeThresholds();
  67 }
  68
  69 std::optional<uint64_t> ProfileSummaryInfo::getProfileCount(
  70     const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const {
  71   assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) &&
  72          "We can only get profile count for call/invoke instruction.");
  73   if (hasSampleProfile()) {
  74     // In sample PGO mode, check if there is a profile metadata on the
  75     // instruction. If it is present, determine hotness solely based on that,
  76     // since the sampled entry count may not be accurate. If there is no
  77     // annotated on the instruction, return std::nullopt.
  78     uint64_t TotalCount;
  79     if (Call.extractProfTotalWeight(TotalCount))
  80       return TotalCount;
  81     return std::nullopt;
  82   }
  83   if (BFI)
  84     return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic);
  85   return std::nullopt;
  86 }
  87
  88 bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const {
  89   assert(hasPartialSampleProfile() && "Expect partial sample profile");
  90   return !F.getEntryCount();
  91 }
  92
  93 /// Returns true if the function's entry is a cold. If it returns false, it
  94 /// either means it is not cold or it is unknown whether it is cold or not (for
  95 /// example, no profile data is available).
  96 bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const {
  97   if (!F)
  98     return false;
  99   if (F->hasFnAttribute(Attribute::Cold))
 100     return true;
 101   if (!hasProfileSummary())
 102     return false;
 103   auto FunctionCount = F->getEntryCount();
 104   // FIXME: The heuristic used below for determining coldness is based on
 105   // preliminary SPEC tuning for inliner. This will eventually be a
 106   // convenience method that calls isHotCount.
 107   return FunctionCount && isColdCount(FunctionCount->getCount());
 108 }
 109
 110 /// Compute the hot and cold thresholds.
 111 void ProfileSummaryInfo::computeThresholds() {
 112   auto &DetailedSummary = Summary->getDetailedSummary();
 113   auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile(
 114       DetailedSummary, ProfileSummaryCutoffHot);
 115   HotCountThreshold =
 116       ProfileSummaryBuilder::getHotCountThreshold(DetailedSummary);
 117   ColdCountThreshold =
 118       ProfileSummaryBuilder::getColdCountThreshold(DetailedSummary);
 119   assert(ColdCountThreshold <= HotCountThreshold &&
 120          "Cold count threshold cannot exceed hot count threshold!");
 121   if (!hasPartialSampleProfile() || !ScalePartialSampleProfileWorkingSetSize) {
 122     HasHugeWorkingSetSize =
 123         HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
 124     HasLargeWorkingSetSize =
 125         HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
 126   } else {
 127     // Scale the working set size of the partial sample profile to reflect the
 128     // size of the program being compiled.
 129     double PartialProfileRatio = Summary->getPartialProfileRatio();
 130     uint64_t ScaledHotEntryNumCounts =
 131         static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio *
 132                               PartialSampleProfileWorkingSetSizeScaleFactor);
 133     HasHugeWorkingSetSize =
 134         ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
 135     HasLargeWorkingSetSize =
 136         ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
 137   }
 138 }
 139
 140 std::optional<uint64_t>
 141 ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {
 142   if (!hasProfileSummary())
 143     return std::nullopt;
 144   auto iter = ThresholdCache.find(PercentileCutoff);
 145   if (iter != ThresholdCache.end()) {
 146     return iter->second;
 147   }
 148   auto &DetailedSummary = Summary->getDetailedSummary();
 149   auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,
 150                                                              PercentileCutoff);
 151   uint64_t CountThreshold = Entry.MinCount;
 152   ThresholdCache[PercentileCutoff] = CountThreshold;
 153   return CountThreshold;
 154 }
 155
 156 bool ProfileSummaryInfo::hasHugeWorkingSetSize() const {
 157   return HasHugeWorkingSetSize && *HasHugeWorkingSetSize;
 158 }
 159
 160 bool ProfileSummaryInfo::hasLargeWorkingSetSize() const {
 161   return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;
 162 }
 163
 164 bool ProfileSummaryInfo::isHotCount(uint64_t C) const {
 165   return HotCountThreshold && C >= *HotCountThreshold;
 166 }
 167
 168 bool ProfileSummaryInfo::isColdCount(uint64_t C) const {
 169   return ColdCountThreshold && C <= *ColdCountThreshold;
 170 }
 171
 172 template <bool isHot>
 173 bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff,
 174                                                        uint64_t C) const {
 175   auto CountThreshold = computeThreshold(PercentileCutoff);
 176   if (isHot)
 177     return CountThreshold && C >= *CountThreshold;
 178   else
 179     return CountThreshold && C <= *CountThreshold;
 180 }
 181
 182 bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff,
 183                                                  uint64_t C) const {
 184   return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C);
 185 }
 186
 187 bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff,
 188                                                   uint64_t C) const {
 189   return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C);
 190 }
 191
 192 uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const {
 193   return HotCountThreshold.value_or(UINT64_MAX);
 194 }
 195
 196 uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const {
 197   return ColdCountThreshold.value_or(0);
 198 }
 199
 200 bool ProfileSummaryInfo::isHotCallSite(const CallBase &CB,
 201                                        BlockFrequencyInfo *BFI) const {
 202   auto C = getProfileCount(CB, BFI);
 203   return C && isHotCount(*C);
 204 }
 205
 206 bool ProfileSummaryInfo::isColdCallSite(const CallBase &CB,
 207                                         BlockFrequencyInfo *BFI) const {
 208   auto C = getProfileCount(CB, BFI);
 209   if (C)
 210     return isColdCount(*C);
 211
 212   // In SamplePGO, if the caller has been sampled, and there is no profile
 213   // annotated on the callsite, we consider the callsite as cold.
 214   return hasSampleProfile() && CB.getCaller()->hasProfileData();
 215 }
 216
 217 bool ProfileSummaryInfo::hasPartialSampleProfile() const {
 218   return hasProfileSummary() &&
 219          Summary->getKind() == ProfileSummary::PSK_Sample &&
 220          (PartialProfile || Summary->isPartialProfile());
 221 }
 222
 223 INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",
 224                 "Profile summary info", false, true)
 225
 226 ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()
 227     : ImmutablePass(ID) {
 228   initializeProfileSummaryInfoWrapperPassPass(*PassRegistry::getPassRegistry());
 229 }
 230
 231 bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {
 232   PSI.reset(new ProfileSummaryInfo(M));
 233   return false;
 234 }
 235
 236 bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) {
 237   PSI.reset();
 238   return false;
 239 }
 240
 241 AnalysisKey ProfileSummaryAnalysis::Key;
 242 ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M,
 243                                                ModuleAnalysisManager &) {
 244   return ProfileSummaryInfo(M);
 245 }
 246
 247 PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M,
 248                                                  ModuleAnalysisManager &AM) {
 249   ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
 250
 251   OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";
 252   for (auto &F : M) {
 253     OS << F.getName();
 254     if (PSI.isFunctionEntryHot(&F))
 255       OS << " :hot entry ";
 256     else if (PSI.isFunctionEntryCold(&F))
 257       OS << " :cold entry ";
 258     OS << "\n";
 259   }
 260   return PreservedAnalyses::all();
 261 }
 262
 263 char ProfileSummaryInfoWrapperPass::ID = 0;