llvm/lib/Analysis/InlineAdvisor.cpp

   1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
  10 // related types.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Analysis/InlineAdvisor.h"
  15 #include "llvm/ADT/Statistic.h"
  16 #include "llvm/Analysis/AssumptionCache.h"
  17 #include "llvm/Analysis/InlineCost.h"
  18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  19 #include "llvm/Analysis/ProfileSummaryInfo.h"
  20 #include "llvm/Analysis/ReplayInlineAdvisor.h"
  21 #include "llvm/Analysis/TargetLibraryInfo.h"
  22 #include "llvm/Analysis/TargetTransformInfo.h"
  23 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
  24 #include "llvm/IR/DebugInfoMetadata.h"
  25 #include "llvm/IR/PassManager.h"
  26 #include "llvm/Support/CommandLine.h"
  27 #include "llvm/Support/raw_ostream.h"
  28
  29 using namespace llvm;
  30 #define DEBUG_TYPE "inline"
  31 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
  32 #define LLVM_HAVE_TF_AOT
  33 #endif
  34
  35 // This weirdly named statistic tracks the number of times that, when attempting
  36 // to inline a function A into B, we analyze the callers of B in order to see
  37 // if those would be more profitable and blocked inline steps.
  38 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
  39
  40 /// Flag to add inline messages as callsite attributes 'inline-remark'.
  41 static cl::opt<bool>
  42     InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
  43                           cl::Hidden,
  44                           cl::desc("Enable adding inline-remark attribute to"
  45                                    " callsites processed by inliner but decided"
  46                                    " to be not inlined"));
  47
  48 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
  49                                           cl::Hidden,
  50                                           cl::desc("Enable deferred inlining"));
  51
  52 // An integer used to limit the cost of inline deferral.  The default negative
  53 // number tells shouldBeDeferred to only take the secondary cost into account.
  54 static cl::opt<int>
  55     InlineDeferralScale("inline-deferral-scale",
  56                         cl::desc("Scale to limit the cost of inline deferral"),
  57                         cl::init(2), cl::Hidden);
  58
  59 static cl::opt<bool>
  60     AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
  61                         cl::desc("If true, annotate inline advisor remarks "
  62                                  "with LTO and pass information."));
  63
  64 namespace llvm {
  65 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
  66 }
  67
  68 namespace {
  69 using namespace llvm::ore;
  70 class MandatoryInlineAdvice : public InlineAdvice {
  71 public:
  72   MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
  73                         OptimizationRemarkEmitter &ORE,
  74                         bool IsInliningMandatory)
  75       : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
  76
  77 private:
  78   void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
  79
  80   void recordInliningImpl() override {
  81     if (IsInliningRecommended)
  82       emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
  83                       [&](OptimizationRemark &Remark) {
  84                         Remark << ": always inline attribute";
  85                       });
  86   }
  87
  88   void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
  89     if (IsInliningRecommended)
  90       ORE.emit([&]() {
  91         return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
  92                                         "NotInlined", DLoc, Block)
  93                << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
  94                << NV("Caller", Caller)
  95                << "': " << NV("Reason", Result.getFailureReason());
  96       });
  97   }
  98
  99   void recordUnattemptedInliningImpl() override {
 100     assert(!IsInliningRecommended && "Expected to attempt inlining");
 101   }
 102 };
 103 } // namespace
 104
 105 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
 106     const InlineResult &Result) {
 107   using namespace ore;
 108   llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
 109                                          "; " + inlineCostStr(*OIC));
 110   ORE.emit([&]() {
 111     return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
 112                                     "NotInlined", DLoc, Block)
 113            << "'" << NV("Callee", Callee) << "' is not inlined into '"
 114            << NV("Caller", Caller)
 115            << "': " << NV("Reason", Result.getFailureReason());
 116   });
 117 }
 118
 119 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
 120   if (EmitRemarks)
 121     emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
 122                                /* ForProfileContext= */ false,
 123                                Advisor->getAnnotatedInlinePassName());
 124 }
 125
 126 void DefaultInlineAdvice::recordInliningImpl() {
 127   if (EmitRemarks)
 128     emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
 129                                /* ForProfileContext= */ false,
 130                                Advisor->getAnnotatedInlinePassName());
 131 }
 132
 133 std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
 134     CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
 135   Function &Caller = *CB.getCaller();
 136   ProfileSummaryInfo *PSI =
 137       FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
 138           .getCachedResult<ProfileSummaryAnalysis>(
 139               *CB.getParent()->getParent()->getParent());
 140
 141   auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
 142   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
 143     return FAM.getResult<AssumptionAnalysis>(F);
 144   };
 145   auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
 146     return FAM.getResult<BlockFrequencyAnalysis>(F);
 147   };
 148   auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
 149     return FAM.getResult<TargetLibraryAnalysis>(F);
 150   };
 151
 152   auto GetInlineCost = [&](CallBase &CB) {
 153     Function &Callee = *CB.getCalledFunction();
 154     auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
 155     bool RemarksEnabled =
 156         Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
 157             DEBUG_TYPE);
 158     return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
 159                          GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
 160   };
 161   return llvm::shouldInline(
 162       CB, GetInlineCost, ORE,
 163       Params.EnableDeferral.value_or(EnableInlineDeferral));
 164 }
 165
 166 std::unique_ptr<InlineAdvice>
 167 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
 168   auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
 169   return std::make_unique<DefaultInlineAdvice>(
 170       this, CB, OIC,
 171       FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()));
 172 }
 173
 174 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
 175                            OptimizationRemarkEmitter &ORE,
 176                            bool IsInliningRecommended)
 177     : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
 178       DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
 179       IsInliningRecommended(IsInliningRecommended) {}
 180
 181 void InlineAdvice::recordInlineStatsIfNeeded() {
 182   if (Advisor->ImportedFunctionsStats)
 183     Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
 184 }
 185
 186 void InlineAdvice::recordInlining() {
 187   markRecorded();
 188   recordInlineStatsIfNeeded();
 189   recordInliningImpl();
 190 }
 191
 192 void InlineAdvice::recordInliningWithCalleeDeleted() {
 193   markRecorded();
 194   recordInlineStatsIfNeeded();
 195   recordInliningWithCalleeDeletedImpl();
 196 }
 197
 198 AnalysisKey InlineAdvisorAnalysis::Key;
 199 AnalysisKey PluginInlineAdvisorAnalysis::Key;
 200 bool PluginInlineAdvisorAnalysis::HasBeenRegistered = false;
 201
 202 bool InlineAdvisorAnalysis::Result::tryCreate(
 203     InlineParams Params, InliningAdvisorMode Mode,
 204     const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
 205   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 206   if (PluginInlineAdvisorAnalysis::HasBeenRegistered) {
 207     auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);
 208     Advisor.reset(DA.Factory(M, FAM, Params, IC));
 209     return !!Advisor;
 210   }
 211   auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
 212     auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
 213     return OIC.has_value();
 214   };
 215   switch (Mode) {
 216   case InliningAdvisorMode::Default:
 217     LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
 218     Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
 219     // Restrict replay to default advisor, ML advisors are stateful so
 220     // replay will need augmentations to interleave with them correctly.
 221     if (!ReplaySettings.ReplayFile.empty()) {
 222       Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(),
 223                                              std::move(Advisor), ReplaySettings,
 224                                              /* EmitRemarks =*/true, IC);
 225     }
 226     break;
 227   case InliningAdvisorMode::Development:
 228 #ifdef LLVM_HAVE_TFLITE
 229     LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
 230     Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
 231 #endif
 232     break;
 233   case InliningAdvisorMode::Release:
 234     LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
 235     Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
 236     break;
 237   }
 238
 239   return !!Advisor;
 240 }
 241
 242 /// Return true if inlining of CB can block the caller from being
 243 /// inlined which is proved to be more beneficial. \p IC is the
 244 /// estimated inline cost associated with callsite \p CB.
 245 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the
 246 /// caller if \p CB is suppressed for inlining.
 247 static bool
 248 shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
 249                  function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
 250   // For now we only handle local or inline functions.
 251   if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
 252     return false;
 253   // If the cost of inlining CB is non-positive, it is not going to prevent the
 254   // caller from being inlined into its callers and hence we don't need to
 255   // defer.
 256   if (IC.getCost() <= 0)
 257     return false;
 258   // Try to detect the case where the current inlining candidate caller (call
 259   // it B) is a static or linkonce-ODR function and is an inlining candidate
 260   // elsewhere, and the current candidate callee (call it C) is large enough
 261   // that inlining it into B would make B too big to inline later. In these
 262   // circumstances it may be best not to inline C into B, but to inline B into
 263   // its callers.
 264   //
 265   // This only applies to static and linkonce-ODR functions because those are
 266   // expected to be available for inlining in the translation units where they
 267   // are used. Thus we will always have the opportunity to make local inlining
 268   // decisions. Importantly the linkonce-ODR linkage covers inline functions
 269   // and templates in C++.
 270   //
 271   // FIXME: All of this logic should be sunk into getInlineCost. It relies on
 272   // the internal implementation of the inline cost metrics rather than
 273   // treating them as truly abstract units etc.
 274   TotalSecondaryCost = 0;
 275   // The candidate cost to be imposed upon the current function.
 276   int CandidateCost = IC.getCost() - 1;
 277   // If the caller has local linkage and can be inlined to all its callers, we
 278   // can apply a huge negative bonus to TotalSecondaryCost.
 279   bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
 280   // This bool tracks what happens if we DO inline C into B.
 281   bool InliningPreventsSomeOuterInline = false;
 282   unsigned NumCallerUsers = 0;
 283   for (User *U : Caller->users()) {
 284     CallBase *CS2 = dyn_cast<CallBase>(U);
 285
 286     // If this isn't a call to Caller (it could be some other sort
 287     // of reference) skip it.  Such references will prevent the caller
 288     // from being removed.
 289     if (!CS2 || CS2->getCalledFunction() != Caller) {
 290       ApplyLastCallBonus = false;
 291       continue;
 292     }
 293
 294     InlineCost IC2 = GetInlineCost(*CS2);
 295     ++NumCallerCallersAnalyzed;
 296     if (!IC2) {
 297       ApplyLastCallBonus = false;
 298       continue;
 299     }
 300     if (IC2.isAlways())
 301       continue;
 302
 303     // See if inlining of the original callsite would erase the cost delta of
 304     // this callsite. We subtract off the penalty for the call instruction,
 305     // which we would be deleting.
 306     if (IC2.getCostDelta() <= CandidateCost) {
 307       InliningPreventsSomeOuterInline = true;
 308       TotalSecondaryCost += IC2.getCost();
 309       NumCallerUsers++;
 310     }
 311   }
 312
 313   if (!InliningPreventsSomeOuterInline)
 314     return false;
 315
 316   // If all outer calls to Caller would get inlined, the cost for the last
 317   // one is set very low by getInlineCost, in anticipation that Caller will
 318   // be removed entirely.  We did not account for this above unless there
 319   // is only one caller of Caller.
 320   if (ApplyLastCallBonus)
 321     TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
 322
 323   // If InlineDeferralScale is negative, then ignore the cost of primary
 324   // inlining -- IC.getCost() multiplied by the number of callers to Caller.
 325   if (InlineDeferralScale < 0)
 326     return TotalSecondaryCost < IC.getCost();
 327
 328   int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
 329   int Allowance = IC.getCost() * InlineDeferralScale;
 330   return TotalCost < Allowance;
 331 }
 332
 333 namespace llvm {
 334 static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
 335   return R << Arg.Val;
 336 }
 337
 338 template <class RemarkT>
 339 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
 340   using namespace ore;
 341   if (IC.isAlways()) {
 342     R << "(cost=always)";
 343   } else if (IC.isNever()) {
 344     R << "(cost=never)";
 345   } else {
 346     R << "(cost=" << ore::NV("Cost", IC.getCost())
 347       << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
 348   }
 349   if (const char *Reason = IC.getReason())
 350     R << ": " << ore::NV("Reason", Reason);
 351   return R;
 352 }
 353 } // namespace llvm
 354
 355 std::string llvm::inlineCostStr(const InlineCost &IC) {
 356   std::string Buffer;
 357   raw_string_ostream Remark(Buffer);
 358   Remark << IC;
 359   return Remark.str();
 360 }
 361
 362 void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
 363   if (!InlineRemarkAttribute)
 364     return;
 365
 366   Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
 367   CB.addFnAttr(Attr);
 368 }
 369
 370 /// Return the cost only if the inliner should attempt to inline at the given
 371 /// CallSite. If we return the cost, we will emit an optimisation remark later
 372 /// using that cost, so we won't do so from this function. Return std::nullopt
 373 /// if inlining should not be attempted.
 374 std::optional<InlineCost>
 375 llvm::shouldInline(CallBase &CB,
 376                    function_ref<InlineCost(CallBase &CB)> GetInlineCost,
 377                    OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
 378   using namespace ore;
 379
 380   InlineCost IC = GetInlineCost(CB);
 381   Instruction *Call = &CB;
 382   Function *Callee = CB.getCalledFunction();
 383   Function *Caller = CB.getCaller();
 384
 385   if (IC.isAlways()) {
 386     LLVM_DEBUG(dbgs() << "    Inlining " << inlineCostStr(IC)
 387                       << ", Call: " << CB << "\n");
 388     return IC;
 389   }
 390
 391   if (!IC) {
 392     LLVM_DEBUG(dbgs() << "    NOT Inlining " << inlineCostStr(IC)
 393                       << ", Call: " << CB << "\n");
 394     if (IC.isNever()) {
 395       ORE.emit([&]() {
 396         return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
 397                << "'" << NV("Callee", Callee) << "' not inlined into '"
 398                << NV("Caller", Caller)
 399                << "' because it should never be inlined " << IC;
 400       });
 401     } else {
 402       ORE.emit([&]() {
 403         return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
 404                << "'" << NV("Callee", Callee) << "' not inlined into '"
 405                << NV("Caller", Caller) << "' because too costly to inline "
 406                << IC;
 407       });
 408     }
 409     setInlineRemark(CB, inlineCostStr(IC));
 410     return std::nullopt;
 411   }
 412
 413   int TotalSecondaryCost = 0;
 414   if (EnableDeferral &&
 415       shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
 416     LLVM_DEBUG(dbgs() << "    NOT Inlining: " << CB
 417                       << " Cost = " << IC.getCost()
 418                       << ", outer Cost = " << TotalSecondaryCost << '\n');
 419     ORE.emit([&]() {
 420       return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
 421                                       Call)
 422              << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
 423              << "' increases the cost of inlining '" << NV("Caller", Caller)
 424              << "' in other contexts";
 425     });
 426     setInlineRemark(CB, "deferred");
 427     return std::nullopt;
 428   }
 429
 430   LLVM_DEBUG(dbgs() << "    Inlining " << inlineCostStr(IC) << ", Call: " << CB
 431                     << '\n');
 432   return IC;
 433 }
 434
 435 std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
 436                                          const CallSiteFormat &Format) {
 437   std::string Buffer;
 438   raw_string_ostream CallSiteLoc(Buffer);
 439   bool First = true;
 440   for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
 441     if (!First)
 442       CallSiteLoc << " @ ";
 443     // Note that negative line offset is actually possible, but we use
 444     // unsigned int to match line offset representation in remarks so
 445     // it's directly consumable by relay advisor.
 446     uint32_t Offset =
 447         DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
 448     uint32_t Discriminator = DIL->getBaseDiscriminator();
 449     StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
 450     if (Name.empty())
 451       Name = DIL->getScope()->getSubprogram()->getName();
 452     CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
 453     if (Format.outputColumn())
 454       CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
 455     if (Format.outputDiscriminator() && Discriminator)
 456       CallSiteLoc << "." << llvm::utostr(Discriminator);
 457     First = false;
 458   }
 459
 460   return CallSiteLoc.str();
 461 }
 462
 463 void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
 464   if (!DLoc) {
 465     return;
 466   }
 467
 468   bool First = true;
 469   Remark << " at callsite ";
 470   for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
 471     if (!First)
 472       Remark << " @ ";
 473     unsigned int Offset = DIL->getLine();
 474     Offset -= DIL->getScope()->getSubprogram()->getLine();
 475     unsigned int Discriminator = DIL->getBaseDiscriminator();
 476     StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
 477     if (Name.empty())
 478       Name = DIL->getScope()->getSubprogram()->getName();
 479     Remark << Name << ":" << ore::NV("Line", Offset) << ":"
 480            << ore::NV("Column", DIL->getColumn());
 481     if (Discriminator)
 482       Remark << "." << ore::NV("Disc", Discriminator);
 483     First = false;
 484   }
 485
 486   Remark << ";";
 487 }
 488
 489 void llvm::emitInlinedInto(
 490     OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
 491     const Function &Callee, const Function &Caller, bool AlwaysInline,
 492     function_ref<void(OptimizationRemark &)> ExtraContext,
 493     const char *PassName) {
 494   ORE.emit([&]() {
 495     StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
 496     OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
 497                               DLoc, Block);
 498     Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
 499            << ore::NV("Caller", &Caller) << "'";
 500     if (ExtraContext)
 501       ExtraContext(Remark);
 502     addLocationToRemarks(Remark, DLoc);
 503     return Remark;
 504   });
 505 }
 506
 507 void llvm::emitInlinedIntoBasedOnCost(
 508     OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
 509     const Function &Callee, const Function &Caller, const InlineCost &IC,
 510     bool ForProfileContext, const char *PassName) {
 511   llvm::emitInlinedInto(
 512       ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
 513       [&](OptimizationRemark &Remark) {
 514         if (ForProfileContext)
 515           Remark << " to match profiling context";
 516         Remark << " with " << IC;
 517       },
 518       PassName);
 519 }
 520
 521 InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
 522                              std::optional<InlineContext> IC)
 523     : M(M), FAM(FAM), IC(IC),
 524       AnnotatedInlinePassName((IC && AnnotateInlinePhase)
 525                                   ? llvm::AnnotateInlinePassName(*IC)
 526                                   : DEBUG_TYPE) {
 527   if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
 528     ImportedFunctionsStats =
 529         std::make_unique<ImportedFunctionsInliningStatistics>();
 530     ImportedFunctionsStats->setModuleInfo(M);
 531   }
 532 }
 533
 534 InlineAdvisor::~InlineAdvisor() {
 535   if (ImportedFunctionsStats) {
 536     assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
 537     ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
 538                                  InlinerFunctionImportStatsOpts::Verbose);
 539   }
 540 }
 541
 542 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
 543                                                                 bool Advice) {
 544   return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
 545                                                  Advice);
 546 }
 547
 548 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
 549   switch (LTOPhase) {
 550   case (ThinOrFullLTOPhase::None):
 551     return "main";
 552   case (ThinOrFullLTOPhase::ThinLTOPreLink):
 553   case (ThinOrFullLTOPhase::FullLTOPreLink):
 554     return "prelink";
 555   case (ThinOrFullLTOPhase::ThinLTOPostLink):
 556   case (ThinOrFullLTOPhase::FullLTOPostLink):
 557     return "postlink";
 558   }
 559   llvm_unreachable("unreachable");
 560 }
 561
 562 static inline const char *getInlineAdvisorContext(InlinePass IP) {
 563   switch (IP) {
 564   case (InlinePass::AlwaysInliner):
 565     return "always-inline";
 566   case (InlinePass::CGSCCInliner):
 567     return "cgscc-inline";
 568   case (InlinePass::EarlyInliner):
 569     return "early-inline";
 570   case (InlinePass::MLInliner):
 571     return "ml-inline";
 572   case (InlinePass::ModuleInliner):
 573     return "module-inline";
 574   case (InlinePass::ReplayCGSCCInliner):
 575     return "replay-cgscc-inline";
 576   case (InlinePass::ReplaySampleProfileInliner):
 577     return "replay-sample-profile-inline";
 578   case (InlinePass::SampleProfileInliner):
 579     return "sample-profile-inline";
 580   }
 581
 582   llvm_unreachable("unreachable");
 583 }
 584
 585 std::string llvm::AnnotateInlinePassName(InlineContext IC) {
 586   return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
 587          std::string(getInlineAdvisorContext(IC.Pass));
 588 }
 589
 590 InlineAdvisor::MandatoryInliningKind
 591 InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
 592                                 OptimizationRemarkEmitter &ORE) {
 593   auto &Callee = *CB.getCalledFunction();
 594
 595   auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
 596     return FAM.getResult<TargetLibraryAnalysis>(F);
 597   };
 598
 599   auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
 600
 601   auto TrivialDecision =
 602       llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
 603
 604   if (TrivialDecision) {
 605     if (TrivialDecision->isSuccess())
 606       return MandatoryInliningKind::Always;
 607     else
 608       return MandatoryInliningKind::Never;
 609   }
 610   return MandatoryInliningKind::NotMandatory;
 611 }
 612
 613 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
 614                                                        bool MandatoryOnly) {
 615   if (!MandatoryOnly)
 616     return getAdviceImpl(CB);
 617   bool Advice = CB.getCaller() != CB.getCalledFunction() &&
 618                 MandatoryInliningKind::Always ==
 619                     getMandatoryKind(CB, FAM, getCallerORE(CB));
 620   return getMandatoryAdvice(CB, Advice);
 621 }
 622
 623 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
 624   return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller());
 625 }
 626
 627 PreservedAnalyses
 628 InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
 629   const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
 630   if (!IA)
 631     OS << "No Inline Advisor\n";
 632   else
 633     IA->getAdvisor()->print(OS);
 634   return PreservedAnalyses::all();
 635 }
 636
 637 PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
 638     LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
 639     CGSCCUpdateResult &UR) {
 640   const auto &MAMProxy =
 641       AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);
 642
 643   if (InitialC.size() == 0) {
 644     OS << "SCC is empty!\n";
 645     return PreservedAnalyses::all();
 646   }
 647   Module &M = *InitialC.begin()->getFunction().getParent();
 648   const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
 649   if (!IA)
 650     OS << "No Inline Advisor\n";
 651   else
 652     IA->getAdvisor()->print(OS);
 653   return PreservedAnalyses::all();
 654 }