llvm/lib/Analysis/TargetTransformInfo.cpp

   1 //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Analysis/TargetTransformInfo.h"
  10 #include "llvm/Analysis/CFG.h"
  11 #include "llvm/Analysis/LoopIterator.h"
  12 #include "llvm/Analysis/TargetLibraryInfo.h"
  13 #include "llvm/Analysis/TargetTransformInfoImpl.h"
  14 #include "llvm/IR/CFG.h"
  15 #include "llvm/IR/Dominators.h"
  16 #include "llvm/IR/Instruction.h"
  17 #include "llvm/IR/Instructions.h"
  18 #include "llvm/IR/IntrinsicInst.h"
  19 #include "llvm/IR/Module.h"
  20 #include "llvm/IR/Operator.h"
  21 #include "llvm/IR/PatternMatch.h"
  22 #include "llvm/InitializePasses.h"
  23 #include "llvm/Support/CommandLine.h"
  24 #include <optional>
  25 #include <utility>
  26
  27 using namespace llvm;
  28 using namespace PatternMatch;
  29
  30 #define DEBUG_TYPE "tti"
  31
  32 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
  33                                      cl::Hidden,
  34                                      cl::desc("Recognize reduction patterns."));
  35
  36 static cl::opt<unsigned> CacheLineSize(
  37     "cache-line-size", cl::init(0), cl::Hidden,
  38     cl::desc("Use this to override the target cache line size when "
  39              "specified by the user."));
  40
  41 static cl::opt<unsigned> MinPageSize(
  42     "min-page-size", cl::init(0), cl::Hidden,
  43     cl::desc("Use this to override the target's minimum page size."));
  44
  45 static cl::opt<unsigned> PredictableBranchThreshold(
  46     "predictable-branch-threshold", cl::init(99), cl::Hidden,
  47     cl::desc(
  48         "Use this to override the target's predictable branch threshold (%)."));
  49
  50 namespace {
  51 /// No-op implementation of the TTI interface using the utility base
  52 /// classes.
  53 ///
  54 /// This is used when no target specific information is available.
  55 struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
  56   explicit NoTTIImpl(const DataLayout &DL)
  57       : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
  58 };
  59 } // namespace
  60
  61 bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) {
  62   // If the loop has irreducible control flow, it can not be converted to
  63   // Hardware loop.
  64   LoopBlocksRPO RPOT(L);
  65   RPOT.perform(&LI);
  66   if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI))
  67     return false;
  68   return true;
  69 }
  70
  71 IntrinsicCostAttributes::IntrinsicCostAttributes(
  72     Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost,
  73     bool TypeBasedOnly)
  74     : II(dyn_cast<IntrinsicInst>(&CI)), RetTy(CI.getType()), IID(Id),
  75       ScalarizationCost(ScalarizationCost) {
  76
  77   if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI))
  78     FMF = FPMO->getFastMathFlags();
  79
  80   if (!TypeBasedOnly)
  81     Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end());
  82   FunctionType *FTy = CI.getCalledFunction()->getFunctionType();
  83   ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end());
  84 }
  85
  86 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
  87                                                  ArrayRef<Type *> Tys,
  88                                                  FastMathFlags Flags,
  89                                                  const IntrinsicInst *I,
  90                                                  InstructionCost ScalarCost)
  91     : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
  92   ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
  93 }
  94
  95 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
  96                                                  ArrayRef<const Value *> Args)
  97     : RetTy(Ty), IID(Id) {
  98
  99   Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
 100   ParamTys.reserve(Arguments.size());
 101   for (const Value *Argument : Arguments)
 102     ParamTys.push_back(Argument->getType());
 103 }
 104
 105 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
 106                                                  ArrayRef<const Value *> Args,
 107                                                  ArrayRef<Type *> Tys,
 108                                                  FastMathFlags Flags,
 109                                                  const IntrinsicInst *I,
 110                                                  InstructionCost ScalarCost)
 111     : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
 112   ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end());
 113   Arguments.insert(Arguments.begin(), Args.begin(), Args.end());
 114 }
 115
 116 HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) {
 117   // Match default options:
 118   // - hardware-loop-counter-bitwidth = 32
 119   // - hardware-loop-decrement = 1
 120   CountType = Type::getInt32Ty(L->getHeader()->getContext());
 121   LoopDecrement = ConstantInt::get(CountType, 1);
 122 }
 123
 124 bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
 125                                                LoopInfo &LI, DominatorTree &DT,
 126                                                bool ForceNestedLoop,
 127                                                bool ForceHardwareLoopPHI) {
 128   SmallVector<BasicBlock *, 4> ExitingBlocks;
 129   L->getExitingBlocks(ExitingBlocks);
 130
 131   for (BasicBlock *BB : ExitingBlocks) {
 132     // If we pass the updated counter back through a phi, we need to know
 133     // which latch the updated value will be coming from.
 134     if (!L->isLoopLatch(BB)) {
 135       if (ForceHardwareLoopPHI || CounterInReg)
 136         continue;
 137     }
 138
 139     const SCEV *EC = SE.getExitCount(L, BB);
 140     if (isa<SCEVCouldNotCompute>(EC))
 141       continue;
 142     if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(EC)) {
 143       if (ConstEC->getValue()->isZero())
 144         continue;
 145     } else if (!SE.isLoopInvariant(EC, L))
 146       continue;
 147
 148     if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth())
 149       continue;
 150
 151     // If this exiting block is contained in a nested loop, it is not eligible
 152     // for insertion of the branch-and-decrement since the inner loop would
 153     // end up messing up the value in the CTR.
 154     if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
 155       continue;
 156
 157     // We now have a loop-invariant count of loop iterations (which is not the
 158     // constant zero) for which we know that this loop will not exit via this
 159     // existing block.
 160
 161     // We need to make sure that this block will run on every loop iteration.
 162     // For this to be true, we must dominate all blocks with backedges. Such
 163     // blocks are in-loop predecessors to the header block.
 164     bool NotAlways = false;
 165     for (BasicBlock *Pred : predecessors(L->getHeader())) {
 166       if (!L->contains(Pred))
 167         continue;
 168
 169       if (!DT.dominates(BB, Pred)) {
 170         NotAlways = true;
 171         break;
 172       }
 173     }
 174
 175     if (NotAlways)
 176       continue;
 177
 178     // Make sure this blocks ends with a conditional branch.
 179     Instruction *TI = BB->getTerminator();
 180     if (!TI)
 181       continue;
 182
 183     if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
 184       if (!BI->isConditional())
 185         continue;
 186
 187       ExitBranch = BI;
 188     } else
 189       continue;
 190
 191     // Note that this block may not be the loop latch block, even if the loop
 192     // has a latch block.
 193     ExitBlock = BB;
 194     ExitCount = EC;
 195     break;
 196   }
 197
 198   if (!ExitBlock)
 199     return false;
 200   return true;
 201 }
 202
 203 TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
 204     : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
 205
 206 TargetTransformInfo::~TargetTransformInfo() = default;
 207
 208 TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
 209     : TTIImpl(std::move(Arg.TTIImpl)) {}
 210
 211 TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
 212   TTIImpl = std::move(RHS.TTIImpl);
 213   return *this;
 214 }
 215
 216 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
 217   return TTIImpl->getInliningThresholdMultiplier();
 218 }
 219
 220 unsigned
 221 TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const {
 222   return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier();
 223 }
 224
 225 unsigned
 226 TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()
 227     const {
 228   return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();
 229 }
 230
 231 unsigned
 232 TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
 233   return TTIImpl->adjustInliningThreshold(CB);
 234 }
 235
 236 unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB,
 237                                                   const AllocaInst *AI) const {
 238   return TTIImpl->getCallerAllocaCost(CB, AI);
 239 }
 240
 241 int TargetTransformInfo::getInlinerVectorBonusPercent() const {
 242   return TTIImpl->getInlinerVectorBonusPercent();
 243 }
 244
 245 InstructionCost TargetTransformInfo::getGEPCost(
 246     Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands,
 247     Type *AccessType, TTI::TargetCostKind CostKind) const {
 248   return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
 249 }
 250
 251 InstructionCost TargetTransformInfo::getPointersChainCost(
 252     ArrayRef<const Value *> Ptrs, const Value *Base,
 253     const TTI::PointersChainInfo &Info, Type *AccessTy,
 254     TTI::TargetCostKind CostKind) const {
 255   assert((Base || !Info.isSameBase()) &&
 256          "If pointers have same base address it has to be provided.");
 257   return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
 258 }
 259
 260 unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters(
 261     const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI,
 262     BlockFrequencyInfo *BFI) const {
 263   return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);
 264 }
 265
 266 InstructionCost
 267 TargetTransformInfo::getInstructionCost(const User *U,
 268                                         ArrayRef<const Value *> Operands,
 269                                         enum TargetCostKind CostKind) const {
 270   InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind);
 271   assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) &&
 272          "TTI should not produce negative costs!");
 273   return Cost;
 274 }
 275
 276 BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {
 277   return PredictableBranchThreshold.getNumOccurrences() > 0
 278              ? BranchProbability(PredictableBranchThreshold, 100)
 279              : TTIImpl->getPredictableBranchThreshold();
 280 }
 281
 282 InstructionCost TargetTransformInfo::getBranchMispredictPenalty() const {
 283   return TTIImpl->getBranchMispredictPenalty();
 284 }
 285
 286 bool TargetTransformInfo::hasBranchDivergence(const Function *F) const {
 287   return TTIImpl->hasBranchDivergence(F);
 288 }
 289
 290 bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
 291   return TTIImpl->isSourceOfDivergence(V);
 292 }
 293
 294 bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
 295   return TTIImpl->isAlwaysUniform(V);
 296 }
 297
 298 bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS,
 299                                                      unsigned ToAS) const {
 300   return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS);
 301 }
 302
 303 bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS,
 304                                                    unsigned ToAS) const {
 305   return TTIImpl->addrspacesMayAlias(FromAS, ToAS);
 306 }
 307
 308 unsigned TargetTransformInfo::getFlatAddressSpace() const {
 309   return TTIImpl->getFlatAddressSpace();
 310 }
 311
 312 bool TargetTransformInfo::collectFlatAddressOperands(
 313     SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const {
 314   return TTIImpl->collectFlatAddressOperands(OpIndexes, IID);
 315 }
 316
 317 bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS,
 318                                               unsigned ToAS) const {
 319   return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS);
 320 }
 321
 322 bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace(
 323     unsigned AS) const {
 324   return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS);
 325 }
 326
 327 unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const {
 328   return TTIImpl->getAssumedAddrSpace(V);
 329 }
 330
 331 bool TargetTransformInfo::isSingleThreaded() const {
 332   return TTIImpl->isSingleThreaded();
 333 }
 334
 335 std::pair<const Value *, unsigned>
 336 TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const {
 337   return TTIImpl->getPredicatedAddrSpace(V);
 338 }
 339
 340 Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace(
 341     IntrinsicInst *II, Value *OldV, Value *NewV) const {
 342   return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV);
 343 }
 344
 345 bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
 346   return TTIImpl->isLoweredToCall(F);
 347 }
 348
 349 bool TargetTransformInfo::isHardwareLoopProfitable(
 350     Loop *L, ScalarEvolution &SE, AssumptionCache &AC,
 351     TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const {
 352   return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
 353 }
 354
 355 bool TargetTransformInfo::preferPredicateOverEpilogue(
 356     TailFoldingInfo *TFI) const {
 357   return TTIImpl->preferPredicateOverEpilogue(TFI);
 358 }
 359
 360 TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle(
 361     bool IVUpdateMayOverflow) const {
 362   return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow);
 363 }
 364
 365 std::optional<Instruction *>
 366 TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,
 367                                           IntrinsicInst &II) const {
 368   return TTIImpl->instCombineIntrinsic(IC, II);
 369 }
 370
 371 std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(
 372     InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
 373     bool &KnownBitsComputed) const {
 374   return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
 375                                                    KnownBitsComputed);
 376 }
 377
 378 std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(
 379     InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
 380     APInt &UndefElts2, APInt &UndefElts3,
 381     std::function<void(Instruction *, unsigned, APInt, APInt &)>
 382         SimplifyAndSetOp) const {
 383   return TTIImpl->simplifyDemandedVectorEltsIntrinsic(
 384       IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
 385       SimplifyAndSetOp);
 386 }
 387
 388 void TargetTransformInfo::getUnrollingPreferences(
 389     Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP,
 390     OptimizationRemarkEmitter *ORE) const {
 391   return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE);
 392 }
 393
 394 void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
 395                                                 PeelingPreferences &PP) const {
 396   return TTIImpl->getPeelingPreferences(L, SE, PP);
 397 }
 398
 399 bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
 400   return TTIImpl->isLegalAddImmediate(Imm);
 401 }
 402
 403 bool TargetTransformInfo::isLegalAddScalableImmediate(int64_t Imm) const {
 404   return TTIImpl->isLegalAddScalableImmediate(Imm);
 405 }
 406
 407 bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
 408   return TTIImpl->isLegalICmpImmediate(Imm);
 409 }
 410
 411 bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
 412                                                 int64_t BaseOffset,
 413                                                 bool HasBaseReg, int64_t Scale,
 414                                                 unsigned AddrSpace,
 415                                                 Instruction *I,
 416                                                 int64_t ScalableOffset) const {
 417   return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
 418                                         Scale, AddrSpace, I, ScalableOffset);
 419 }
 420
 421 bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1,
 422                                         const LSRCost &C2) const {
 423   return TTIImpl->isLSRCostLess(C1, C2);
 424 }
 425
 426 bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
 427   return TTIImpl->isNumRegsMajorCostOfLSR();
 428 }
 429
 430 bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const {
 431   return TTIImpl->shouldFoldTerminatingConditionAfterLSR();
 432 }
 433
 434 bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {
 435   return TTIImpl->shouldDropLSRSolutionIfLessProfitable();
 436 }
 437
 438 bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const {
 439   return TTIImpl->isProfitableLSRChainElement(I);
 440 }
 441
 442 bool TargetTransformInfo::canMacroFuseCmp() const {
 443   return TTIImpl->canMacroFuseCmp();
 444 }
 445
 446 bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI,
 447                                      ScalarEvolution *SE, LoopInfo *LI,
 448                                      DominatorTree *DT, AssumptionCache *AC,
 449                                      TargetLibraryInfo *LibInfo) const {
 450   return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
 451 }
 452
 453 TTI::AddressingModeKind
 454 TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
 455                                                 ScalarEvolution *SE) const {
 456   return TTIImpl->getPreferredAddressingMode(L, SE);
 457 }
 458
 459 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
 460                                              Align Alignment) const {
 461   return TTIImpl->isLegalMaskedStore(DataType, Alignment);
 462 }
 463
 464 bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
 465                                             Align Alignment) const {
 466   return TTIImpl->isLegalMaskedLoad(DataType, Alignment);
 467 }
 468
 469 bool TargetTransformInfo::isLegalNTStore(Type *DataType,
 470                                          Align Alignment) const {
 471   return TTIImpl->isLegalNTStore(DataType, Alignment);
 472 }
 473
 474 bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
 475   return TTIImpl->isLegalNTLoad(DataType, Alignment);
 476 }
 477
 478 bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy,
 479                                                ElementCount NumElements) const {
 480   return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements);
 481 }
 482
 483 bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
 484                                               Align Alignment) const {
 485   return TTIImpl->isLegalMaskedGather(DataType, Alignment);
 486 }
 487
 488 bool TargetTransformInfo::isLegalAltInstr(
 489     VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
 490     const SmallBitVector &OpcodeMask) const {
 491   return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
 492 }
 493
 494 bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
 495                                                Align Alignment) const {
 496   return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
 497 }
 498
 499 bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType,
 500                                                      Align Alignment) const {
 501   return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment);
 502 }
 503
 504 bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType,
 505                                                       Align Alignment) const {
 506   return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment);
 507 }
 508
 509 bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType,
 510                                                      Align Alignment) const {
 511   return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment);
 512 }
 513
 514 bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType,
 515                                                   Align Alignment) const {
 516   return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment);
 517 }
 518
 519 bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
 520                                                   Align Alignment) const {
 521   return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
 522 }
 523
 524 bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
 525                                                        Type *DataType) const {
 526   return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
 527 }
 528
 529 bool TargetTransformInfo::enableOrderedReductions() const {
 530   return TTIImpl->enableOrderedReductions();
 531 }
 532
 533 bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
 534   return TTIImpl->hasDivRemOp(DataType, IsSigned);
 535 }
 536
 537 bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
 538                                              unsigned AddrSpace) const {
 539   return TTIImpl->hasVolatileVariant(I, AddrSpace);
 540 }
 541
 542 bool TargetTransformInfo::prefersVectorizedAddressing() const {
 543   return TTIImpl->prefersVectorizedAddressing();
 544 }
 545
 546 InstructionCost TargetTransformInfo::getScalingFactorCost(
 547     Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg,
 548     int64_t Scale, unsigned AddrSpace) const {
 549   InstructionCost Cost = TTIImpl->getScalingFactorCost(
 550       Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);
 551   assert(Cost >= 0 && "TTI should not produce negative costs!");
 552   return Cost;
 553 }
 554
 555 bool TargetTransformInfo::LSRWithInstrQueries() const {
 556   return TTIImpl->LSRWithInstrQueries();
 557 }
 558
 559 bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
 560   return TTIImpl->isTruncateFree(Ty1, Ty2);
 561 }
 562
 563 bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
 564   return TTIImpl->isProfitableToHoist(I);
 565 }
 566
 567 bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
 568
 569 bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
 570   return TTIImpl->isTypeLegal(Ty);
 571 }
 572
 573 unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const {
 574   return TTIImpl->getRegUsageForType(Ty);
 575 }
 576
 577 bool TargetTransformInfo::shouldBuildLookupTables() const {
 578   return TTIImpl->shouldBuildLookupTables();
 579 }
 580
 581 bool TargetTransformInfo::shouldBuildLookupTablesForConstant(
 582     Constant *C) const {
 583   return TTIImpl->shouldBuildLookupTablesForConstant(C);
 584 }
 585
 586 bool TargetTransformInfo::shouldBuildRelLookupTables() const {
 587   return TTIImpl->shouldBuildRelLookupTables();
 588 }
 589
 590 bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
 591   return TTIImpl->useColdCCForColdCall(F);
 592 }
 593
 594 InstructionCost TargetTransformInfo::getScalarizationOverhead(
 595     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
 596     TTI::TargetCostKind CostKind) const {
 597   return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
 598                                            CostKind);
 599 }
 600
 601 InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
 602     ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
 603     TTI::TargetCostKind CostKind) const {
 604   return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind);
 605 }
 606
 607 bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
 608   return TTIImpl->supportsEfficientVectorElementLoadStore();
 609 }
 610
 611 bool TargetTransformInfo::supportsTailCalls() const {
 612   return TTIImpl->supportsTailCalls();
 613 }
 614
 615 bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const {
 616   return TTIImpl->supportsTailCallFor(CB);
 617 }
 618
 619 bool TargetTransformInfo::enableAggressiveInterleaving(
 620     bool LoopHasReductions) const {
 621   return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
 622 }
 623
 624 TargetTransformInfo::MemCmpExpansionOptions
 625 TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
 626   return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp);
 627 }
 628
 629 bool TargetTransformInfo::enableSelectOptimize() const {
 630   return TTIImpl->enableSelectOptimize();
 631 }
 632
 633 bool TargetTransformInfo::shouldTreatInstructionLikeSelect(
 634     const Instruction *I) const {
 635   return TTIImpl->shouldTreatInstructionLikeSelect(I);
 636 }
 637
 638 bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
 639   return TTIImpl->enableInterleavedAccessVectorization();
 640 }
 641
 642 bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
 643   return TTIImpl->enableMaskedInterleavedAccessVectorization();
 644 }
 645
 646 bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
 647   return TTIImpl->isFPVectorizationPotentiallyUnsafe();
 648 }
 649
 650 bool
 651 TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
 652                                                     unsigned BitWidth,
 653                                                     unsigned AddressSpace,
 654                                                     Align Alignment,
 655                                                     unsigned *Fast) const {
 656   return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth,
 657                                                  AddressSpace, Alignment, Fast);
 658 }
 659
 660 TargetTransformInfo::PopcntSupportKind
 661 TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
 662   return TTIImpl->getPopcntSupport(IntTyWidthInBit);
 663 }
 664
 665 bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
 666   return TTIImpl->haveFastSqrt(Ty);
 667 }
 668
 669 bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(
 670     const Instruction *I) const {
 671   return TTIImpl->isExpensiveToSpeculativelyExecute(I);
 672 }
 673
 674 bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
 675   return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
 676 }
 677
 678 InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const {
 679   InstructionCost Cost = TTIImpl->getFPOpCost(Ty);
 680   assert(Cost >= 0 && "TTI should not produce negative costs!");
 681   return Cost;
 682 }
 683
 684 InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode,
 685                                                            unsigned Idx,
 686                                                            const APInt &Imm,
 687                                                            Type *Ty) const {
 688   InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
 689   assert(Cost >= 0 && "TTI should not produce negative costs!");
 690   return Cost;
 691 }
 692
 693 InstructionCost
 694 TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty,
 695                                    TTI::TargetCostKind CostKind) const {
 696   InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind);
 697   assert(Cost >= 0 && "TTI should not produce negative costs!");
 698   return Cost;
 699 }
 700
 701 InstructionCost TargetTransformInfo::getIntImmCostInst(
 702     unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty,
 703     TTI::TargetCostKind CostKind, Instruction *Inst) const {
 704   InstructionCost Cost =
 705       TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);
 706   assert(Cost >= 0 && "TTI should not produce negative costs!");
 707   return Cost;
 708 }
 709
 710 InstructionCost
 711 TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
 712                                          const APInt &Imm, Type *Ty,
 713                                          TTI::TargetCostKind CostKind) const {
 714   InstructionCost Cost =
 715       TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
 716   assert(Cost >= 0 && "TTI should not produce negative costs!");
 717   return Cost;
 718 }
 719
 720 bool TargetTransformInfo::preferToKeepConstantsAttached(
 721     const Instruction &Inst, const Function &Fn) const {
 722   return TTIImpl->preferToKeepConstantsAttached(Inst, Fn);
 723 }
 724
 725 unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {
 726   return TTIImpl->getNumberOfRegisters(ClassID);
 727 }
 728
 729 bool TargetTransformInfo::hasConditionalLoadStoreForType(Type *Ty) const {
 730   return TTIImpl->hasConditionalLoadStoreForType(Ty);
 731 }
 732
 733 unsigned TargetTransformInfo::getRegisterClassForType(bool Vector,
 734                                                       Type *Ty) const {
 735   return TTIImpl->getRegisterClassForType(Vector, Ty);
 736 }
 737
 738 const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {
 739   return TTIImpl->getRegisterClassName(ClassID);
 740 }
 741
 742 TypeSize TargetTransformInfo::getRegisterBitWidth(
 743     TargetTransformInfo::RegisterKind K) const {
 744   return TTIImpl->getRegisterBitWidth(K);
 745 }
 746
 747 unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
 748   return TTIImpl->getMinVectorRegisterBitWidth();
 749 }
 750
 751 std::optional<unsigned> TargetTransformInfo::getMaxVScale() const {
 752   return TTIImpl->getMaxVScale();
 753 }
 754
 755 std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const {
 756   return TTIImpl->getVScaleForTuning();
 757 }
 758
 759 bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const {
 760   return TTIImpl->isVScaleKnownToBeAPowerOfTwo();
 761 }
 762
 763 bool TargetTransformInfo::shouldMaximizeVectorBandwidth(
 764     TargetTransformInfo::RegisterKind K) const {
 765   return TTIImpl->shouldMaximizeVectorBandwidth(K);
 766 }
 767
 768 ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth,
 769                                                bool IsScalable) const {
 770   return TTIImpl->getMinimumVF(ElemWidth, IsScalable);
 771 }
 772
 773 unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth,
 774                                            unsigned Opcode) const {
 775   return TTIImpl->getMaximumVF(ElemWidth, Opcode);
 776 }
 777
 778 unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
 779                                                 Type *ScalarValTy) const {
 780   return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);
 781 }
 782
 783 bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
 784     const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
 785   return TTIImpl->shouldConsiderAddressTypePromotion(
 786       I, AllowPromotionWithoutCommonHeader);
 787 }
 788
 789 unsigned TargetTransformInfo::getCacheLineSize() const {
 790   return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize
 791                                                : TTIImpl->getCacheLineSize();
 792 }
 793
 794 std::optional<unsigned>
 795 TargetTransformInfo::getCacheSize(CacheLevel Level) const {
 796   return TTIImpl->getCacheSize(Level);
 797 }
 798
 799 std::optional<unsigned>
 800 TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const {
 801   return TTIImpl->getCacheAssociativity(Level);
 802 }
 803
 804 std::optional<unsigned> TargetTransformInfo::getMinPageSize() const {
 805   return MinPageSize.getNumOccurrences() > 0 ? MinPageSize
 806                                              : TTIImpl->getMinPageSize();
 807 }
 808
 809 unsigned TargetTransformInfo::getPrefetchDistance() const {
 810   return TTIImpl->getPrefetchDistance();
 811 }
 812
 813 unsigned TargetTransformInfo::getMinPrefetchStride(
 814     unsigned NumMemAccesses, unsigned NumStridedMemAccesses,
 815     unsigned NumPrefetches, bool HasCall) const {
 816   return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
 817                                        NumPrefetches, HasCall);
 818 }
 819
 820 unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
 821   return TTIImpl->getMaxPrefetchIterationsAhead();
 822 }
 823
 824 bool TargetTransformInfo::enableWritePrefetching() const {
 825   return TTIImpl->enableWritePrefetching();
 826 }
 827
 828 bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
 829   return TTIImpl->shouldPrefetchAddressSpace(AS);
 830 }
 831
 832 unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
 833   return TTIImpl->getMaxInterleaveFactor(VF);
 834 }
 835
 836 TargetTransformInfo::OperandValueInfo
 837 TargetTransformInfo::getOperandInfo(const Value *V) {
 838   OperandValueKind OpInfo = OK_AnyValue;
 839   OperandValueProperties OpProps = OP_None;
 840
 841   if (isa<ConstantInt>(V) || isa<ConstantFP>(V)) {
 842     if (const auto *CI = dyn_cast<ConstantInt>(V)) {
 843       if (CI->getValue().isPowerOf2())
 844         OpProps = OP_PowerOf2;
 845       else if (CI->getValue().isNegatedPowerOf2())
 846         OpProps = OP_NegatedPowerOf2;
 847     }
 848     return {OK_UniformConstantValue, OpProps};
 849   }
 850
 851   // A broadcast shuffle creates a uniform value.
 852   // TODO: Add support for non-zero index broadcasts.
 853   // TODO: Add support for different source vector width.
 854   if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))
 855     if (ShuffleInst->isZeroEltSplat())
 856       OpInfo = OK_UniformValue;
 857
 858   const Value *Splat = getSplatValue(V);
 859
 860   // Check for a splat of a constant or for a non uniform vector of constants
 861   // and check if the constant(s) are all powers of two.
 862   if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {
 863     OpInfo = OK_NonUniformConstantValue;
 864     if (Splat) {
 865       OpInfo = OK_UniformConstantValue;
 866       if (auto *CI = dyn_cast<ConstantInt>(Splat)) {
 867         if (CI->getValue().isPowerOf2())
 868           OpProps = OP_PowerOf2;
 869         else if (CI->getValue().isNegatedPowerOf2())
 870           OpProps = OP_NegatedPowerOf2;
 871       }
 872     } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
 873       bool AllPow2 = true, AllNegPow2 = true;
 874       for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
 875         if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I))) {
 876           AllPow2 &= CI->getValue().isPowerOf2();
 877           AllNegPow2 &= CI->getValue().isNegatedPowerOf2();
 878           if (AllPow2 || AllNegPow2)
 879             continue;
 880         }
 881         AllPow2 = AllNegPow2 = false;
 882         break;
 883       }
 884       OpProps = AllPow2 ? OP_PowerOf2 : OpProps;
 885       OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps;
 886     }
 887   }
 888
 889   // Check for a splat of a uniform value. This is not loop aware, so return
 890   // true only for the obviously uniform cases (argument, globalvalue)
 891   if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat)))
 892     OpInfo = OK_UniformValue;
 893
 894   return {OpInfo, OpProps};
 895 }
 896
 897 InstructionCost TargetTransformInfo::getArithmeticInstrCost(
 898     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
 899     OperandValueInfo Op1Info, OperandValueInfo Op2Info,
 900     ArrayRef<const Value *> Args, const Instruction *CxtI,
 901     const TargetLibraryInfo *TLibInfo) const {
 902
 903   // Use call cost for frem intructions that have platform specific vector math
 904   // functions, as those will be replaced with calls later by SelectionDAG or
 905   // ReplaceWithVecLib pass.
 906   if (TLibInfo && Opcode == Instruction::FRem) {
 907     VectorType *VecTy = dyn_cast<VectorType>(Ty);
 908     LibFunc Func;
 909     if (VecTy &&
 910         TLibInfo->getLibFunc(Instruction::FRem, Ty->getScalarType(), Func) &&
 911         TLibInfo->isFunctionVectorizable(TLibInfo->getName(Func),
 912                                          VecTy->getElementCount()))
 913       return getCallInstrCost(nullptr, VecTy, {VecTy, VecTy}, CostKind);
 914   }
 915
 916   InstructionCost Cost =
 917       TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind,
 918                                       Op1Info, Op2Info,
 919                                       Args, CxtI);
 920   assert(Cost >= 0 && "TTI should not produce negative costs!");
 921   return Cost;
 922 }
 923
 924 InstructionCost TargetTransformInfo::getAltInstrCost(
 925     VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
 926     const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {
 927   InstructionCost Cost =
 928       TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
 929   assert(Cost >= 0 && "TTI should not produce negative costs!");
 930   return Cost;
 931 }
 932
 933 InstructionCost TargetTransformInfo::getShuffleCost(
 934     ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask,
 935     TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
 936     ArrayRef<const Value *> Args, const Instruction *CxtI) const {
 937   InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, CostKind,
 938                                                  Index, SubTp, Args, CxtI);
 939   assert(Cost >= 0 && "TTI should not produce negative costs!");
 940   return Cost;
 941 }
 942
 943 TTI::CastContextHint
 944 TargetTransformInfo::getCastContextHint(const Instruction *I) {
 945   if (!I)
 946     return CastContextHint::None;
 947
 948   auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,
 949                              unsigned GatScatOp) {
 950     const Instruction *I = dyn_cast<Instruction>(V);
 951     if (!I)
 952       return CastContextHint::None;
 953
 954     if (I->getOpcode() == LdStOp)
 955       return CastContextHint::Normal;
 956
 957     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
 958       if (II->getIntrinsicID() == MaskedOp)
 959         return TTI::CastContextHint::Masked;
 960       if (II->getIntrinsicID() == GatScatOp)
 961         return TTI::CastContextHint::GatherScatter;
 962     }
 963
 964     return TTI::CastContextHint::None;
 965   };
 966
 967   switch (I->getOpcode()) {
 968   case Instruction::ZExt:
 969   case Instruction::SExt:
 970   case Instruction::FPExt:
 971     return getLoadStoreKind(I->getOperand(0), Instruction::Load,
 972                             Intrinsic::masked_load, Intrinsic::masked_gather);
 973   case Instruction::Trunc:
 974   case Instruction::FPTrunc:
 975     if (I->hasOneUse())
 976       return getLoadStoreKind(*I->user_begin(), Instruction::Store,
 977                               Intrinsic::masked_store,
 978                               Intrinsic::masked_scatter);
 979     break;
 980   default:
 981     return CastContextHint::None;
 982   }
 983
 984   return TTI::CastContextHint::None;
 985 }
 986
 987 InstructionCost TargetTransformInfo::getCastInstrCost(
 988     unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH,
 989     TTI::TargetCostKind CostKind, const Instruction *I) const {
 990   assert((I == nullptr || I->getOpcode() == Opcode) &&
 991          "Opcode should reflect passed instruction.");
 992   InstructionCost Cost =
 993       TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
 994   assert(Cost >= 0 && "TTI should not produce negative costs!");
 995   return Cost;
 996 }
 997
 998 InstructionCost TargetTransformInfo::getExtractWithExtendCost(
 999     unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index) const {
1000   InstructionCost Cost =
1001       TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
1002   assert(Cost >= 0 && "TTI should not produce negative costs!");
1003   return Cost;
1004 }
1005
1006 InstructionCost TargetTransformInfo::getCFInstrCost(
1007     unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const {
1008   assert((I == nullptr || I->getOpcode() == Opcode) &&
1009          "Opcode should reflect passed instruction.");
1010   InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I);
1011   assert(Cost >= 0 && "TTI should not produce negative costs!");
1012   return Cost;
1013 }
1014
1015 InstructionCost TargetTransformInfo::getCmpSelInstrCost(
1016     unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
1017     TTI::TargetCostKind CostKind, const Instruction *I) const {
1018   assert((I == nullptr || I->getOpcode() == Opcode) &&
1019          "Opcode should reflect passed instruction.");
1020   InstructionCost Cost =
1021       TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
1022   assert(Cost >= 0 && "TTI should not produce negative costs!");
1023   return Cost;
1024 }
1025
1026 InstructionCost TargetTransformInfo::getVectorInstrCost(
1027     unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1028     Value *Op0, Value *Op1) const {
1029   // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
1030   // This is mentioned in the interface description and respected by all
1031   // callers, but never asserted upon.
1032   InstructionCost Cost =
1033       TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
1034   assert(Cost >= 0 && "TTI should not produce negative costs!");
1035   return Cost;
1036 }
1037
1038 InstructionCost
1039 TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
1040                                         TTI::TargetCostKind CostKind,
1041                                         unsigned Index) const {
1042   // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
1043   // This is mentioned in the interface description and respected by all
1044   // callers, but never asserted upon.
1045   InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index);
1046   assert(Cost >= 0 && "TTI should not produce negative costs!");
1047   return Cost;
1048 }
1049
1050 InstructionCost TargetTransformInfo::getReplicationShuffleCost(
1051     Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
1052     TTI::TargetCostKind CostKind) const {
1053   InstructionCost Cost = TTIImpl->getReplicationShuffleCost(
1054       EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind);
1055   assert(Cost >= 0 && "TTI should not produce negative costs!");
1056   return Cost;
1057 }
1058
1059 InstructionCost TargetTransformInfo::getMemoryOpCost(
1060     unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1061     TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
1062     const Instruction *I) const {
1063   assert((I == nullptr || I->getOpcode() == Opcode) &&
1064          "Opcode should reflect passed instruction.");
1065   InstructionCost Cost = TTIImpl->getMemoryOpCost(
1066       Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I);
1067   assert(Cost >= 0 && "TTI should not produce negative costs!");
1068   return Cost;
1069 }
1070
1071 InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
1072     unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1073     TTI::TargetCostKind CostKind) const {
1074   InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
1075                                                         AddressSpace, CostKind);
1076   assert(Cost >= 0 && "TTI should not produce negative costs!");
1077   return Cost;
1078 }
1079
1080 InstructionCost TargetTransformInfo::getGatherScatterOpCost(
1081     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1082     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1083   InstructionCost Cost = TTIImpl->getGatherScatterOpCost(
1084       Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1085   assert((!Cost.isValid() || Cost >= 0) &&
1086          "TTI should not produce negative costs!");
1087   return Cost;
1088 }
1089
1090 InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
1091     unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1092     Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1093   InstructionCost Cost = TTIImpl->getStridedMemoryOpCost(
1094       Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1095   assert(Cost >= 0 && "TTI should not produce negative costs!");
1096   return Cost;
1097 }
1098
1099 InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
1100     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1101     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1102     bool UseMaskForCond, bool UseMaskForGaps) const {
1103   InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost(
1104       Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
1105       UseMaskForCond, UseMaskForGaps);
1106   assert(Cost >= 0 && "TTI should not produce negative costs!");
1107   return Cost;
1108 }
1109
1110 InstructionCost
1111 TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1112                                            TTI::TargetCostKind CostKind) const {
1113   InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind);
1114   assert(Cost >= 0 && "TTI should not produce negative costs!");
1115   return Cost;
1116 }
1117
1118 InstructionCost
1119 TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
1120                                       ArrayRef<Type *> Tys,
1121                                       TTI::TargetCostKind CostKind) const {
1122   InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind);
1123   assert(Cost >= 0 && "TTI should not produce negative costs!");
1124   return Cost;
1125 }
1126
1127 unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
1128   return TTIImpl->getNumberOfParts(Tp);
1129 }
1130
1131 InstructionCost
1132 TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
1133                                                const SCEV *Ptr) const {
1134   InstructionCost Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
1135   assert(Cost >= 0 && "TTI should not produce negative costs!");
1136   return Cost;
1137 }
1138
1139 InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const {
1140   InstructionCost Cost = TTIImpl->getMemcpyCost(I);
1141   assert(Cost >= 0 && "TTI should not produce negative costs!");
1142   return Cost;
1143 }
1144
1145 uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const {
1146   return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold();
1147 }
1148
1149 InstructionCost TargetTransformInfo::getArithmeticReductionCost(
1150     unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
1151     TTI::TargetCostKind CostKind) const {
1152   InstructionCost Cost =
1153       TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1154   assert(Cost >= 0 && "TTI should not produce negative costs!");
1155   return Cost;
1156 }
1157
1158 InstructionCost TargetTransformInfo::getMinMaxReductionCost(
1159     Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
1160     TTI::TargetCostKind CostKind) const {
1161   InstructionCost Cost =
1162       TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1163   assert(Cost >= 0 && "TTI should not produce negative costs!");
1164   return Cost;
1165 }
1166
1167 InstructionCost TargetTransformInfo::getExtendedReductionCost(
1168     unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
1169     FastMathFlags FMF, TTI::TargetCostKind CostKind) const {
1170   return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF,
1171                                            CostKind);
1172 }
1173
1174 InstructionCost TargetTransformInfo::getMulAccReductionCost(
1175     bool IsUnsigned, Type *ResTy, VectorType *Ty,
1176     TTI::TargetCostKind CostKind) const {
1177   return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind);
1178 }
1179
1180 InstructionCost
1181 TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1182   return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
1183 }
1184
1185 bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
1186                                              MemIntrinsicInfo &Info) const {
1187   return TTIImpl->getTgtMemIntrinsic(Inst, Info);
1188 }
1189
1190 unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
1191   return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
1192 }
1193
1194 Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
1195     IntrinsicInst *Inst, Type *ExpectedType) const {
1196   return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
1197 }
1198
1199 Type *TargetTransformInfo::getMemcpyLoopLoweringType(
1200     LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1201     unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
1202     std::optional<uint32_t> AtomicElementSize) const {
1203   return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
1204                                             DestAddrSpace, SrcAlign, DestAlign,
1205                                             AtomicElementSize);
1206 }
1207
1208 void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
1209     SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
1210     unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1211     unsigned SrcAlign, unsigned DestAlign,
1212     std::optional<uint32_t> AtomicCpySize) const {
1213   TTIImpl->getMemcpyLoopResidualLoweringType(
1214       OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
1215       DestAlign, AtomicCpySize);
1216 }
1217
1218 bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
1219                                               const Function *Callee) const {
1220   return TTIImpl->areInlineCompatible(Caller, Callee);
1221 }
1222
1223 unsigned
1224 TargetTransformInfo::getInlineCallPenalty(const Function *F,
1225                                           const CallBase &Call,
1226                                           unsigned DefaultCallPenalty) const {
1227   return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty);
1228 }
1229
1230 bool TargetTransformInfo::areTypesABICompatible(
1231     const Function *Caller, const Function *Callee,
1232     const ArrayRef<Type *> &Types) const {
1233   return TTIImpl->areTypesABICompatible(Caller, Callee, Types);
1234 }
1235
1236 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
1237                                              Type *Ty) const {
1238   return TTIImpl->isIndexedLoadLegal(Mode, Ty);
1239 }
1240
1241 bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
1242                                               Type *Ty) const {
1243   return TTIImpl->isIndexedStoreLegal(Mode, Ty);
1244 }
1245
1246 unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
1247   return TTIImpl->getLoadStoreVecRegBitWidth(AS);
1248 }
1249
1250 bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
1251   return TTIImpl->isLegalToVectorizeLoad(LI);
1252 }
1253
1254 bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
1255   return TTIImpl->isLegalToVectorizeStore(SI);
1256 }
1257
1258 bool TargetTransformInfo::isLegalToVectorizeLoadChain(
1259     unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1260   return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
1261                                               AddrSpace);
1262 }
1263
1264 bool TargetTransformInfo::isLegalToVectorizeStoreChain(
1265     unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1266   return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
1267                                                AddrSpace);
1268 }
1269
1270 bool TargetTransformInfo::isLegalToVectorizeReduction(
1271     const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
1272   return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF);
1273 }
1274
1275 bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const {
1276   return TTIImpl->isElementTypeLegalForScalableVector(Ty);
1277 }
1278
1279 unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
1280                                                   unsigned LoadSize,
1281                                                   unsigned ChainSizeInBytes,
1282                                                   VectorType *VecTy) const {
1283   return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
1284 }
1285
1286 unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
1287                                                    unsigned StoreSize,
1288                                                    unsigned ChainSizeInBytes,
1289                                                    VectorType *VecTy) const {
1290   return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
1291 }
1292
1293 bool TargetTransformInfo::preferFixedOverScalableIfEqualCost() const {
1294   return TTIImpl->preferFixedOverScalableIfEqualCost();
1295 }
1296
1297 bool TargetTransformInfo::preferInLoopReduction(unsigned Opcode, Type *Ty,
1298                                                 ReductionFlags Flags) const {
1299   return TTIImpl->preferInLoopReduction(Opcode, Ty, Flags);
1300 }
1301
1302 bool TargetTransformInfo::preferPredicatedReductionSelect(
1303     unsigned Opcode, Type *Ty, ReductionFlags Flags) const {
1304   return TTIImpl->preferPredicatedReductionSelect(Opcode, Ty, Flags);
1305 }
1306
1307 bool TargetTransformInfo::preferEpilogueVectorization() const {
1308   return TTIImpl->preferEpilogueVectorization();
1309 }
1310
1311 TargetTransformInfo::VPLegalization
1312 TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
1313   return TTIImpl->getVPLegalizationStrategy(VPI);
1314 }
1315
1316 bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const {
1317   return TTIImpl->hasArmWideBranch(Thumb);
1318 }
1319
1320 unsigned TargetTransformInfo::getMaxNumArgs() const {
1321   return TTIImpl->getMaxNumArgs();
1322 }
1323
1324 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
1325   return TTIImpl->shouldExpandReduction(II);
1326 }
1327
1328 TargetTransformInfo::ReductionShuffle
1329 TargetTransformInfo::getPreferredExpandedReductionShuffle(
1330     const IntrinsicInst *II) const {
1331   return TTIImpl->getPreferredExpandedReductionShuffle(II);
1332 }
1333
1334 unsigned TargetTransformInfo::getGISelRematGlobalCost() const {
1335   return TTIImpl->getGISelRematGlobalCost();
1336 }
1337
1338 unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const {
1339   return TTIImpl->getMinTripCountTailFoldingThreshold();
1340 }
1341
1342 bool TargetTransformInfo::supportsScalableVectors() const {
1343   return TTIImpl->supportsScalableVectors();
1344 }
1345
1346 bool TargetTransformInfo::enableScalableVectorization() const {
1347   return TTIImpl->enableScalableVectorization();
1348 }
1349
1350 bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType,
1351                                                 Align Alignment) const {
1352   return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment);
1353 }
1354
1355 TargetTransformInfo::Concept::~Concept() = default;
1356
1357 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1358
1359 TargetIRAnalysis::TargetIRAnalysis(
1360     std::function<Result(const Function &)> TTICallback)
1361     : TTICallback(std::move(TTICallback)) {}
1362
1363 TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
1364                                                FunctionAnalysisManager &) {
1365   return TTICallback(F);
1366 }
1367
1368 AnalysisKey TargetIRAnalysis::Key;
1369
1370 TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
1371   return Result(F.getDataLayout());
1372 }
1373
1374 // Register the basic pass.
1375 INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1376                 "Target Transform Information", false, true)
1377 char TargetTransformInfoWrapperPass::ID = 0;
1378
1379 void TargetTransformInfoWrapperPass::anchor() {}
1380
1381 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
1382     : ImmutablePass(ID) {
1383   initializeTargetTransformInfoWrapperPassPass(
1384       *PassRegistry::getPassRegistry());
1385 }
1386
1387 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
1388     TargetIRAnalysis TIRA)
1389     : ImmutablePass(ID), TIRA(std::move(TIRA)) {
1390   initializeTargetTransformInfoWrapperPassPass(
1391       *PassRegistry::getPassRegistry());
1392 }
1393
1394 TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
1395   FunctionAnalysisManager DummyFAM;
1396   TTI = TIRA.run(F, DummyFAM);
1397   return *TTI;
1398 }
1399
1400 ImmutablePass *
1401 llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1402   return new TargetTransformInfoWrapperPass(std::move(TIRA));
1403 }