lib/Analysis/TargetTransformInfo.cpp

   1 //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Analysis/TargetTransformInfo.h"
  10 #include "llvm/Analysis/TargetTransformInfoImpl.h"
  11 #include "llvm/IR/CallSite.h"
  12 #include "llvm/IR/DataLayout.h"
  13 #include "llvm/IR/Instruction.h"
  14 #include "llvm/IR/Instructions.h"
  15 #include "llvm/IR/IntrinsicInst.h"
  16 #include "llvm/IR/Module.h"
  17 #include "llvm/IR/Operator.h"
  18 #include "llvm/IR/PatternMatch.h"
  19 #include "llvm/Support/CommandLine.h"
  20 #include "llvm/Support/ErrorHandling.h"
  21 #include <utility>
  22
  23 using namespace llvm;
  24 using namespace PatternMatch;
  25
  26 #define DEBUG_TYPE "tti"
  27
  28 static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(false),
  29                                      cl::Hidden,
  30                                      cl::desc("Recognize reduction patterns."));
  31
  32 namespace {
  33 /// No-op implementation of the TTI interface using the utility base
  34 /// classes.
  35 ///
  36 /// This is used when no target specific information is available.
  37 struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
  38   explicit NoTTIImpl(const DataLayout &DL)
  39       : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
  40 };
  41 }
  42
  43 TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
  44     : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
  45
  46 TargetTransformInfo::~TargetTransformInfo() {}
  47
  48 TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
  49     : TTIImpl(std::move(Arg.TTIImpl)) {}
  50
  51 TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
  52   TTIImpl = std::move(RHS.TTIImpl);
  53   return *this;
  54 }
  55
  56 int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty,
  57                                           Type *OpTy) const {
  58   int Cost = TTIImpl->getOperationCost(Opcode, Ty, OpTy);
  59   assert(Cost >= 0 && "TTI should not produce negative costs!");
  60   return Cost;
  61 }
  62
  63 int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs) const {
  64   int Cost = TTIImpl->getCallCost(FTy, NumArgs);
  65   assert(Cost >= 0 && "TTI should not produce negative costs!");
  66   return Cost;
  67 }
  68
  69 int TargetTransformInfo::getCallCost(const Function *F,
  70                                      ArrayRef<const Value *> Arguments) const {
  71   int Cost = TTIImpl->getCallCost(F, Arguments);
  72   assert(Cost >= 0 && "TTI should not produce negative costs!");
  73   return Cost;
  74 }
  75
  76 unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
  77   return TTIImpl->getInliningThresholdMultiplier();
  78 }
  79
  80 int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr,
  81                                     ArrayRef<const Value *> Operands) const {
  82   return TTIImpl->getGEPCost(PointeeType, Ptr, Operands);
  83 }
  84
  85 int TargetTransformInfo::getExtCost(const Instruction *I,
  86                                     const Value *Src) const {
  87   return TTIImpl->getExtCost(I, Src);
  88 }
  89
  90 int TargetTransformInfo::getIntrinsicCost(
  91     Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments) const {
  92   int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments);
  93   assert(Cost >= 0 && "TTI should not produce negative costs!");
  94   return Cost;
  95 }
  96
  97 unsigned
  98 TargetTransformInfo::getEstimatedNumberOfCaseClusters(const SwitchInst &SI,
  99                                                       unsigned &JTSize) const {
 100   return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize);
 101 }
 102
 103 int TargetTransformInfo::getUserCost(const User *U,
 104     ArrayRef<const Value *> Operands) const {
 105   int Cost = TTIImpl->getUserCost(U, Operands);
 106   assert(Cost >= 0 && "TTI should not produce negative costs!");
 107   return Cost;
 108 }
 109
 110 bool TargetTransformInfo::hasBranchDivergence() const {
 111   return TTIImpl->hasBranchDivergence();
 112 }
 113
 114 bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
 115   return TTIImpl->isSourceOfDivergence(V);
 116 }
 117
 118 bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
 119   return TTIImpl->isAlwaysUniform(V);
 120 }
 121
 122 unsigned TargetTransformInfo::getFlatAddressSpace() const {
 123   return TTIImpl->getFlatAddressSpace();
 124 }
 125
 126 bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
 127   return TTIImpl->isLoweredToCall(F);
 128 }
 129
 130 void TargetTransformInfo::getUnrollingPreferences(
 131     Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const {
 132   return TTIImpl->getUnrollingPreferences(L, SE, UP);
 133 }
 134
 135 bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
 136   return TTIImpl->isLegalAddImmediate(Imm);
 137 }
 138
 139 bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
 140   return TTIImpl->isLegalICmpImmediate(Imm);
 141 }
 142
 143 bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
 144                                                 int64_t BaseOffset,
 145                                                 bool HasBaseReg,
 146                                                 int64_t Scale,
 147                                                 unsigned AddrSpace,
 148                                                 Instruction *I) const {
 149   return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
 150                                         Scale, AddrSpace, I);
 151 }
 152
 153 bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const {
 154   return TTIImpl->isLSRCostLess(C1, C2);
 155 }
 156
 157 bool TargetTransformInfo::canMacroFuseCmp() const {
 158   return TTIImpl->canMacroFuseCmp();
 159 }
 160
 161 bool TargetTransformInfo::shouldFavorPostInc() const {
 162   return TTIImpl->shouldFavorPostInc();
 163 }
 164
 165 bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const {
 166   return TTIImpl->shouldFavorBackedgeIndex(L);
 167 }
 168
 169 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const {
 170   return TTIImpl->isLegalMaskedStore(DataType);
 171 }
 172
 173 bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const {
 174   return TTIImpl->isLegalMaskedLoad(DataType);
 175 }
 176
 177 bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const {
 178   return TTIImpl->isLegalMaskedGather(DataType);
 179 }
 180
 181 bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const {
 182   return TTIImpl->isLegalMaskedScatter(DataType);
 183 }
 184
 185 bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
 186   return TTIImpl->hasDivRemOp(DataType, IsSigned);
 187 }
 188
 189 bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
 190                                              unsigned AddrSpace) const {
 191   return TTIImpl->hasVolatileVariant(I, AddrSpace);
 192 }
 193
 194 bool TargetTransformInfo::prefersVectorizedAddressing() const {
 195   return TTIImpl->prefersVectorizedAddressing();
 196 }
 197
 198 int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
 199                                               int64_t BaseOffset,
 200                                               bool HasBaseReg,
 201                                               int64_t Scale,
 202                                               unsigned AddrSpace) const {
 203   int Cost = TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg,
 204                                            Scale, AddrSpace);
 205   assert(Cost >= 0 && "TTI should not produce negative costs!");
 206   return Cost;
 207 }
 208
 209 bool TargetTransformInfo::LSRWithInstrQueries() const {
 210   return TTIImpl->LSRWithInstrQueries();
 211 }
 212
 213 bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
 214   return TTIImpl->isTruncateFree(Ty1, Ty2);
 215 }
 216
 217 bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
 218   return TTIImpl->isProfitableToHoist(I);
 219 }
 220
 221 bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
 222
 223 bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
 224   return TTIImpl->isTypeLegal(Ty);
 225 }
 226
 227 unsigned TargetTransformInfo::getJumpBufAlignment() const {
 228   return TTIImpl->getJumpBufAlignment();
 229 }
 230
 231 unsigned TargetTransformInfo::getJumpBufSize() const {
 232   return TTIImpl->getJumpBufSize();
 233 }
 234
 235 bool TargetTransformInfo::shouldBuildLookupTables() const {
 236   return TTIImpl->shouldBuildLookupTables();
 237 }
 238 bool TargetTransformInfo::shouldBuildLookupTablesForConstant(Constant *C) const {
 239   return TTIImpl->shouldBuildLookupTablesForConstant(C);
 240 }
 241
 242 bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
 243   return TTIImpl->useColdCCForColdCall(F);
 244 }
 245
 246 unsigned TargetTransformInfo::
 247 getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const {
 248   return TTIImpl->getScalarizationOverhead(Ty, Insert, Extract);
 249 }
 250
 251 unsigned TargetTransformInfo::
 252 getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
 253                                  unsigned VF) const {
 254   return TTIImpl->getOperandsScalarizationOverhead(Args, VF);
 255 }
 256
 257 bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
 258   return TTIImpl->supportsEfficientVectorElementLoadStore();
 259 }
 260
 261 bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const {
 262   return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
 263 }
 264
 265 const TargetTransformInfo::MemCmpExpansionOptions *
 266 TargetTransformInfo::enableMemCmpExpansion(bool IsZeroCmp) const {
 267   return TTIImpl->enableMemCmpExpansion(IsZeroCmp);
 268 }
 269
 270 bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
 271   return TTIImpl->enableInterleavedAccessVectorization();
 272 }
 273
 274 bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
 275   return TTIImpl->enableMaskedInterleavedAccessVectorization();
 276 }
 277
 278 bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
 279   return TTIImpl->isFPVectorizationPotentiallyUnsafe();
 280 }
 281
 282 bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
 283                                                          unsigned BitWidth,
 284                                                          unsigned AddressSpace,
 285                                                          unsigned Alignment,
 286                                                          bool *Fast) const {
 287   return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace,
 288                                                  Alignment, Fast);
 289 }
 290
 291 TargetTransformInfo::PopcntSupportKind
 292 TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
 293   return TTIImpl->getPopcntSupport(IntTyWidthInBit);
 294 }
 295
 296 bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
 297   return TTIImpl->haveFastSqrt(Ty);
 298 }
 299
 300 bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
 301   return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
 302 }
 303
 304 int TargetTransformInfo::getFPOpCost(Type *Ty) const {
 305   int Cost = TTIImpl->getFPOpCost(Ty);
 306   assert(Cost >= 0 && "TTI should not produce negative costs!");
 307   return Cost;
 308 }
 309
 310 int TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
 311                                                const APInt &Imm,
 312                                                Type *Ty) const {
 313   int Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
 314   assert(Cost >= 0 && "TTI should not produce negative costs!");
 315   return Cost;
 316 }
 317
 318 int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const {
 319   int Cost = TTIImpl->getIntImmCost(Imm, Ty);
 320   assert(Cost >= 0 && "TTI should not produce negative costs!");
 321   return Cost;
 322 }
 323
 324 int TargetTransformInfo::getIntImmCost(unsigned Opcode, unsigned Idx,
 325                                        const APInt &Imm, Type *Ty) const {
 326   int Cost = TTIImpl->getIntImmCost(Opcode, Idx, Imm, Ty);
 327   assert(Cost >= 0 && "TTI should not produce negative costs!");
 328   return Cost;
 329 }
 330
 331 int TargetTransformInfo::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
 332                                        const APInt &Imm, Type *Ty) const {
 333   int Cost = TTIImpl->getIntImmCost(IID, Idx, Imm, Ty);
 334   assert(Cost >= 0 && "TTI should not produce negative costs!");
 335   return Cost;
 336 }
 337
 338 unsigned TargetTransformInfo::getNumberOfRegisters(bool Vector) const {
 339   return TTIImpl->getNumberOfRegisters(Vector);
 340 }
 341
 342 unsigned TargetTransformInfo::getRegisterBitWidth(bool Vector) const {
 343   return TTIImpl->getRegisterBitWidth(Vector);
 344 }
 345
 346 unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
 347   return TTIImpl->getMinVectorRegisterBitWidth();
 348 }
 349
 350 bool TargetTransformInfo::shouldMaximizeVectorBandwidth(bool OptSize) const {
 351   return TTIImpl->shouldMaximizeVectorBandwidth(OptSize);
 352 }
 353
 354 unsigned TargetTransformInfo::getMinimumVF(unsigned ElemWidth) const {
 355   return TTIImpl->getMinimumVF(ElemWidth);
 356 }
 357
 358 bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
 359     const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
 360   return TTIImpl->shouldConsiderAddressTypePromotion(
 361       I, AllowPromotionWithoutCommonHeader);
 362 }
 363
 364 unsigned TargetTransformInfo::getCacheLineSize() const {
 365   return TTIImpl->getCacheLineSize();
 366 }
 367
 368 llvm::Optional<unsigned> TargetTransformInfo::getCacheSize(CacheLevel Level)
 369   const {
 370   return TTIImpl->getCacheSize(Level);
 371 }
 372
 373 llvm::Optional<unsigned> TargetTransformInfo::getCacheAssociativity(
 374   CacheLevel Level) const {
 375   return TTIImpl->getCacheAssociativity(Level);
 376 }
 377
 378 unsigned TargetTransformInfo::getPrefetchDistance() const {
 379   return TTIImpl->getPrefetchDistance();
 380 }
 381
 382 unsigned TargetTransformInfo::getMinPrefetchStride() const {
 383   return TTIImpl->getMinPrefetchStride();
 384 }
 385
 386 unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
 387   return TTIImpl->getMaxPrefetchIterationsAhead();
 388 }
 389
 390 unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const {
 391   return TTIImpl->getMaxInterleaveFactor(VF);
 392 }
 393
 394 TargetTransformInfo::OperandValueKind
 395 TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) {
 396   OperandValueKind OpInfo = OK_AnyValue;
 397   OpProps = OP_None;
 398
 399   if (auto *CI = dyn_cast<ConstantInt>(V)) {
 400     if (CI->getValue().isPowerOf2())
 401       OpProps = OP_PowerOf2;
 402     return OK_UniformConstantValue;
 403   }
 404
 405   // A broadcast shuffle creates a uniform value.
 406   // TODO: Add support for non-zero index broadcasts.
 407   // TODO: Add support for different source vector width.
 408   if (auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V))
 409     if (ShuffleInst->isZeroEltSplat())
 410       OpInfo = OK_UniformValue;
 411
 412   const Value *Splat = getSplatValue(V);
 413
 414   // Check for a splat of a constant or for a non uniform vector of constants
 415   // and check if the constant(s) are all powers of two.
 416   if (isa<ConstantVector>(V) || isa<ConstantDataVector>(V)) {
 417     OpInfo = OK_NonUniformConstantValue;
 418     if (Splat) {
 419       OpInfo = OK_UniformConstantValue;
 420       if (auto *CI = dyn_cast<ConstantInt>(Splat))
 421         if (CI->getValue().isPowerOf2())
 422           OpProps = OP_PowerOf2;
 423     } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) {
 424       OpProps = OP_PowerOf2;
 425       for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
 426         if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I)))
 427           if (CI->getValue().isPowerOf2())
 428             continue;
 429         OpProps = OP_None;
 430         break;
 431       }
 432     }
 433   }
 434
 435   // Check for a splat of a uniform value. This is not loop aware, so return
 436   // true only for the obviously uniform cases (argument, globalvalue)
 437   if (Splat && (isa<Argument>(Splat) || isa<GlobalValue>(Splat)))
 438     OpInfo = OK_UniformValue;
 439
 440   return OpInfo;
 441 }
 442
 443 int TargetTransformInfo::getArithmeticInstrCost(
 444     unsigned Opcode, Type *Ty, OperandValueKind Opd1Info,
 445     OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo,
 446     OperandValueProperties Opd2PropInfo,
 447     ArrayRef<const Value *> Args) const {
 448   int Cost = TTIImpl->getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info,
 449                                              Opd1PropInfo, Opd2PropInfo, Args);
 450   assert(Cost >= 0 && "TTI should not produce negative costs!");
 451   return Cost;
 452 }
 453
 454 int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, Type *Ty, int Index,
 455                                         Type *SubTp) const {
 456   int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp);
 457   assert(Cost >= 0 && "TTI should not produce negative costs!");
 458   return Cost;
 459 }
 460
 461 int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst,
 462                                  Type *Src, const Instruction *I) const {
 463   assert ((I == nullptr || I->getOpcode() == Opcode) &&
 464           "Opcode should reflect passed instruction.");
 465   int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, I);
 466   assert(Cost >= 0 && "TTI should not produce negative costs!");
 467   return Cost;
 468 }
 469
 470 int TargetTransformInfo::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
 471                                                   VectorType *VecTy,
 472                                                   unsigned Index) const {
 473   int Cost = TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
 474   assert(Cost >= 0 && "TTI should not produce negative costs!");
 475   return Cost;
 476 }
 477
 478 int TargetTransformInfo::getCFInstrCost(unsigned Opcode) const {
 479   int Cost = TTIImpl->getCFInstrCost(Opcode);
 480   assert(Cost >= 0 && "TTI should not produce negative costs!");
 481   return Cost;
 482 }
 483
 484 int TargetTransformInfo::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
 485                                  Type *CondTy, const Instruction *I) const {
 486   assert ((I == nullptr || I->getOpcode() == Opcode) &&
 487           "Opcode should reflect passed instruction.");
 488   int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
 489   assert(Cost >= 0 && "TTI should not produce negative costs!");
 490   return Cost;
 491 }
 492
 493 int TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val,
 494                                             unsigned Index) const {
 495   int Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index);
 496   assert(Cost >= 0 && "TTI should not produce negative costs!");
 497   return Cost;
 498 }
 499
 500 int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src,
 501                                          unsigned Alignment,
 502                                          unsigned AddressSpace,
 503                                          const Instruction *I) const {
 504   assert ((I == nullptr || I->getOpcode() == Opcode) &&
 505           "Opcode should reflect passed instruction.");
 506   int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I);
 507   assert(Cost >= 0 && "TTI should not produce negative costs!");
 508   return Cost;
 509 }
 510
 511 int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
 512                                                unsigned Alignment,
 513                                                unsigned AddressSpace) const {
 514   int Cost =
 515       TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
 516   assert(Cost >= 0 && "TTI should not produce negative costs!");
 517   return Cost;
 518 }
 519
 520 int TargetTransformInfo::getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
 521                                                 Value *Ptr, bool VariableMask,
 522                                                 unsigned Alignment) const {
 523   int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
 524                                              Alignment);
 525   assert(Cost >= 0 && "TTI should not produce negative costs!");
 526   return Cost;
 527 }
 528
 529 int TargetTransformInfo::getInterleavedMemoryOpCost(
 530     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
 531     unsigned Alignment, unsigned AddressSpace, bool UseMaskForCond,
 532     bool UseMaskForGaps) const {
 533   int Cost = TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
 534                                                  Alignment, AddressSpace,
 535                                                  UseMaskForCond,
 536                                                  UseMaskForGaps);
 537   assert(Cost >= 0 && "TTI should not produce negative costs!");
 538   return Cost;
 539 }
 540
 541 int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
 542                                     ArrayRef<Type *> Tys, FastMathFlags FMF,
 543                                     unsigned ScalarizationCostPassed) const {
 544   int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys, FMF,
 545                                             ScalarizationCostPassed);
 546   assert(Cost >= 0 && "TTI should not produce negative costs!");
 547   return Cost;
 548 }
 549
 550 int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
 551            ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF) const {
 552   int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF);
 553   assert(Cost >= 0 && "TTI should not produce negative costs!");
 554   return Cost;
 555 }
 556
 557 int TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
 558                                           ArrayRef<Type *> Tys) const {
 559   int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys);
 560   assert(Cost >= 0 && "TTI should not produce negative costs!");
 561   return Cost;
 562 }
 563
 564 unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
 565   return TTIImpl->getNumberOfParts(Tp);
 566 }
 567
 568 int TargetTransformInfo::getAddressComputationCost(Type *Tp,
 569                                                    ScalarEvolution *SE,
 570                                                    const SCEV *Ptr) const {
 571   int Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
 572   assert(Cost >= 0 && "TTI should not produce negative costs!");
 573   return Cost;
 574 }
 575
 576 int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty,
 577                                                     bool IsPairwiseForm) const {
 578   int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm);
 579   assert(Cost >= 0 && "TTI should not produce negative costs!");
 580   return Cost;
 581 }
 582
 583 int TargetTransformInfo::getMinMaxReductionCost(Type *Ty, Type *CondTy,
 584                                                 bool IsPairwiseForm,
 585                                                 bool IsUnsigned) const {
 586   int Cost =
 587       TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned);
 588   assert(Cost >= 0 && "TTI should not produce negative costs!");
 589   return Cost;
 590 }
 591
 592 unsigned
 593 TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
 594   return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
 595 }
 596
 597 bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
 598                                              MemIntrinsicInfo &Info) const {
 599   return TTIImpl->getTgtMemIntrinsic(Inst, Info);
 600 }
 601
 602 unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
 603   return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
 604 }
 605
 606 Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
 607     IntrinsicInst *Inst, Type *ExpectedType) const {
 608   return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
 609 }
 610
 611 Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context,
 612                                                      Value *Length,
 613                                                      unsigned SrcAlign,
 614                                                      unsigned DestAlign) const {
 615   return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign,
 616                                             DestAlign);
 617 }
 618
 619 void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
 620     SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
 621     unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const {
 622   TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes,
 623                                              SrcAlign, DestAlign);
 624 }
 625
 626 bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
 627                                               const Function *Callee) const {
 628   return TTIImpl->areInlineCompatible(Caller, Callee);
 629 }
 630
 631 bool TargetTransformInfo::areFunctionArgsABICompatible(
 632     const Function *Caller, const Function *Callee,
 633     SmallPtrSetImpl<Argument *> &Args) const {
 634   return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args);
 635 }
 636
 637 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
 638                                              Type *Ty) const {
 639   return TTIImpl->isIndexedLoadLegal(Mode, Ty);
 640 }
 641
 642 bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
 643                                               Type *Ty) const {
 644   return TTIImpl->isIndexedStoreLegal(Mode, Ty);
 645 }
 646
 647 unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
 648   return TTIImpl->getLoadStoreVecRegBitWidth(AS);
 649 }
 650
 651 bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
 652   return TTIImpl->isLegalToVectorizeLoad(LI);
 653 }
 654
 655 bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
 656   return TTIImpl->isLegalToVectorizeStore(SI);
 657 }
 658
 659 bool TargetTransformInfo::isLegalToVectorizeLoadChain(
 660     unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
 661   return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
 662                                               AddrSpace);
 663 }
 664
 665 bool TargetTransformInfo::isLegalToVectorizeStoreChain(
 666     unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const {
 667   return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
 668                                                AddrSpace);
 669 }
 670
 671 unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
 672                                                   unsigned LoadSize,
 673                                                   unsigned ChainSizeInBytes,
 674                                                   VectorType *VecTy) const {
 675   return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
 676 }
 677
 678 unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
 679                                                    unsigned StoreSize,
 680                                                    unsigned ChainSizeInBytes,
 681                                                    VectorType *VecTy) const {
 682   return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
 683 }
 684
 685 bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode,
 686                                                 Type *Ty, ReductionFlags Flags) const {
 687   return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags);
 688 }
 689
 690 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
 691   return TTIImpl->shouldExpandReduction(II);
 692 }
 693
 694 int TargetTransformInfo::getInstructionLatency(const Instruction *I) const {
 695   return TTIImpl->getInstructionLatency(I);
 696 }
 697
 698 static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft,
 699                                      unsigned Level) {
 700   // We don't need a shuffle if we just want to have element 0 in position 0 of
 701   // the vector.
 702   if (!SI && Level == 0 && IsLeft)
 703     return true;
 704   else if (!SI)
 705     return false;
 706
 707   SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1);
 708
 709   // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether
 710   // we look at the left or right side.
 711   for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2)
 712     Mask[i] = val;
 713
 714   SmallVector<int, 16> ActualMask = SI->getShuffleMask();
 715   return Mask == ActualMask;
 716 }
 717
 718 namespace {
 719 /// Kind of the reduction data.
 720 enum ReductionKind {
 721   RK_None,           /// Not a reduction.
 722   RK_Arithmetic,     /// Binary reduction data.
 723   RK_MinMax,         /// Min/max reduction data.
 724   RK_UnsignedMinMax, /// Unsigned min/max reduction data.
 725 };
 726 /// Contains opcode + LHS/RHS parts of the reduction operations.
 727 struct ReductionData {
 728   ReductionData() = delete;
 729   ReductionData(ReductionKind Kind, unsigned Opcode, Value *LHS, Value *RHS)
 730       : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind) {
 731     assert(Kind != RK_None && "expected binary or min/max reduction only.");
 732   }
 733   unsigned Opcode = 0;
 734   Value *LHS = nullptr;
 735   Value *RHS = nullptr;
 736   ReductionKind Kind = RK_None;
 737   bool hasSameData(ReductionData &RD) const {
 738     return Kind == RD.Kind && Opcode == RD.Opcode;
 739   }
 740 };
 741 } // namespace
 742
 743 static Optional<ReductionData> getReductionData(Instruction *I) {
 744   Value *L, *R;
 745   if (m_BinOp(m_Value(L), m_Value(R)).match(I))
 746     return ReductionData(RK_Arithmetic, I->getOpcode(), L, R);
 747   if (auto *SI = dyn_cast<SelectInst>(I)) {
 748     if (m_SMin(m_Value(L), m_Value(R)).match(SI) ||
 749         m_SMax(m_Value(L), m_Value(R)).match(SI) ||
 750         m_OrdFMin(m_Value(L), m_Value(R)).match(SI) ||
 751         m_OrdFMax(m_Value(L), m_Value(R)).match(SI) ||
 752         m_UnordFMin(m_Value(L), m_Value(R)).match(SI) ||
 753         m_UnordFMax(m_Value(L), m_Value(R)).match(SI)) {
 754       auto *CI = cast<CmpInst>(SI->getCondition());
 755       return ReductionData(RK_MinMax, CI->getOpcode(), L, R);
 756     }
 757     if (m_UMin(m_Value(L), m_Value(R)).match(SI) ||
 758         m_UMax(m_Value(L), m_Value(R)).match(SI)) {
 759       auto *CI = cast<CmpInst>(SI->getCondition());
 760       return ReductionData(RK_UnsignedMinMax, CI->getOpcode(), L, R);
 761     }
 762   }
 763   return llvm::None;
 764 }
 765
 766 static ReductionKind matchPairwiseReductionAtLevel(Instruction *I,
 767                                                    unsigned Level,
 768                                                    unsigned NumLevels) {
 769   // Match one level of pairwise operations.
 770   // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
 771   //       <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
 772   // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
 773   //       <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
 774   // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
 775   if (!I)
 776     return RK_None;
 777
 778   assert(I->getType()->isVectorTy() && "Expecting a vector type");
 779
 780   Optional<ReductionData> RD = getReductionData(I);
 781   if (!RD)
 782     return RK_None;
 783
 784   ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(RD->LHS);
 785   if (!LS && Level)
 786     return RK_None;
 787   ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(RD->RHS);
 788   if (!RS && Level)
 789     return RK_None;
 790
 791   // On level 0 we can omit one shufflevector instruction.
 792   if (!Level && !RS && !LS)
 793     return RK_None;
 794
 795   // Shuffle inputs must match.
 796   Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr;
 797   Value *NextLevelOpR = RS ? RS->getOperand(0) : nullptr;
 798   Value *NextLevelOp = nullptr;
 799   if (NextLevelOpR && NextLevelOpL) {
 800     // If we have two shuffles their operands must match.
 801     if (NextLevelOpL != NextLevelOpR)
 802       return RK_None;
 803
 804     NextLevelOp = NextLevelOpL;
 805   } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) {
 806     // On the first level we can omit the shufflevector <0, undef,...>. So the
 807     // input to the other shufflevector <1, undef> must match with one of the
 808     // inputs to the current binary operation.
 809     // Example:
 810     //  %NextLevelOpL = shufflevector %R, <1, undef ...>
 811     //  %BinOp        = fadd          %NextLevelOpL, %R
 812     if (NextLevelOpL && NextLevelOpL != RD->RHS)
 813       return RK_None;
 814     else if (NextLevelOpR && NextLevelOpR != RD->LHS)
 815       return RK_None;
 816
 817     NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS;
 818   } else
 819     return RK_None;
 820
 821   // Check that the next levels binary operation exists and matches with the
 822   // current one.
 823   if (Level + 1 != NumLevels) {
 824     Optional<ReductionData> NextLevelRD =
 825         getReductionData(cast<Instruction>(NextLevelOp));
 826     if (!NextLevelRD || !RD->hasSameData(*NextLevelRD))
 827       return RK_None;
 828   }
 829
 830   // Shuffle mask for pairwise operation must match.
 831   if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) {
 832     if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level))
 833       return RK_None;
 834   } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) {
 835     if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level))
 836       return RK_None;
 837   } else {
 838     return RK_None;
 839   }
 840
 841   if (++Level == NumLevels)
 842     return RD->Kind;
 843
 844   // Match next level.
 845   return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level,
 846                                        NumLevels);
 847 }
 848
 849 static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot,
 850                                             unsigned &Opcode, Type *&Ty) {
 851   if (!EnableReduxCost)
 852     return RK_None;
 853
 854   // Need to extract the first element.
 855   ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
 856   unsigned Idx = ~0u;
 857   if (CI)
 858     Idx = CI->getZExtValue();
 859   if (Idx != 0)
 860     return RK_None;
 861
 862   auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0));
 863   if (!RdxStart)
 864     return RK_None;
 865   Optional<ReductionData> RD = getReductionData(RdxStart);
 866   if (!RD)
 867     return RK_None;
 868
 869   Type *VecTy = RdxStart->getType();
 870   unsigned NumVecElems = VecTy->getVectorNumElements();
 871   if (!isPowerOf2_32(NumVecElems))
 872     return RK_None;
 873
 874   // We look for a sequence of shuffle,shuffle,add triples like the following
 875   // that builds a pairwise reduction tree.
 876   //
 877   //  (X0, X1, X2, X3)
 878   //   (X0 + X1, X2 + X3, undef, undef)
 879   //    ((X0 + X1) + (X2 + X3), undef, undef, undef)
 880   //
 881   // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef,
 882   //       <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef>
 883   // %rdx.shuf.0.1 = shufflevector <4 x float> %rdx, <4 x float> undef,
 884   //       <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
 885   // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1
 886   // %rdx.shuf.1.0 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
 887   //       <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
 888   // %rdx.shuf.1.1 = shufflevector <4 x float> %bin.rdx.0, <4 x float> undef,
 889   //       <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 890   // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1
 891   // %r = extractelement <4 x float> %bin.rdx8, i32 0
 892   if (matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)) ==
 893       RK_None)
 894     return RK_None;
 895
 896   Opcode = RD->Opcode;
 897   Ty = VecTy;
 898
 899   return RD->Kind;
 900 }
 901
 902 static std::pair<Value *, ShuffleVectorInst *>
 903 getShuffleAndOtherOprd(Value *L, Value *R) {
 904   ShuffleVectorInst *S = nullptr;
 905
 906   if ((S = dyn_cast<ShuffleVectorInst>(L)))
 907     return std::make_pair(R, S);
 908
 909   S = dyn_cast<ShuffleVectorInst>(R);
 910   return std::make_pair(L, S);
 911 }
 912
 913 static ReductionKind
 914 matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot,
 915                               unsigned &Opcode, Type *&Ty) {
 916   if (!EnableReduxCost)
 917     return RK_None;
 918
 919   // Need to extract the first element.
 920   ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1));
 921   unsigned Idx = ~0u;
 922   if (CI)
 923     Idx = CI->getZExtValue();
 924   if (Idx != 0)
 925     return RK_None;
 926
 927   auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0));
 928   if (!RdxStart)
 929     return RK_None;
 930   Optional<ReductionData> RD = getReductionData(RdxStart);
 931   if (!RD)
 932     return RK_None;
 933
 934   Type *VecTy = ReduxRoot->getOperand(0)->getType();
 935   unsigned NumVecElems = VecTy->getVectorNumElements();
 936   if (!isPowerOf2_32(NumVecElems))
 937     return RK_None;
 938
 939   // We look for a sequence of shuffles and adds like the following matching one
 940   // fadd, shuffle vector pair at a time.
 941   //
 942   // %rdx.shuf = shufflevector <4 x float> %rdx, <4 x float> undef,
 943   //                           <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 944   // %bin.rdx = fadd <4 x float> %rdx, %rdx.shuf
 945   // %rdx.shuf7 = shufflevector <4 x float> %bin.rdx, <4 x float> undef,
 946   //                          <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 947   // %bin.rdx8 = fadd <4 x float> %bin.rdx, %rdx.shuf7
 948   // %r = extractelement <4 x float> %bin.rdx8, i32 0
 949
 950   unsigned MaskStart = 1;
 951   Instruction *RdxOp = RdxStart;
 952   SmallVector<int, 32> ShuffleMask(NumVecElems, 0);
 953   unsigned NumVecElemsRemain = NumVecElems;
 954   while (NumVecElemsRemain - 1) {
 955     // Check for the right reduction operation.
 956     if (!RdxOp)
 957       return RK_None;
 958     Optional<ReductionData> RDLevel = getReductionData(RdxOp);
 959     if (!RDLevel || !RDLevel->hasSameData(*RD))
 960       return RK_None;
 961
 962     Value *NextRdxOp;
 963     ShuffleVectorInst *Shuffle;
 964     std::tie(NextRdxOp, Shuffle) =
 965         getShuffleAndOtherOprd(RDLevel->LHS, RDLevel->RHS);
 966
 967     // Check the current reduction operation and the shuffle use the same value.
 968     if (Shuffle == nullptr)
 969       return RK_None;
 970     if (Shuffle->getOperand(0) != NextRdxOp)
 971       return RK_None;
 972
 973     // Check that shuffle masks matches.
 974     for (unsigned j = 0; j != MaskStart; ++j)
 975       ShuffleMask[j] = MaskStart + j;
 976     // Fill the rest of the mask with -1 for undef.
 977     std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1);
 978
 979     SmallVector<int, 16> Mask = Shuffle->getShuffleMask();
 980     if (ShuffleMask != Mask)
 981       return RK_None;
 982
 983     RdxOp = dyn_cast<Instruction>(NextRdxOp);
 984     NumVecElemsRemain /= 2;
 985     MaskStart *= 2;
 986   }
 987
 988   Opcode = RD->Opcode;
 989   Ty = VecTy;
 990   return RD->Kind;
 991 }
 992
 993 int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const {
 994   switch (I->getOpcode()) {
 995   case Instruction::GetElementPtr:
 996     return getUserCost(I);
 997
 998   case Instruction::Ret:
 999   case Instruction::PHI:
1000   case Instruction::Br: {
1001     return getCFInstrCost(I->getOpcode());
1002   }
1003   case Instruction::Add:
1004   case Instruction::FAdd:
1005   case Instruction::Sub:
1006   case Instruction::FSub:
1007   case Instruction::Mul:
1008   case Instruction::FMul:
1009   case Instruction::UDiv:
1010   case Instruction::SDiv:
1011   case Instruction::FDiv:
1012   case Instruction::URem:
1013   case Instruction::SRem:
1014   case Instruction::FRem:
1015   case Instruction::Shl:
1016   case Instruction::LShr:
1017   case Instruction::AShr:
1018   case Instruction::And:
1019   case Instruction::Or:
1020   case Instruction::Xor: {
1021     TargetTransformInfo::OperandValueKind Op1VK, Op2VK;
1022     TargetTransformInfo::OperandValueProperties Op1VP, Op2VP;
1023     Op1VK = getOperandInfo(I->getOperand(0), Op1VP);
1024     Op2VK = getOperandInfo(I->getOperand(1), Op2VP);
1025     SmallVector<const Value *, 2> Operands(I->operand_values());
1026     return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK,
1027                                   Op1VP, Op2VP, Operands);
1028   }
1029   case Instruction::Select: {
1030     const SelectInst *SI = cast<SelectInst>(I);
1031     Type *CondTy = SI->getCondition()->getType();
1032     return getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy, I);
1033   }
1034   case Instruction::ICmp:
1035   case Instruction::FCmp: {
1036     Type *ValTy = I->getOperand(0)->getType();
1037     return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), I);
1038   }
1039   case Instruction::Store: {
1040     const StoreInst *SI = cast<StoreInst>(I);
1041     Type *ValTy = SI->getValueOperand()->getType();
1042     return getMemoryOpCost(I->getOpcode(), ValTy,
1043                                 SI->getAlignment(),
1044                                 SI->getPointerAddressSpace(), I);
1045   }
1046   case Instruction::Load: {
1047     const LoadInst *LI = cast<LoadInst>(I);
1048     return getMemoryOpCost(I->getOpcode(), I->getType(),
1049                                 LI->getAlignment(),
1050                                 LI->getPointerAddressSpace(), I);
1051   }
1052   case Instruction::ZExt:
1053   case Instruction::SExt:
1054   case Instruction::FPToUI:
1055   case Instruction::FPToSI:
1056   case Instruction::FPExt:
1057   case Instruction::PtrToInt:
1058   case Instruction::IntToPtr:
1059   case Instruction::SIToFP:
1060   case Instruction::UIToFP:
1061   case Instruction::Trunc:
1062   case Instruction::FPTrunc:
1063   case Instruction::BitCast:
1064   case Instruction::AddrSpaceCast: {
1065     Type *SrcTy = I->getOperand(0)->getType();
1066     return getCastInstrCost(I->getOpcode(), I->getType(), SrcTy, I);
1067   }
1068   case Instruction::ExtractElement: {
1069     const ExtractElementInst * EEI = cast<ExtractElementInst>(I);
1070     ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
1071     unsigned Idx = -1;
1072     if (CI)
1073       Idx = CI->getZExtValue();
1074
1075     // Try to match a reduction sequence (series of shufflevector and vector
1076     // adds followed by a extractelement).
1077     unsigned ReduxOpCode;
1078     Type *ReduxType;
1079
1080     switch (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType)) {
1081     case RK_Arithmetic:
1082       return getArithmeticReductionCost(ReduxOpCode, ReduxType,
1083                                              /*IsPairwiseForm=*/false);
1084     case RK_MinMax:
1085       return getMinMaxReductionCost(
1086           ReduxType, CmpInst::makeCmpResultType(ReduxType),
1087           /*IsPairwiseForm=*/false, /*IsUnsigned=*/false);
1088     case RK_UnsignedMinMax:
1089       return getMinMaxReductionCost(
1090           ReduxType, CmpInst::makeCmpResultType(ReduxType),
1091           /*IsPairwiseForm=*/false, /*IsUnsigned=*/true);
1092     case RK_None:
1093       break;
1094     }
1095
1096     switch (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType)) {
1097     case RK_Arithmetic:
1098       return getArithmeticReductionCost(ReduxOpCode, ReduxType,
1099                                              /*IsPairwiseForm=*/true);
1100     case RK_MinMax:
1101       return getMinMaxReductionCost(
1102           ReduxType, CmpInst::makeCmpResultType(ReduxType),
1103           /*IsPairwiseForm=*/true, /*IsUnsigned=*/false);
1104     case RK_UnsignedMinMax:
1105       return getMinMaxReductionCost(
1106           ReduxType, CmpInst::makeCmpResultType(ReduxType),
1107           /*IsPairwiseForm=*/true, /*IsUnsigned=*/true);
1108     case RK_None:
1109       break;
1110     }
1111
1112     return getVectorInstrCost(I->getOpcode(),
1113                                    EEI->getOperand(0)->getType(), Idx);
1114   }
1115   case Instruction::InsertElement: {
1116     const InsertElementInst * IE = cast<InsertElementInst>(I);
1117     ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
1118     unsigned Idx = -1;
1119     if (CI)
1120       Idx = CI->getZExtValue();
1121     return getVectorInstrCost(I->getOpcode(),
1122                                    IE->getType(), Idx);
1123   }
1124   case Instruction::ShuffleVector: {
1125     const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I);
1126     Type *Ty = Shuffle->getType();
1127     Type *SrcTy = Shuffle->getOperand(0)->getType();
1128
1129     // TODO: Identify and add costs for insert subvector, etc.
1130     int SubIndex;
1131     if (Shuffle->isExtractSubvectorMask(SubIndex))
1132       return TTIImpl->getShuffleCost(SK_ExtractSubvector, SrcTy, SubIndex, Ty);
1133
1134     if (Shuffle->changesLength())
1135       return -1;
1136
1137     if (Shuffle->isIdentity())
1138       return 0;
1139
1140     if (Shuffle->isReverse())
1141       return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr);
1142
1143     if (Shuffle->isSelect())
1144       return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr);
1145
1146     if (Shuffle->isTranspose())
1147       return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr);
1148
1149     if (Shuffle->isZeroEltSplat())
1150       return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr);
1151
1152     if (Shuffle->isSingleSource())
1153       return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr);
1154
1155     return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr);
1156   }
1157   case Instruction::Call:
1158     if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
1159       SmallVector<Value *, 4> Args(II->arg_operands());
1160
1161       FastMathFlags FMF;
1162       if (auto *FPMO = dyn_cast<FPMathOperator>(II))
1163         FMF = FPMO->getFastMathFlags();
1164
1165       return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(),
1166                                         Args, FMF);
1167     }
1168     return -1;
1169   default:
1170     // We don't have any information on this instruction.
1171     return -1;
1172   }
1173 }
1174
1175 TargetTransformInfo::Concept::~Concept() {}
1176
1177 TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1178
1179 TargetIRAnalysis::TargetIRAnalysis(
1180     std::function<Result(const Function &)> TTICallback)
1181     : TTICallback(std::move(TTICallback)) {}
1182
1183 TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
1184                                                FunctionAnalysisManager &) {
1185   return TTICallback(F);
1186 }
1187
1188 AnalysisKey TargetIRAnalysis::Key;
1189
1190 TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
1191   return Result(F.getParent()->getDataLayout());
1192 }
1193
1194 // Register the basic pass.
1195 INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1196                 "Target Transform Information", false, true)
1197 char TargetTransformInfoWrapperPass::ID = 0;
1198
1199 void TargetTransformInfoWrapperPass::anchor() {}
1200
1201 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
1202     : ImmutablePass(ID) {
1203   initializeTargetTransformInfoWrapperPassPass(
1204       *PassRegistry::getPassRegistry());
1205 }
1206
1207 TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
1208     TargetIRAnalysis TIRA)
1209     : ImmutablePass(ID), TIRA(std::move(TIRA)) {
1210   initializeTargetTransformInfoWrapperPassPass(
1211       *PassRegistry::getPassRegistry());
1212 }
1213
1214 TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
1215   FunctionAnalysisManager DummyFAM;
1216   TTI = TIRA.run(F, DummyFAM);
1217   return *TTI;
1218 }
1219
1220 ImmutablePass *
1221 llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1222   return new TargetTransformInfoWrapperPass(std::move(TIRA));
1223 }