llvm/lib/CodeGen/InterleavedAccessPass.cpp

   1 //===- InterleavedAccessPass.cpp ------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the Interleaved Access pass, which identifies
  10 // interleaved memory accesses and transforms them into target specific
  11 // intrinsics.
  12 //
  13 // An interleaved load reads data from memory into several vectors, with
  14 // DE-interleaving the data on a factor. An interleaved store writes several
  15 // vectors to memory with RE-interleaving the data on a factor.
  16 //
  17 // As interleaved accesses are difficult to identified in CodeGen (mainly
  18 // because the VECTOR_SHUFFLE DAG node is quite different from the shufflevector
  19 // IR), we identify and transform them to intrinsics in this pass so the
  20 // intrinsics can be easily matched into target specific instructions later in
  21 // CodeGen.
  22 //
  23 // E.g. An interleaved load (Factor = 2):
  24 //        %wide.vec = load <8 x i32>, <8 x i32>* %ptr
  25 //        %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6>
  26 //        %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7>
  27 //
  28 // It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2
  29 // intrinsic in ARM backend.
  30 //
  31 // In X86, this can be further optimized into a set of target
  32 // specific loads followed by an optimized sequence of shuffles.
  33 //
  34 // E.g. An interleaved store (Factor = 3):
  35 //        %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
  36 //                                    <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
  37 //        store <12 x i32> %i.vec, <12 x i32>* %ptr
  38 //
  39 // It could be transformed into a st3 intrinsic in AArch64 backend or a vst3
  40 // intrinsic in ARM backend.
  41 //
  42 // Similarly, a set of interleaved stores can be transformed into an optimized
  43 // sequence of shuffles followed by a set of target specific stores for X86.
  44 //
  45 //===----------------------------------------------------------------------===//
  46
  47 #include "llvm/ADT/ArrayRef.h"
  48 #include "llvm/ADT/DenseMap.h"
  49 #include "llvm/ADT/SetVector.h"
  50 #include "llvm/ADT/SmallVector.h"
  51 #include "llvm/CodeGen/InterleavedAccess.h"
  52 #include "llvm/CodeGen/TargetLowering.h"
  53 #include "llvm/CodeGen/TargetPassConfig.h"
  54 #include "llvm/CodeGen/TargetSubtargetInfo.h"
  55 #include "llvm/IR/Constants.h"
  56 #include "llvm/IR/Dominators.h"
  57 #include "llvm/IR/Function.h"
  58 #include "llvm/IR/IRBuilder.h"
  59 #include "llvm/IR/InstIterator.h"
  60 #include "llvm/IR/Instruction.h"
  61 #include "llvm/IR/Instructions.h"
  62 #include "llvm/IR/IntrinsicInst.h"
  63 #include "llvm/InitializePasses.h"
  64 #include "llvm/Pass.h"
  65 #include "llvm/Support/Casting.h"
  66 #include "llvm/Support/CommandLine.h"
  67 #include "llvm/Support/Debug.h"
  68 #include "llvm/Support/MathExtras.h"
  69 #include "llvm/Support/raw_ostream.h"
  70 #include "llvm/Target/TargetMachine.h"
  71 #include "llvm/Transforms/Utils/Local.h"
  72 #include <cassert>
  73 #include <utility>
  74
  75 using namespace llvm;
  76
  77 #define DEBUG_TYPE "interleaved-access"
  78
  79 static cl::opt<bool> LowerInterleavedAccesses(
  80     "lower-interleaved-accesses",
  81     cl::desc("Enable lowering interleaved accesses to intrinsics"),
  82     cl::init(true), cl::Hidden);
  83
  84 namespace {
  85
  86 class InterleavedAccessImpl {
  87   friend class InterleavedAccess;
  88
  89 public:
  90   InterleavedAccessImpl() = default;
  91   InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI)
  92       : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {}
  93   bool runOnFunction(Function &F);
  94
  95 private:
  96   DominatorTree *DT = nullptr;
  97   const TargetLowering *TLI = nullptr;
  98
  99   /// The maximum supported interleave factor.
 100   unsigned MaxFactor = 0u;
 101
 102   /// Transform an interleaved load into target specific intrinsics.
 103   bool lowerInterleavedLoad(LoadInst *LI,
 104                             SmallVector<Instruction *, 32> &DeadInsts);
 105
 106   /// Transform an interleaved store into target specific intrinsics.
 107   bool lowerInterleavedStore(StoreInst *SI,
 108                              SmallVector<Instruction *, 32> &DeadInsts);
 109
 110   /// Transform a load and a deinterleave intrinsic into target specific
 111   /// instructions.
 112   bool lowerDeinterleaveIntrinsic(IntrinsicInst *II,
 113                                   SmallVector<Instruction *, 32> &DeadInsts);
 114
 115   /// Transform an interleave intrinsic and a store into target specific
 116   /// instructions.
 117   bool lowerInterleaveIntrinsic(IntrinsicInst *II,
 118                                 SmallVector<Instruction *, 32> &DeadInsts);
 119
 120   /// Returns true if the uses of an interleaved load by the
 121   /// extractelement instructions in \p Extracts can be replaced by uses of the
 122   /// shufflevector instructions in \p Shuffles instead. If so, the necessary
 123   /// replacements are also performed.
 124   bool tryReplaceExtracts(ArrayRef<ExtractElementInst *> Extracts,
 125                           ArrayRef<ShuffleVectorInst *> Shuffles);
 126
 127   /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them
 128   /// to binop(shuffle(x), shuffle(y)) to allow the formation of an
 129   /// interleaving load. Any newly created shuffles that operate on \p LI will
 130   /// be added to \p Shuffles. Returns true, if any changes to the IR have been
 131   /// made.
 132   bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles,
 133                             SmallVectorImpl<ShuffleVectorInst *> &Shuffles,
 134                             LoadInst *LI);
 135 };
 136
 137 class InterleavedAccess : public FunctionPass {
 138   InterleavedAccessImpl Impl;
 139
 140 public:
 141   static char ID;
 142
 143   InterleavedAccess() : FunctionPass(ID) {
 144     initializeInterleavedAccessPass(*PassRegistry::getPassRegistry());
 145   }
 146
 147   StringRef getPassName() const override { return "Interleaved Access Pass"; }
 148
 149   bool runOnFunction(Function &F) override;
 150
 151   void getAnalysisUsage(AnalysisUsage &AU) const override {
 152     AU.addRequired<DominatorTreeWrapperPass>();
 153     AU.setPreservesCFG();
 154   }
 155 };
 156
 157 } // end anonymous namespace.
 158
 159 PreservedAnalyses InterleavedAccessPass::run(Function &F,
 160                                              FunctionAnalysisManager &FAM) {
 161   auto *DT = &FAM.getResult<DominatorTreeAnalysis>(F);
 162   auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering();
 163   InterleavedAccessImpl Impl(DT, TLI);
 164   bool Changed = Impl.runOnFunction(F);
 165
 166   if (!Changed)
 167     return PreservedAnalyses::all();
 168
 169   PreservedAnalyses PA;
 170   PA.preserveSet<CFGAnalyses>();
 171   return PA;
 172 }
 173
 174 char InterleavedAccess::ID = 0;
 175
 176 bool InterleavedAccess::runOnFunction(Function &F) {
 177   auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
 178   if (!TPC || !LowerInterleavedAccesses)
 179     return false;
 180
 181   LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");
 182
 183   Impl.DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
 184   auto &TM = TPC->getTM<TargetMachine>();
 185   Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering();
 186   Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor();
 187
 188   return Impl.runOnFunction(F);
 189 }
 190
 191 INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE,
 192     "Lower interleaved memory accesses to target specific intrinsics", false,
 193     false)
 194 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 195 INITIALIZE_PASS_END(InterleavedAccess, DEBUG_TYPE,
 196     "Lower interleaved memory accesses to target specific intrinsics", false,
 197     false)
 198
 199 FunctionPass *llvm::createInterleavedAccessPass() {
 200   return new InterleavedAccess();
 201 }
 202
 203 /// Check if the mask is a DE-interleave mask of the given factor
 204 /// \p Factor like:
 205 ///     <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
 206 static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor,
 207                                        unsigned &Index) {
 208   // Check all potential start indices from 0 to (Factor - 1).
 209   for (Index = 0; Index < Factor; Index++) {
 210     unsigned i = 0;
 211
 212     // Check that elements are in ascending order by Factor. Ignore undef
 213     // elements.
 214     for (; i < Mask.size(); i++)
 215       if (Mask[i] >= 0 && static_cast<unsigned>(Mask[i]) != Index + i * Factor)
 216         break;
 217
 218     if (i == Mask.size())
 219       return true;
 220   }
 221
 222   return false;
 223 }
 224
 225 /// Check if the mask is a DE-interleave mask for an interleaved load.
 226 ///
 227 /// E.g. DE-interleave masks (Factor = 2) could be:
 228 ///     <0, 2, 4, 6>    (mask of index 0 to extract even elements)
 229 ///     <1, 3, 5, 7>    (mask of index 1 to extract odd elements)
 230 static bool isDeInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
 231                                unsigned &Index, unsigned MaxFactor,
 232                                unsigned NumLoadElements) {
 233   if (Mask.size() < 2)
 234     return false;
 235
 236   // Check potential Factors.
 237   for (Factor = 2; Factor <= MaxFactor; Factor++) {
 238     // Make sure we don't produce a load wider than the input load.
 239     if (Mask.size() * Factor > NumLoadElements)
 240       return false;
 241     if (isDeInterleaveMaskOfFactor(Mask, Factor, Index))
 242       return true;
 243   }
 244
 245   return false;
 246 }
 247
 248 /// Check if the mask can be used in an interleaved store.
 249 //
 250 /// It checks for a more general pattern than the RE-interleave mask.
 251 /// I.e. <x, y, ... z, x+1, y+1, ...z+1, x+2, y+2, ...z+2, ...>
 252 /// E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35>
 253 /// E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
 254 /// E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5>
 255 ///
 256 /// The particular case of an RE-interleave mask is:
 257 /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...>
 258 /// E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7>
 259 static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor,
 260                                unsigned MaxFactor) {
 261   unsigned NumElts = SVI->getShuffleMask().size();
 262   if (NumElts < 4)
 263     return false;
 264
 265   // Check potential Factors.
 266   for (Factor = 2; Factor <= MaxFactor; Factor++) {
 267     if (SVI->isInterleave(Factor))
 268       return true;
 269   }
 270
 271   return false;
 272 }
 273
 274 bool InterleavedAccessImpl::lowerInterleavedLoad(
 275     LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
 276   if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))
 277     return false;
 278
 279   // Check if all users of this load are shufflevectors. If we encounter any
 280   // users that are extractelement instructions or binary operators, we save
 281   // them to later check if they can be modified to extract from one of the
 282   // shufflevectors instead of the load.
 283
 284   SmallVector<ShuffleVectorInst *, 4> Shuffles;
 285   SmallVector<ExtractElementInst *, 4> Extracts;
 286   // BinOpShuffles need to be handled a single time in case both operands of the
 287   // binop are the same load.
 288   SmallSetVector<ShuffleVectorInst *, 4> BinOpShuffles;
 289
 290   for (auto *User : LI->users()) {
 291     auto *Extract = dyn_cast<ExtractElementInst>(User);
 292     if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) {
 293       Extracts.push_back(Extract);
 294       continue;
 295     }
 296     if (auto *BI = dyn_cast<BinaryOperator>(User)) {
 297       if (!BI->user_empty() && all_of(BI->users(), [](auto *U) {
 298             auto *SVI = dyn_cast<ShuffleVectorInst>(U);
 299             return SVI && isa<UndefValue>(SVI->getOperand(1));
 300           })) {
 301         for (auto *SVI : BI->users())
 302           BinOpShuffles.insert(cast<ShuffleVectorInst>(SVI));
 303         continue;
 304       }
 305     }
 306     auto *SVI = dyn_cast<ShuffleVectorInst>(User);
 307     if (!SVI || !isa<UndefValue>(SVI->getOperand(1)))
 308       return false;
 309
 310     Shuffles.push_back(SVI);
 311   }
 312
 313   if (Shuffles.empty() && BinOpShuffles.empty())
 314     return false;
 315
 316   unsigned Factor, Index;
 317
 318   unsigned NumLoadElements =
 319       cast<FixedVectorType>(LI->getType())->getNumElements();
 320   auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0];
 321   // Check if the first shufflevector is DE-interleave shuffle.
 322   if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor,
 323                           NumLoadElements))
 324     return false;
 325
 326   // Holds the corresponding index for each DE-interleave shuffle.
 327   SmallVector<unsigned, 4> Indices;
 328
 329   Type *VecTy = FirstSVI->getType();
 330
 331   // Check if other shufflevectors are also DE-interleaved of the same type
 332   // and factor as the first shufflevector.
 333   for (auto *Shuffle : Shuffles) {
 334     if (Shuffle->getType() != VecTy)
 335       return false;
 336     if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
 337                                     Index))
 338       return false;
 339
 340     assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
 341     Indices.push_back(Index);
 342   }
 343   for (auto *Shuffle : BinOpShuffles) {
 344     if (Shuffle->getType() != VecTy)
 345       return false;
 346     if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
 347                                     Index))
 348       return false;
 349
 350     assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
 351
 352     if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI)
 353       Indices.push_back(Index);
 354     if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI)
 355       Indices.push_back(Index);
 356   }
 357
 358   // Try and modify users of the load that are extractelement instructions to
 359   // use the shufflevector instructions instead of the load.
 360   if (!tryReplaceExtracts(Extracts, Shuffles))
 361     return false;
 362
 363   bool BinOpShuffleChanged =
 364       replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, LI);
 365
 366   LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n");
 367
 368   // Try to create target specific intrinsics to replace the load and shuffles.
 369   if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) {
 370     // If Extracts is not empty, tryReplaceExtracts made changes earlier.
 371     return !Extracts.empty() || BinOpShuffleChanged;
 372   }
 373
 374   append_range(DeadInsts, Shuffles);
 375
 376   DeadInsts.push_back(LI);
 377   return true;
 378 }
 379
 380 bool InterleavedAccessImpl::replaceBinOpShuffles(
 381     ArrayRef<ShuffleVectorInst *> BinOpShuffles,
 382     SmallVectorImpl<ShuffleVectorInst *> &Shuffles, LoadInst *LI) {
 383   for (auto *SVI : BinOpShuffles) {
 384     BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0));
 385     Type *BIOp0Ty = BI->getOperand(0)->getType();
 386     ArrayRef<int> Mask = SVI->getShuffleMask();
 387     assert(all_of(Mask, [&](int Idx) {
 388       return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements();
 389     }));
 390
 391     auto *NewSVI1 =
 392         new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty),
 393                               Mask, SVI->getName(), SVI);
 394     auto *NewSVI2 = new ShuffleVectorInst(
 395         BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask,
 396         SVI->getName(), SVI);
 397     BinaryOperator *NewBI = BinaryOperator::CreateWithCopiedFlags(
 398         BI->getOpcode(), NewSVI1, NewSVI2, BI, BI->getName(), SVI);
 399     SVI->replaceAllUsesWith(NewBI);
 400     LLVM_DEBUG(dbgs() << "  Replaced: " << *BI << "\n    And   : " << *SVI
 401                       << "\n  With    : " << *NewSVI1 << "\n    And   : "
 402                       << *NewSVI2 << "\n    And   : " << *NewBI << "\n");
 403     RecursivelyDeleteTriviallyDeadInstructions(SVI);
 404     if (NewSVI1->getOperand(0) == LI)
 405       Shuffles.push_back(NewSVI1);
 406     if (NewSVI2->getOperand(0) == LI)
 407       Shuffles.push_back(NewSVI2);
 408   }
 409
 410   return !BinOpShuffles.empty();
 411 }
 412
 413 bool InterleavedAccessImpl::tryReplaceExtracts(
 414     ArrayRef<ExtractElementInst *> Extracts,
 415     ArrayRef<ShuffleVectorInst *> Shuffles) {
 416   // If there aren't any extractelement instructions to modify, there's nothing
 417   // to do.
 418   if (Extracts.empty())
 419     return true;
 420
 421   // Maps extractelement instructions to vector-index pairs. The extractlement
 422   // instructions will be modified to use the new vector and index operands.
 423   DenseMap<ExtractElementInst *, std::pair<Value *, int>> ReplacementMap;
 424
 425   for (auto *Extract : Extracts) {
 426     // The vector index that is extracted.
 427     auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand());
 428     auto Index = IndexOperand->getSExtValue();
 429
 430     // Look for a suitable shufflevector instruction. The goal is to modify the
 431     // extractelement instruction (which uses an interleaved load) to use one
 432     // of the shufflevector instructions instead of the load.
 433     for (auto *Shuffle : Shuffles) {
 434       // If the shufflevector instruction doesn't dominate the extract, we
 435       // can't create a use of it.
 436       if (!DT->dominates(Shuffle, Extract))
 437         continue;
 438
 439       // Inspect the indices of the shufflevector instruction. If the shuffle
 440       // selects the same index that is extracted, we can modify the
 441       // extractelement instruction.
 442       SmallVector<int, 4> Indices;
 443       Shuffle->getShuffleMask(Indices);
 444       for (unsigned I = 0; I < Indices.size(); ++I)
 445         if (Indices[I] == Index) {
 446           assert(Extract->getOperand(0) == Shuffle->getOperand(0) &&
 447                  "Vector operations do not match");
 448           ReplacementMap[Extract] = std::make_pair(Shuffle, I);
 449           break;
 450         }
 451
 452       // If we found a suitable shufflevector instruction, stop looking.
 453       if (ReplacementMap.count(Extract))
 454         break;
 455     }
 456
 457     // If we did not find a suitable shufflevector instruction, the
 458     // extractelement instruction cannot be modified, so we must give up.
 459     if (!ReplacementMap.count(Extract))
 460       return false;
 461   }
 462
 463   // Finally, perform the replacements.
 464   IRBuilder<> Builder(Extracts[0]->getContext());
 465   for (auto &Replacement : ReplacementMap) {
 466     auto *Extract = Replacement.first;
 467     auto *Vector = Replacement.second.first;
 468     auto Index = Replacement.second.second;
 469     Builder.SetInsertPoint(Extract);
 470     Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index));
 471     Extract->eraseFromParent();
 472   }
 473
 474   return true;
 475 }
 476
 477 bool InterleavedAccessImpl::lowerInterleavedStore(
 478     StoreInst *SI, SmallVector<Instruction *, 32> &DeadInsts) {
 479   if (!SI->isSimple())
 480     return false;
 481
 482   auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
 483   if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType()))
 484     return false;
 485
 486   // Check if the shufflevector is RE-interleave shuffle.
 487   unsigned Factor;
 488   if (!isReInterleaveMask(SVI, Factor, MaxFactor))
 489     return false;
 490
 491   LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n");
 492
 493   // Try to create target specific intrinsics to replace the store and shuffle.
 494   if (!TLI->lowerInterleavedStore(SI, SVI, Factor))
 495     return false;
 496
 497   // Already have a new target specific interleaved store. Erase the old store.
 498   DeadInsts.push_back(SI);
 499   DeadInsts.push_back(SVI);
 500   return true;
 501 }
 502
 503 bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
 504     IntrinsicInst *DI, SmallVector<Instruction *, 32> &DeadInsts) {
 505   LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0));
 506
 507   if (!LI || !LI->hasOneUse() || !LI->isSimple())
 508     return false;
 509
 510   LLVM_DEBUG(dbgs() << "IA: Found a deinterleave intrinsic: " << *DI << "\n");
 511
 512   // Try and match this with target specific intrinsics.
 513   if (!TLI->lowerDeinterleaveIntrinsicToLoad(DI, LI))
 514     return false;
 515
 516   // We now have a target-specific load, so delete the old one.
 517   DeadInsts.push_back(DI);
 518   DeadInsts.push_back(LI);
 519   return true;
 520 }
 521
 522 bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
 523     IntrinsicInst *II, SmallVector<Instruction *, 32> &DeadInsts) {
 524   if (!II->hasOneUse())
 525     return false;
 526
 527   StoreInst *SI = dyn_cast<StoreInst>(*(II->users().begin()));
 528
 529   if (!SI || !SI->isSimple())
 530     return false;
 531
 532   LLVM_DEBUG(dbgs() << "IA: Found an interleave intrinsic: " << *II << "\n");
 533
 534   // Try and match this with target specific intrinsics.
 535   if (!TLI->lowerInterleaveIntrinsicToStore(II, SI))
 536     return false;
 537
 538   // We now have a target-specific store, so delete the old one.
 539   DeadInsts.push_back(SI);
 540   DeadInsts.push_back(II);
 541   return true;
 542 }
 543
 544 bool InterleavedAccessImpl::runOnFunction(Function &F) {
 545   // Holds dead instructions that will be erased later.
 546   SmallVector<Instruction *, 32> DeadInsts;
 547   bool Changed = false;
 548
 549   for (auto &I : instructions(F)) {
 550     if (auto *LI = dyn_cast<LoadInst>(&I))
 551       Changed |= lowerInterleavedLoad(LI, DeadInsts);
 552
 553     if (auto *SI = dyn_cast<StoreInst>(&I))
 554       Changed |= lowerInterleavedStore(SI, DeadInsts);
 555
 556     if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
 557       // At present, we only have intrinsics to represent (de)interleaving
 558       // with a factor of 2.
 559       if (II->getIntrinsicID() == Intrinsic::experimental_vector_deinterleave2)
 560         Changed |= lowerDeinterleaveIntrinsic(II, DeadInsts);
 561       if (II->getIntrinsicID() == Intrinsic::experimental_vector_interleave2)
 562         Changed |= lowerInterleaveIntrinsic(II, DeadInsts);
 563     }
 564   }
 565
 566   for (auto *I : DeadInsts)
 567     I->eraseFromParent();
 568
 569   return Changed;
 570 }