unittests/Transforms/Vectorize/VPlanSlpTest.cpp

   1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "../lib/Transforms/Vectorize/VPlan.h"
  10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
  11 #include "../lib/Transforms/Vectorize/VPlanHCFGTransforms.h"
  12 #include "VPlanTestBase.h"
  13 #include "llvm/Analysis/VectorUtils.h"
  14 #include "gtest/gtest.h"
  15
  16 namespace llvm {
  17 namespace {
  18
  19 class VPlanSlpTest : public VPlanTestBase {
  20 protected:
  21   TargetLibraryInfoImpl TLII;
  22   TargetLibraryInfo TLI;
  23   DataLayout DL;
  24
  25   std::unique_ptr<AssumptionCache> AC;
  26   std::unique_ptr<ScalarEvolution> SE;
  27   std::unique_ptr<AAResults> AARes;
  28   std::unique_ptr<BasicAAResult> BasicAA;
  29   std::unique_ptr<LoopAccessInfo> LAI;
  30   std::unique_ptr<PredicatedScalarEvolution> PSE;
  31   std::unique_ptr<InterleavedAccessInfo> IAI;
  32
  33   VPlanSlpTest()
  34       : TLII(), TLI(TLII),
  35         DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
  36            "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:"
  37            "16:32:64-S128") {}
  38
  39   VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L,
  40                                                    VPlan &Plan) {
  41     AC.reset(new AssumptionCache(F));
  42     SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
  43     BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT, &*LI));
  44     AARes.reset(new AAResults(TLI));
  45     AARes->addAAResult(*BasicAA);
  46     PSE.reset(new PredicatedScalarEvolution(*SE, *L));
  47     LAI.reset(new LoopAccessInfo(L, &*SE, &TLI, &*AARes, &*DT, &*LI));
  48     IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI));
  49     IAI->analyzeInterleaving(false);
  50     return {Plan, *IAI};
  51   }
  52 };
  53
  54 TEST_F(VPlanSlpTest, testSlpSimple_2) {
  55   const char *ModuleString =
  56       "%struct.Test = type { i32, i32 }\n"
  57       "%struct.Test3 = type { i32, i32, i32 }\n"
  58       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
  59       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
  60       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
  61       "entry:\n"
  62       "  br label %for.body\n"
  63       "for.body:                                         ; preds = %for.body, "
  64       "%entry\n"
  65       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
  66       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
  67       "%indvars.iv, i32 0\n"
  68       "  %vA0 = load i32, i32* %A0, align 4\n"
  69       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
  70       "%indvars.iv, i32 0\n"
  71       "  %vB0 = load i32, i32* %B0, align 4\n"
  72       "  %add0 = add nsw i32 %vA0, %vB0\n"
  73       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
  74       "%indvars.iv, i32 1\n"
  75       "  %vA1 = load i32, i32* %A1, align 4\n"
  76       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
  77       "%indvars.iv, i32 1\n"
  78       "  %vB1 = load i32, i32* %B1, align 4\n"
  79       "  %add1 = add nsw i32 %vA1, %vB1\n"
  80       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
  81       "%indvars.iv, i32 0\n"
  82       "  store i32 %add0, i32* %C0, align 4\n"
  83       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
  84       "%indvars.iv, i32 1\n"
  85       "  store i32 %add1, i32* %C1, align 4\n"
  86       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
  87       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
  88       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
  89       "for.cond.cleanup:                                 ; preds = %for.body\n"
  90       "  ret void\n"
  91       "}\n";
  92
  93   Module &M = parseModule(ModuleString);
  94
  95   Function *F = M.getFunction("add_x2");
  96   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
  97   auto Plan = buildHCFG(LoopHeader);
  98   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
  99
 100   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 101   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 102   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 103
 104   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 105   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 106
 107   VPlanSlp Slp(VPIAI, *Body);
 108   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 109   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 110   EXPECT_EQ(64u, Slp.getWidestBundleBits());
 111   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 112
 113   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
 114   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 115
 116   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
 117   auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
 118   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 119   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 120 }
 121
 122 TEST_F(VPlanSlpTest, testSlpSimple_3) {
 123   const char *ModuleString =
 124       "%struct.Test = type { i32, i32 }\n"
 125       "%struct.Test3 = type { i32, i32, i32 }\n"
 126       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 127       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 128       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 129       "entry:\n"
 130       "  br label %for.body\n"
 131       "for.body:                                         ; preds = %for.body, "
 132       "%entry\n"
 133       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 134       "  %A0 = getelementptr %struct.Test, %struct.Test* %A, i64 "
 135       "                      %indvars.iv, i32 0\n"
 136       "  %vA0 = load i32, i32* %A0, align 4\n"
 137       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 138       "                      %indvars.iv, i32 0\n"
 139       "  %vB0 = load i32, i32* %B0, align 4\n"
 140       "  %add0 = add nsw i32 %vA0, %vB0\n"
 141       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 142       "                      %indvars.iv, i32 1\n"
 143       "  %vA1 = load i32, i32* %A1, align 4\n"
 144       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 145       "                      %indvars.iv, i32 1\n"
 146       "  %vB1 = load i32, i32* %B1, align 4\n"
 147       "  %add1 = add nsw i32 %vA1, %vB1\n"
 148       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 149       "                      %indvars.iv, i32 0\n"
 150       "  store i32 %add0, i32* %C0, align 4\n"
 151       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 152       "                      %indvars.iv, i32 1\n"
 153       "  store i32 %add1, i32* %C1, align 4\n"
 154       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 155       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 156       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 157       "for.cond.cleanup:                                 ; preds = %for.body\n"
 158       "  ret void\n"
 159       "}\n";
 160
 161   Module &M = parseModule(ModuleString);
 162
 163   Function *F = M.getFunction("add_x2");
 164   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 165   auto Plan = buildHCFG(LoopHeader);
 166
 167   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 168   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 169   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 170
 171   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 172   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 173
 174   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 175
 176   VPlanSlp Slp(VPIAI, *Body);
 177   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 178   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 179   EXPECT_EQ(64u, Slp.getWidestBundleBits());
 180   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 181
 182   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
 183   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 184
 185   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
 186   auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
 187   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 188   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 189
 190   VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1));
 191   VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3));
 192   EXPECT_EQ(GetA, CombinedLoadA->getOperand(0));
 193   EXPECT_EQ(GetB, CombinedLoadB->getOperand(0));
 194 }
 195
 196 TEST_F(VPlanSlpTest, testSlpReuse_1) {
 197   const char *ModuleString =
 198       "%struct.Test = type { i32, i32 }\n"
 199       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 200       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 201       "entry:\n"
 202       "  br label %for.body\n"
 203       "for.body:                                         ; preds = %for.body, "
 204       "%entry\n"
 205       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 206       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 207       "%indvars.iv, i32 0\n"
 208       "  %vA0 = load i32, i32* %A0, align 4\n"
 209       "  %add0 = add nsw i32 %vA0, %vA0\n"
 210       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 211       "%indvars.iv, i32 1\n"
 212       "  %vA1 = load i32, i32* %A1, align 4\n"
 213       "  %add1 = add nsw i32 %vA1, %vA1\n"
 214       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 215       "%indvars.iv, i32 0\n"
 216       "  store i32 %add0, i32* %C0, align 4\n"
 217       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 218       "%indvars.iv, i32 1\n"
 219       "  store i32 %add1, i32* %C1, align 4\n"
 220       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 221       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 222       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 223       "for.cond.cleanup:                                 ; preds = %for.body\n"
 224       "  ret void\n"
 225       "}\n";
 226
 227   Module &M = parseModule(ModuleString);
 228
 229   Function *F = M.getFunction("add_x2");
 230   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 231   auto Plan = buildHCFG(LoopHeader);
 232   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 233
 234   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 235   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 236   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 237
 238   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8));
 239   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
 240
 241   VPlanSlp Slp(VPIAI, *Body);
 242   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 243   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 244   EXPECT_EQ(64u, Slp.getWidestBundleBits());
 245   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 246
 247   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
 248   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 249
 250   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
 251   EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1));
 252   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 253 }
 254
 255 TEST_F(VPlanSlpTest, testSlpReuse_2) {
 256   const char *ModuleString =
 257       "%struct.Test = type { i32, i32 }\n"
 258       "define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 259       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 260       "entry:\n"
 261       "  br label %for.body\n"
 262       "for.body:                                         ; preds = %for.body, "
 263       "%entry\n"
 264       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 265       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 266       "%indvars.iv, i32 0\n"
 267       "  %vA0 = load i32, i32* %A0, align 4\n"
 268       "  %add0 = add nsw i32 %vA0, %vA0\n"
 269       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 270       "%indvars.iv, i32 0\n"
 271       "  store i32 %add0, i32* %C0, align 4\n"
 272       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 273       "%indvars.iv, i32 1\n"
 274       "  %vA1 = load i32, i32* %A1, align 4\n"
 275       "  %add1 = add nsw i32 %vA1, %vA1\n"
 276       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 277       "%indvars.iv, i32 1\n"
 278       "  store i32 %add1, i32* %C1, align 4\n"
 279       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 280       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 281       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 282       "for.cond.cleanup:                                 ; preds = %for.body\n"
 283       "  ret i32 %vA1\n"
 284       "}\n";
 285
 286   Module &M = parseModule(ModuleString);
 287
 288   Function *F = M.getFunction("add_x2");
 289   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 290   auto Plan = buildHCFG(LoopHeader);
 291   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 292
 293   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 294   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 295   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 296
 297   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5));
 298   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
 299
 300   VPlanSlp Slp(VPIAI, *Body);
 301   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 302   Slp.buildGraph(StoreRoot);
 303   EXPECT_FALSE(Slp.isCompletelySLP());
 304 }
 305
 306 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2,
 307                                 VPBasicBlock *Body,
 308                                 VPInterleavedAccessInfo &&IAI) {
 309   VPlanSlp Slp(IAI, *Body);
 310   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 311   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 312
 313   EXPECT_TRUE(Slp.isCompletelySLP());
 314   EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore);
 315
 316   VPInstruction *CombinedAdd =
 317       cast<VPInstruction>(CombinedStore->getOperand(0));
 318   EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add);
 319
 320   VPInstruction *CombinedMulAB =
 321       cast<VPInstruction>(CombinedAdd->getOperand(0));
 322   VPInstruction *CombinedMulCD =
 323       cast<VPInstruction>(CombinedAdd->getOperand(1));
 324   EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul);
 325
 326   VPInstruction *CombinedLoadA =
 327       cast<VPInstruction>(CombinedMulAB->getOperand(0));
 328   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 329   VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2));
 330   VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 331   EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0));
 332   EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1));
 333
 334   VPInstruction *CombinedLoadB =
 335       cast<VPInstruction>(CombinedMulAB->getOperand(1));
 336   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 337   VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4));
 338   VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 339   EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0));
 340   EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1));
 341
 342   EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul);
 343
 344   VPInstruction *CombinedLoadC =
 345       cast<VPInstruction>(CombinedMulCD->getOperand(0));
 346   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode());
 347   VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7));
 348   VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17));
 349   EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0));
 350   EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1));
 351
 352   VPInstruction *CombinedLoadD =
 353       cast<VPInstruction>(CombinedMulCD->getOperand(1));
 354   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode());
 355   VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9));
 356   VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19));
 357   EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0));
 358   EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1));
 359 }
 360
 361 TEST_F(VPlanSlpTest, testSlpReorder_1) {
 362   LLVMContext Ctx;
 363   const char *ModuleString =
 364       "%struct.Test = type { i32, i32 }\n"
 365       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
 366       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
 367       "entry:\n"
 368       "  br label %for.body\n"
 369       "for.body:                                         ; preds = %for.body, "
 370       "%entry\n"
 371       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 372       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 373       "%indvars.iv, i32 0\n"
 374       "  %vA0 = load i32, i32* %A0, align 4\n"
 375       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 376       "%indvars.iv, i32 0\n"
 377       "  %vB0 = load i32, i32* %B0, align 4\n"
 378       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
 379       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 380       "%indvars.iv, i32 0\n"
 381       "  %vC0 = load i32, i32* %C0, align 4\n"
 382       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 383       "%indvars.iv, i32 0\n"
 384       "  %vD0 = load i32, i32* %D0, align 4\n"
 385       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 386       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 387       "%indvars.iv, i32 1\n"
 388       "  %vA1 = load i32, i32* %A1, align 4\n"
 389       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 390       "%indvars.iv, i32 1\n"
 391       "  %vB1 = load i32, i32* %B1, align 4\n"
 392       "  %mul21 = mul nsw i32 %vA1, %vB1\n"
 393       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 394       "%indvars.iv, i32 1\n"
 395       "  %vC1 = load i32, i32* %C1, align 4\n"
 396       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 397       "%indvars.iv, i32 1\n"
 398       "  %vD1 = load i32, i32* %D1, align 4\n"
 399       "  %mul22 = mul nsw i32 %vC1, %vD1\n"
 400       "  %add1 = add nsw i32 %mul11, %mul12\n"
 401       "  %add2 = add nsw i32 %mul22, %mul21\n"
 402       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 403       "%indvars.iv, i32 0\n"
 404       "  store i32 %add1, i32* %E0, align 4\n"
 405       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 406       "%indvars.iv, i32 1\n"
 407       "  store i32 %add2, i32* %E1, align 4\n"
 408       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 409       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 410       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 411       "for.cond.cleanup:                                 ; preds = %for.body\n"
 412       "  ret void\n"
 413       "}\n";
 414
 415   Module &M = parseModule(ModuleString);
 416
 417   Function *F = M.getFunction("add_x3");
 418   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 419   auto Plan = buildHCFG(LoopHeader);
 420
 421   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 422   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 423   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 424
 425   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 426   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 427
 428   checkReorderExample(
 429       Store1, Store2, Body,
 430       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
 431 }
 432
 433 TEST_F(VPlanSlpTest, testSlpReorder_2) {
 434   LLVMContext Ctx;
 435   const char *ModuleString =
 436       "%struct.Test = type { i32, i32 }\n"
 437       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
 438       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
 439       "entry:\n"
 440       "  br label %for.body\n"
 441       "for.body:                                         ; preds = %for.body, "
 442       "%entry\n"
 443       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 444       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 445       "%indvars.iv, i32 0\n"
 446       "  %vA0 = load i32, i32* %A0, align 4\n"
 447       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 448       "%indvars.iv, i32 0\n"
 449       "  %vB0 = load i32, i32* %B0, align 4\n"
 450       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
 451       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 452       "%indvars.iv, i32 0\n"
 453       "  %vC0 = load i32, i32* %C0, align 4\n"
 454       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 455       "%indvars.iv, i32 0\n"
 456       "  %vD0 = load i32, i32* %D0, align 4\n"
 457       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 458       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 459       "%indvars.iv, i32 1\n"
 460       "  %vA1 = load i32, i32* %A1, align 4\n"
 461       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 462       "%indvars.iv, i32 1\n"
 463       "  %vB1 = load i32, i32* %B1, align 4\n"
 464       "  %mul21 = mul nsw i32 %vB1, %vA1\n"
 465       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 466       "%indvars.iv, i32 1\n"
 467       "  %vC1 = load i32, i32* %C1, align 4\n"
 468       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 469       "%indvars.iv, i32 1\n"
 470       "  %vD1 = load i32, i32* %D1, align 4\n"
 471       "  %mul22 = mul nsw i32 %vD1, %vC1\n"
 472       "  %add1 = add nsw i32 %mul11, %mul12\n"
 473       "  %add2 = add nsw i32 %mul22, %mul21\n"
 474       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 475       "%indvars.iv, i32 0\n"
 476       "  store i32 %add1, i32* %E0, align 4\n"
 477       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 478       "%indvars.iv, i32 1\n"
 479       "  store i32 %add2, i32* %E1, align 4\n"
 480       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 481       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 482       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 483       "for.cond.cleanup:                                 ; preds = %for.body\n"
 484       "  ret void\n"
 485       "}\n";
 486
 487   Module &M = parseModule(ModuleString);
 488
 489   Function *F = M.getFunction("add_x3");
 490   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 491   auto Plan = buildHCFG(LoopHeader);
 492
 493   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 494   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 495   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 496
 497   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 498   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 499
 500   checkReorderExample(
 501       Store1, Store2, Body,
 502       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
 503 }
 504
 505 TEST_F(VPlanSlpTest, testSlpReorder_3) {
 506   LLVMContext Ctx;
 507   const char *ModuleString =
 508       "%struct.Test = type { i32, i32 }\n"
 509       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
 510       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
 511       "entry:\n"
 512       "  br label %for.body\n"
 513       "for.body:                                         ; preds = %for.body, "
 514       "%entry\n"
 515       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 516       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 517       "%indvars.iv, i32 1\n"
 518       "  %vA1 = load i32, i32* %A1, align 4\n"
 519       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 520       "%indvars.iv, i32 0\n"
 521       "  %vB0 = load i32, i32* %B0, align 4\n"
 522       "  %mul11 = mul nsw i32 %vA1, %vB0\n"
 523       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 524       "%indvars.iv, i32 0\n"
 525       "  %vC0 = load i32, i32* %C0, align 4\n"
 526       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 527       "%indvars.iv, i32 0\n"
 528       "  %vD0 = load i32, i32* %D0, align 4\n"
 529       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 530       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 531       "%indvars.iv, i32 0\n"
 532       "  %vA0 = load i32, i32* %A0, align 4\n"
 533       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 534       "%indvars.iv, i32 1\n"
 535       "  %vB1 = load i32, i32* %B1, align 4\n"
 536       "  %mul21 = mul nsw i32 %vB1, %vA0\n"
 537       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 538       "%indvars.iv, i32 1\n"
 539       "  %vC1 = load i32, i32* %C1, align 4\n"
 540       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 541       "%indvars.iv, i32 1\n"
 542       "  %vD1 = load i32, i32* %D1, align 4\n"
 543       "  %mul22 = mul nsw i32 %vD1, %vC1\n"
 544       "  %add1 = add nsw i32 %mul11, %mul12\n"
 545       "  %add2 = add nsw i32 %mul22, %mul21\n"
 546       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 547       "%indvars.iv, i32 0\n"
 548       "  store i32 %add1, i32* %E0, align 4\n"
 549       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 550       "%indvars.iv, i32 1\n"
 551       "  store i32 %add2, i32* %E1, align 4\n"
 552       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 553       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 554       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 555       "for.cond.cleanup:                                 ; preds = %for.body\n"
 556       "  ret void\n"
 557       "}\n";
 558
 559   Module &M = parseModule(ModuleString);
 560
 561   Function *F = M.getFunction("add_x3");
 562   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 563   auto Plan = buildHCFG(LoopHeader);
 564
 565   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 566   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 567   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 568
 569   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 570   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 571
 572   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 573   VPlanSlp Slp(VPIAI, *Body);
 574   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 575   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 576
 577   // FIXME Need to select better first value for lane0.
 578   EXPECT_FALSE(Slp.isCompletelySLP());
 579 }
 580
 581 TEST_F(VPlanSlpTest, testSlpReorder_4) {
 582   LLVMContext Ctx;
 583   const char *ModuleString =
 584       "%struct.Test = type { i32, i32 }\n"
 585       "define void @add_x3(%struct.Test* %A, %struct.Test* %B, %struct.Test* "
 586       "%C,  %struct.Test* %D,  %struct.Test* %E)  {\n"
 587       "entry:\n"
 588       "  br label %for.body\n"
 589       "for.body:                                         ; preds = %for.body, "
 590       "%entry\n"
 591       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 592       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 593       "%indvars.iv, i32 0\n"
 594       "  %vA0 = load i32, i32* %A0, align 4\n"
 595       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 596       "%indvars.iv, i32 0\n"
 597       "  %vB0 = load i32, i32* %B0, align 4\n"
 598       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
 599       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 600       "%indvars.iv, i32 0\n"
 601       "  %vC0 = load i32, i32* %C0, align 4\n"
 602       "  %D0 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 603       "%indvars.iv, i32 0\n"
 604       "  %vD0 = load i32, i32* %D0, align 4\n"
 605       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 606       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 607       "%indvars.iv, i32 1\n"
 608       "  %vA1 = load i32, i32* %A1, align 4\n"
 609       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 610       "%indvars.iv, i32 1\n"
 611       "  %vB1 = load i32, i32* %B1, align 4\n"
 612       "  %mul21 = mul nsw i32 %vA1, %vB1\n"
 613       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 614       "%indvars.iv, i32 1\n"
 615       "  %vC1 = load i32, i32* %C1, align 4\n"
 616       "  %D1 = getelementptr inbounds %struct.Test, %struct.Test* %D, i64 "
 617       "%indvars.iv, i32 1\n"
 618       "  %vD1 = load i32, i32* %D1, align 4\n"
 619       "  %mul22 = mul nsw i32 %vC1, %vD1\n"
 620       "  %add1 = add nsw i32 %mul11, %mul12\n"
 621       "  %add2 = add nsw i32 %mul22, %mul21\n"
 622       "  %E0 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 623       "%indvars.iv, i32 0\n"
 624       "  store i32 %add1, i32* %E0, align 4\n"
 625       "  %E1 = getelementptr inbounds %struct.Test, %struct.Test* %E, i64 "
 626       "%indvars.iv, i32 1\n"
 627       "  store i32 %add2, i32* %E1, align 4\n"
 628       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 629       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 630       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 631       "for.cond.cleanup:                                 ; preds = %for.body\n"
 632       "  ret void\n"
 633       "}\n";
 634
 635   Module &M = parseModule(ModuleString);
 636
 637   Function *F = M.getFunction("add_x3");
 638   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 639   auto Plan = buildHCFG(LoopHeader);
 640
 641   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 642   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 643   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 644
 645   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 646   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 647
 648   checkReorderExample(
 649       Store1, Store2, Body,
 650       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
 651 }
 652
 653 // Make sure we do not combine instructions with operands in different BBs.
 654 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) {
 655   const char *ModuleString =
 656       "%struct.Test = type { i32, i32 }\n"
 657       "%struct.Test3 = type { i32, i32, i32 }\n"
 658       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 659       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 660       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 661       "entry:\n"
 662       "  br label %for.body\n"
 663       "for.body:                                         ; preds = %for.body, "
 664       "%entry\n"
 665       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 666       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 667       "%indvars.iv, i32 0\n"
 668       "  %vA0 = load i32, i32* %A0, align 4\n"
 669       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 670       "%indvars.iv, i32 0\n"
 671       "  %vB0 = load i32, i32* %B0, align 4\n"
 672       "  %add0 = add nsw i32 %vA0, %vB0\n"
 673       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 674       "%indvars.iv, i32 1\n"
 675       "  %vA1 = load i32, i32* %A1, align 4\n"
 676       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 677       "%indvars.iv, i32 1\n"
 678       "  br label %bb2\n"
 679       "bb2:\n"
 680       "  %vB1 = load i32, i32* %B1, align 4\n"
 681       "  %add1 = add nsw i32 %vA1, %vB1\n"
 682       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 683       "%indvars.iv, i32 0\n"
 684       "  store i32 %add0, i32* %C0, align 4\n"
 685       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 686       "%indvars.iv, i32 1\n"
 687       "  store i32 %add1, i32* %C1, align 4\n"
 688       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 689       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 690       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 691       "for.cond.cleanup:                                 ; preds = %for.body\n"
 692       "  ret void\n"
 693       "}\n";
 694
 695   Module &M = parseModule(ModuleString);
 696
 697   Function *F = M.getFunction("add_x2");
 698   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 699   auto Plan = buildHCFG(LoopHeader);
 700   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 701
 702   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 703   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 704   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 705   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 706
 707   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
 708   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5));
 709
 710   VPlanSlp Slp(VPIAI, *BB2);
 711   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 712   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 713   EXPECT_EQ(0u, Slp.getWidestBundleBits());
 714 }
 715
 716 // Make sure we do not combine instructions with operands in different BBs.
 717 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
 718   const char *ModuleString =
 719       "%struct.Test = type { i32, i32 }\n"
 720       "%struct.Test3 = type { i32, i32, i32 }\n"
 721       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 722       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 723       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 724       "entry:\n"
 725       "  br label %for.body\n"
 726       "for.body:                                         ; preds = %for.body, "
 727       "%entry\n"
 728       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 729       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 730       "%indvars.iv, i32 0\n"
 731       "  %vA0 = load i32, i32* %A0, align 4\n"
 732       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 733       "%indvars.iv, i32 0\n"
 734       "  %vB0 = load i32, i32* %B0, align 4\n"
 735       "  %add0 = add nsw i32 %vA0, %vB0\n"
 736       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 737       "%indvars.iv, i32 1\n"
 738       "  %vA1 = load i32, i32* %A1, align 4\n"
 739       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 740       "%indvars.iv, i32 1\n"
 741       "  %vB1 = load i32, i32* %B1, align 4\n"
 742       "  %add1 = add nsw i32 %vA1, %vB1\n"
 743       "  br label %bb2\n"
 744       "bb2:\n"
 745       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 746       "%indvars.iv, i32 0\n"
 747       "  store i32 %add0, i32* %C0, align 4\n"
 748       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 749       "%indvars.iv, i32 1\n"
 750       "  store i32 %add1, i32* %C1, align 4\n"
 751       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 752       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 753       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 754       "for.cond.cleanup:                                 ; preds = %for.body\n"
 755       "  ret void\n"
 756       "}\n";
 757
 758   Module &M = parseModule(ModuleString);
 759
 760   Function *F = M.getFunction("add_x2");
 761   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 762   auto Plan = buildHCFG(LoopHeader);
 763   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 764
 765   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 766   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 767   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 768   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 769
 770   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1));
 771   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
 772
 773   VPlanSlp Slp(VPIAI, *BB2);
 774   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 775   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 776   EXPECT_EQ(0u, Slp.getWidestBundleBits());
 777 }
 778
 779 TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
 780   const char *ModuleString =
 781       "%struct.Test = type { i32, i32 }\n"
 782       "%struct.Test3 = type { i32, i32, i32 }\n"
 783       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 784       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 785       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 786       "entry:\n"
 787       "  br label %for.body\n"
 788       "for.body:                                         ; preds = %for.body, "
 789       "%entry\n"
 790       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 791       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 792       "%indvars.iv, i32 0\n"
 793       "  %vA0 = load atomic i32, i32* %A0 monotonic, align 4\n"
 794       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 795       "%indvars.iv, i32 0\n"
 796       "  %vB0 = load i32, i32* %B0, align 4\n"
 797       "  %add0 = add nsw i32 %vA0, %vB0\n"
 798       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 799       "%indvars.iv, i32 1\n"
 800       "  %vA1 = load i32, i32* %A1, align 4\n"
 801       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 802       "%indvars.iv, i32 1\n"
 803       "  %vB1 = load i32, i32* %B1, align 4\n"
 804       "  %add1 = add nsw i32 %vA1, %vB1\n"
 805       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 806       "%indvars.iv, i32 0\n"
 807       "  store i32 %add0, i32* %C0, align 4\n"
 808       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 809       "%indvars.iv, i32 1\n"
 810       "  store i32 %add1, i32* %C1, align 4\n"
 811       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 812       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 813       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 814       "for.cond.cleanup:                                 ; preds = %for.body\n"
 815       "  ret void\n"
 816       "}\n";
 817
 818   Module &M = parseModule(ModuleString);
 819
 820   Function *F = M.getFunction("add_x2");
 821   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 822   auto Plan = buildHCFG(LoopHeader);
 823   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 824
 825   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 826   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 827   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 828
 829   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 830   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 831
 832   VPlanSlp Slp(VPIAI, *Body);
 833   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 834   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 835   EXPECT_FALSE(Slp.isCompletelySLP());
 836 }
 837
 838 TEST_F(VPlanSlpTest, testSlpAtomicStore) {
 839   const char *ModuleString =
 840       "%struct.Test = type { i32, i32 }\n"
 841       "%struct.Test3 = type { i32, i32, i32 }\n"
 842       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 843       "define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
 844       "nocapture readonly %B, %struct.Test* nocapture %C)  {\n"
 845       "entry:\n"
 846       "  br label %for.body\n"
 847       "for.body:                                         ; preds = %for.body, "
 848       "%entry\n"
 849       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 850       "  %A0 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 851       "%indvars.iv, i32 0\n"
 852       "  %vA0 = load i32, i32* %A0, align 4\n"
 853       "  %B0 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 854       "%indvars.iv, i32 0\n"
 855       "  %vB0 = load i32, i32* %B0, align 4\n"
 856       "  %add0 = add nsw i32 %vA0, %vB0\n"
 857       "  %A1 = getelementptr inbounds %struct.Test, %struct.Test* %A, i64 "
 858       "%indvars.iv, i32 1\n"
 859       "  %vA1 = load i32, i32* %A1, align 4\n"
 860       "  %B1 = getelementptr inbounds %struct.Test, %struct.Test* %B, i64 "
 861       "%indvars.iv, i32 1\n"
 862       "  %vB1 = load i32, i32* %B1, align 4\n"
 863       "  %add1 = add nsw i32 %vA1, %vB1\n"
 864       "  %C0 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 865       "%indvars.iv, i32 0\n"
 866       "  store atomic i32 %add0, i32* %C0 monotonic, align 4\n"
 867       "  %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
 868       "%indvars.iv, i32 1\n"
 869       "  store i32 %add1, i32* %C1, align 4\n"
 870       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 871       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 872       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 873       "for.cond.cleanup:                                 ; preds = %for.body\n"
 874       "  ret void\n"
 875       "}\n";
 876
 877   Module &M = parseModule(ModuleString);
 878
 879   Function *F = M.getFunction("add_x2");
 880   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 881   auto Plan = buildHCFG(LoopHeader);
 882   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 883
 884   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 885   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 886   VPBasicBlock *Body = Entry->getSingleSuccessor()->getEntryBasicBlock();
 887
 888   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 889   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 890
 891   VPlanSlp Slp(VPIAI, *Body);
 892   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 893   Slp.buildGraph(StoreRoot);
 894   EXPECT_FALSE(Slp.isCompletelySLP());
 895 }
 896
 897 } // namespace
 898 } // namespace llvm