llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp

   1 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "../lib/Transforms/Vectorize/VPlan.h"
  10 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
  11 #include "VPlanTestBase.h"
  12 #include "llvm/Analysis/TargetLibraryInfo.h"
  13 #include "llvm/Analysis/VectorUtils.h"
  14 #include "gtest/gtest.h"
  15
  16 namespace llvm {
  17 namespace {
  18
  19 class VPlanSlpTest : public VPlanTestBase {
  20 protected:
  21   TargetLibraryInfoImpl TLII;
  22   TargetLibraryInfo TLI;
  23   DataLayout DL;
  24
  25   std::unique_ptr<AssumptionCache> AC;
  26   std::unique_ptr<ScalarEvolution> SE;
  27   std::unique_ptr<AAResults> AARes;
  28   std::unique_ptr<BasicAAResult> BasicAA;
  29   std::unique_ptr<LoopAccessInfo> LAI;
  30   std::unique_ptr<PredicatedScalarEvolution> PSE;
  31   std::unique_ptr<InterleavedAccessInfo> IAI;
  32
  33   VPlanSlpTest()
  34       : TLII(), TLI(TLII),
  35         DL("e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-"
  36            "f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:"
  37            "16:32:64-S128") {}
  38
  39   VPInterleavedAccessInfo getInterleavedAccessInfo(Function &F, Loop *L,
  40                                                    VPlan &Plan) {
  41     AC.reset(new AssumptionCache(F));
  42     SE.reset(new ScalarEvolution(F, TLI, *AC, *DT, *LI));
  43     BasicAA.reset(new BasicAAResult(DL, F, TLI, *AC, &*DT));
  44     AARes.reset(new AAResults(TLI));
  45     AARes->addAAResult(*BasicAA);
  46     PSE.reset(new PredicatedScalarEvolution(*SE, *L));
  47     LAI.reset(new LoopAccessInfo(L, &*SE, nullptr, &TLI, &*AARes, &*DT, &*LI));
  48     IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI));
  49     IAI->analyzeInterleaving(false);
  50     return {Plan, *IAI};
  51   }
  52 };
  53
  54 TEST_F(VPlanSlpTest, testSlpSimple_2) {
  55   const char *ModuleString =
  56       "%struct.Test = type { i32, i32 }\n"
  57       "%struct.Test3 = type { i32, i32, i32 }\n"
  58       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
  59       "define void @add_x2(ptr nocapture readonly %A, ptr "
  60       "nocapture readonly %B, ptr nocapture %C)  {\n"
  61       "entry:\n"
  62       "  br label %for.body\n"
  63       "for.body:                                         ; preds = %for.body, "
  64       "%entry\n"
  65       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
  66       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
  67       "%indvars.iv, i32 0\n"
  68       "  %vA0 = load i32, ptr %A0, align 4\n"
  69       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
  70       "%indvars.iv, i32 0\n"
  71       "  %vB0 = load i32, ptr %B0, align 4\n"
  72       "  %add0 = add nsw i32 %vA0, %vB0\n"
  73       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
  74       "%indvars.iv, i32 1\n"
  75       "  %vA1 = load i32, ptr %A1, align 4\n"
  76       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
  77       "%indvars.iv, i32 1\n"
  78       "  %vB1 = load i32, ptr %B1, align 4\n"
  79       "  %add1 = add nsw i32 %vA1, %vB1\n"
  80       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
  81       "%indvars.iv, i32 0\n"
  82       "  store i32 %add0, ptr %C0, align 4\n"
  83       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
  84       "%indvars.iv, i32 1\n"
  85       "  store i32 %add1, ptr %C1, align 4\n"
  86       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
  87       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
  88       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
  89       "for.cond.cleanup:                                 ; preds = %for.body\n"
  90       "  ret void\n"
  91       "}\n";
  92
  93   Module &M = parseModule(ModuleString);
  94
  95   Function *F = M.getFunction("add_x2");
  96   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
  97   auto Plan = buildHCFG(LoopHeader);
  98   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
  99
 100   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 101   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 102   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 103
 104   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 105   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 106
 107   VPlanSlp Slp(VPIAI, *Body);
 108   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 109   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 110   EXPECT_EQ(64u, Slp.getWidestBundleBits());
 111   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 112
 113   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
 114   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 115
 116   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
 117   auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
 118   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 119   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 120
 121   delete CombinedStore;
 122   delete CombinedAdd;
 123   delete CombinedLoadA;
 124   delete CombinedLoadB;
 125 }
 126
 127 TEST_F(VPlanSlpTest, testSlpSimple_3) {
 128   const char *ModuleString =
 129       "%struct.Test = type { i32, i32 }\n"
 130       "%struct.Test3 = type { i32, i32, i32 }\n"
 131       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 132       "define void @add_x2(ptr nocapture readonly %A, ptr "
 133       "nocapture readonly %B, ptr nocapture %C)  {\n"
 134       "entry:\n"
 135       "  br label %for.body\n"
 136       "for.body:                                         ; preds = %for.body, "
 137       "%entry\n"
 138       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 139       "  %A0 = getelementptr %struct.Test, ptr %A, i64 "
 140       "                      %indvars.iv, i32 0\n"
 141       "  %vA0 = load i32, ptr %A0, align 4\n"
 142       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 143       "                      %indvars.iv, i32 0\n"
 144       "  %vB0 = load i32, ptr %B0, align 4\n"
 145       "  %add0 = add nsw i32 %vA0, %vB0\n"
 146       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 147       "                      %indvars.iv, i32 1\n"
 148       "  %vA1 = load i32, ptr %A1, align 4\n"
 149       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 150       "                      %indvars.iv, i32 1\n"
 151       "  %vB1 = load i32, ptr %B1, align 4\n"
 152       "  %add1 = add nsw i32 %vA1, %vB1\n"
 153       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 154       "                      %indvars.iv, i32 0\n"
 155       "  store i32 %add0, ptr %C0, align 4\n"
 156       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 157       "                      %indvars.iv, i32 1\n"
 158       "  store i32 %add1, ptr %C1, align 4\n"
 159       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 160       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 161       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 162       "for.cond.cleanup:                                 ; preds = %for.body\n"
 163       "  ret void\n"
 164       "}\n";
 165
 166   Module &M = parseModule(ModuleString);
 167
 168   Function *F = M.getFunction("add_x2");
 169   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 170   auto Plan = buildHCFG(LoopHeader);
 171
 172   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 173   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 174   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 175
 176   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 177   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 178
 179   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 180
 181   VPlanSlp Slp(VPIAI, *Body);
 182   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 183   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 184   EXPECT_EQ(64u, Slp.getWidestBundleBits());
 185   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 186
 187   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
 188   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 189
 190   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
 191   auto *CombinedLoadB = cast<VPInstruction>(CombinedAdd->getOperand(1));
 192   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 193   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 194
 195   VPInstruction *GetA = cast<VPInstruction>(&*std::next(Body->begin(), 1));
 196   VPInstruction *GetB = cast<VPInstruction>(&*std::next(Body->begin(), 3));
 197   EXPECT_EQ(GetA, CombinedLoadA->getOperand(0));
 198   EXPECT_EQ(GetB, CombinedLoadB->getOperand(0));
 199
 200   delete CombinedStore;
 201   delete CombinedAdd;
 202   delete CombinedLoadA;
 203   delete CombinedLoadB;
 204 }
 205
 206 TEST_F(VPlanSlpTest, testSlpReuse_1) {
 207   const char *ModuleString =
 208       "%struct.Test = type { i32, i32 }\n"
 209       "define void @add_x2(ptr nocapture readonly %A, ptr "
 210       "nocapture readonly %B, ptr nocapture %C)  {\n"
 211       "entry:\n"
 212       "  br label %for.body\n"
 213       "for.body:                                         ; preds = %for.body, "
 214       "%entry\n"
 215       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 216       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 217       "%indvars.iv, i32 0\n"
 218       "  %vA0 = load i32, ptr %A0, align 4\n"
 219       "  %add0 = add nsw i32 %vA0, %vA0\n"
 220       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 221       "%indvars.iv, i32 1\n"
 222       "  %vA1 = load i32, ptr %A1, align 4\n"
 223       "  %add1 = add nsw i32 %vA1, %vA1\n"
 224       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 225       "%indvars.iv, i32 0\n"
 226       "  store i32 %add0, ptr %C0, align 4\n"
 227       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 228       "%indvars.iv, i32 1\n"
 229       "  store i32 %add1, ptr %C1, align 4\n"
 230       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 231       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 232       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 233       "for.cond.cleanup:                                 ; preds = %for.body\n"
 234       "  ret void\n"
 235       "}\n";
 236
 237   Module &M = parseModule(ModuleString);
 238
 239   Function *F = M.getFunction("add_x2");
 240   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 241   auto Plan = buildHCFG(LoopHeader);
 242   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 243
 244   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 245   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 246   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 247
 248   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 8));
 249   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
 250
 251   VPlanSlp Slp(VPIAI, *Body);
 252   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 253   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 254   EXPECT_EQ(64u, Slp.getWidestBundleBits());
 255   EXPECT_EQ(VPInstruction::SLPStore, CombinedStore->getOpcode());
 256
 257   auto *CombinedAdd = cast<VPInstruction>(CombinedStore->getOperand(0));
 258   EXPECT_EQ(Instruction::Add, CombinedAdd->getOpcode());
 259
 260   auto *CombinedLoadA = cast<VPInstruction>(CombinedAdd->getOperand(0));
 261   EXPECT_EQ(CombinedLoadA, CombinedAdd->getOperand(1));
 262   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 263
 264   delete CombinedStore;
 265   delete CombinedAdd;
 266   delete CombinedLoadA;
 267 }
 268
 269 TEST_F(VPlanSlpTest, testSlpReuse_2) {
 270   const char *ModuleString =
 271       "%struct.Test = type { i32, i32 }\n"
 272       "define void @add_x2(ptr nocapture readonly %A, ptr "
 273       "nocapture readonly %B, ptr nocapture %C)  {\n"
 274       "entry:\n"
 275       "  br label %for.body\n"
 276       "for.body:                                         ; preds = %for.body, "
 277       "%entry\n"
 278       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 279       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 280       "%indvars.iv, i32 0\n"
 281       "  %vA0 = load i32, ptr %A0, align 4\n"
 282       "  %add0 = add nsw i32 %vA0, %vA0\n"
 283       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 284       "%indvars.iv, i32 0\n"
 285       "  store i32 %add0, ptr %C0, align 4\n"
 286       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 287       "%indvars.iv, i32 1\n"
 288       "  %vA1 = load i32, ptr %A1, align 4\n"
 289       "  %add1 = add nsw i32 %vA1, %vA1\n"
 290       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 291       "%indvars.iv, i32 1\n"
 292       "  store i32 %add1, ptr %C1, align 4\n"
 293       "  %use = add i32 %vA1, 1\n"
 294       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 295       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 296       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 297       "for.cond.cleanup:                                 ; preds = %for.body\n"
 298       "  ret void\n"
 299       "}\n";
 300
 301   Module &M = parseModule(ModuleString);
 302
 303   Function *F = M.getFunction("add_x2");
 304   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 305   auto Plan = buildHCFG(LoopHeader);
 306   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 307
 308   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 309   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 310   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 311
 312   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 5));
 313   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 10));
 314
 315   VPlanSlp Slp(VPIAI, *Body);
 316   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 317   Slp.buildGraph(StoreRoot);
 318   EXPECT_FALSE(Slp.isCompletelySLP());
 319 }
 320
 321 static void checkReorderExample(VPInstruction *Store1, VPInstruction *Store2,
 322                                 VPBasicBlock *Body,
 323                                 VPInterleavedAccessInfo &&IAI) {
 324   VPlanSlp Slp(IAI, *Body);
 325   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 326   VPInstruction *CombinedStore = Slp.buildGraph(StoreRoot);
 327
 328   EXPECT_TRUE(Slp.isCompletelySLP());
 329   EXPECT_EQ(CombinedStore->getOpcode(), VPInstruction::SLPStore);
 330
 331   VPInstruction *CombinedAdd =
 332       cast<VPInstruction>(CombinedStore->getOperand(0));
 333   EXPECT_EQ(CombinedAdd->getOpcode(), Instruction::Add);
 334
 335   VPInstruction *CombinedMulAB =
 336       cast<VPInstruction>(CombinedAdd->getOperand(0));
 337   VPInstruction *CombinedMulCD =
 338       cast<VPInstruction>(CombinedAdd->getOperand(1));
 339   EXPECT_EQ(CombinedMulAB->getOpcode(), Instruction::Mul);
 340
 341   VPInstruction *CombinedLoadA =
 342       cast<VPInstruction>(CombinedMulAB->getOperand(0));
 343   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadA->getOpcode());
 344   VPInstruction *LoadvA0 = cast<VPInstruction>(&*std::next(Body->begin(), 2));
 345   VPInstruction *LoadvA1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 346   EXPECT_EQ(LoadvA0->getOperand(0), CombinedLoadA->getOperand(0));
 347   EXPECT_EQ(LoadvA1->getOperand(0), CombinedLoadA->getOperand(1));
 348
 349   VPInstruction *CombinedLoadB =
 350       cast<VPInstruction>(CombinedMulAB->getOperand(1));
 351   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadB->getOpcode());
 352   VPInstruction *LoadvB0 = cast<VPInstruction>(&*std::next(Body->begin(), 4));
 353   VPInstruction *LoadvB1 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 354   EXPECT_EQ(LoadvB0->getOperand(0), CombinedLoadB->getOperand(0));
 355   EXPECT_EQ(LoadvB1->getOperand(0), CombinedLoadB->getOperand(1));
 356
 357   EXPECT_EQ(CombinedMulCD->getOpcode(), Instruction::Mul);
 358
 359   VPInstruction *CombinedLoadC =
 360       cast<VPInstruction>(CombinedMulCD->getOperand(0));
 361   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadC->getOpcode());
 362   VPInstruction *LoadvC0 = cast<VPInstruction>(&*std::next(Body->begin(), 7));
 363   VPInstruction *LoadvC1 = cast<VPInstruction>(&*std::next(Body->begin(), 17));
 364   EXPECT_EQ(LoadvC0->getOperand(0), CombinedLoadC->getOperand(0));
 365   EXPECT_EQ(LoadvC1->getOperand(0), CombinedLoadC->getOperand(1));
 366
 367   VPInstruction *CombinedLoadD =
 368       cast<VPInstruction>(CombinedMulCD->getOperand(1));
 369   EXPECT_EQ(VPInstruction::SLPLoad, CombinedLoadD->getOpcode());
 370   VPInstruction *LoadvD0 = cast<VPInstruction>(&*std::next(Body->begin(), 9));
 371   VPInstruction *LoadvD1 = cast<VPInstruction>(&*std::next(Body->begin(), 19));
 372   EXPECT_EQ(LoadvD0->getOperand(0), CombinedLoadD->getOperand(0));
 373   EXPECT_EQ(LoadvD1->getOperand(0), CombinedLoadD->getOperand(1));
 374
 375   delete CombinedStore;
 376   delete CombinedAdd;
 377   delete CombinedMulAB;
 378   delete CombinedMulCD;
 379   delete CombinedLoadA;
 380   delete CombinedLoadB;
 381   delete CombinedLoadC;
 382   delete CombinedLoadD;
 383 }
 384
 385 TEST_F(VPlanSlpTest, testSlpReorder_1) {
 386   LLVMContext Ctx;
 387   const char *ModuleString =
 388       "%struct.Test = type { i32, i32 }\n"
 389       "define void @add_x3(ptr %A, ptr %B, ptr "
 390       "%C,  ptr %D,  ptr %E)  {\n"
 391       "entry:\n"
 392       "  br label %for.body\n"
 393       "for.body:                                         ; preds = %for.body, "
 394       "%entry\n"
 395       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 396       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 397       "%indvars.iv, i32 0\n"
 398       "  %vA0 = load i32, ptr %A0, align 4\n"
 399       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 400       "%indvars.iv, i32 0\n"
 401       "  %vB0 = load i32, ptr %B0, align 4\n"
 402       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
 403       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 404       "%indvars.iv, i32 0\n"
 405       "  %vC0 = load i32, ptr %C0, align 4\n"
 406       "  %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 407       "%indvars.iv, i32 0\n"
 408       "  %vD0 = load i32, ptr %D0, align 4\n"
 409       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 410       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 411       "%indvars.iv, i32 1\n"
 412       "  %vA1 = load i32, ptr %A1, align 4\n"
 413       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 414       "%indvars.iv, i32 1\n"
 415       "  %vB1 = load i32, ptr %B1, align 4\n"
 416       "  %mul21 = mul nsw i32 %vA1, %vB1\n"
 417       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 418       "%indvars.iv, i32 1\n"
 419       "  %vC1 = load i32, ptr %C1, align 4\n"
 420       "  %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 421       "%indvars.iv, i32 1\n"
 422       "  %vD1 = load i32, ptr %D1, align 4\n"
 423       "  %mul22 = mul nsw i32 %vC1, %vD1\n"
 424       "  %add1 = add nsw i32 %mul11, %mul12\n"
 425       "  %add2 = add nsw i32 %mul22, %mul21\n"
 426       "  %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 427       "%indvars.iv, i32 0\n"
 428       "  store i32 %add1, ptr %E0, align 4\n"
 429       "  %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 430       "%indvars.iv, i32 1\n"
 431       "  store i32 %add2, ptr %E1, align 4\n"
 432       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 433       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 434       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 435       "for.cond.cleanup:                                 ; preds = %for.body\n"
 436       "  ret void\n"
 437       "}\n";
 438
 439   Module &M = parseModule(ModuleString);
 440
 441   Function *F = M.getFunction("add_x3");
 442   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 443   auto Plan = buildHCFG(LoopHeader);
 444
 445   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 446   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 447   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 448
 449   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 450   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 451
 452   checkReorderExample(
 453       Store1, Store2, Body,
 454       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
 455 }
 456
 457 TEST_F(VPlanSlpTest, testSlpReorder_2) {
 458   LLVMContext Ctx;
 459   const char *ModuleString =
 460       "%struct.Test = type { i32, i32 }\n"
 461       "define void @add_x3(ptr %A, ptr %B, ptr "
 462       "%C,  ptr %D,  ptr %E)  {\n"
 463       "entry:\n"
 464       "  br label %for.body\n"
 465       "for.body:                                         ; preds = %for.body, "
 466       "%entry\n"
 467       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 468       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 469       "%indvars.iv, i32 0\n"
 470       "  %vA0 = load i32, ptr %A0, align 4\n"
 471       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 472       "%indvars.iv, i32 0\n"
 473       "  %vB0 = load i32, ptr %B0, align 4\n"
 474       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
 475       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 476       "%indvars.iv, i32 0\n"
 477       "  %vC0 = load i32, ptr %C0, align 4\n"
 478       "  %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 479       "%indvars.iv, i32 0\n"
 480       "  %vD0 = load i32, ptr %D0, align 4\n"
 481       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 482       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 483       "%indvars.iv, i32 1\n"
 484       "  %vA1 = load i32, ptr %A1, align 4\n"
 485       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 486       "%indvars.iv, i32 1\n"
 487       "  %vB1 = load i32, ptr %B1, align 4\n"
 488       "  %mul21 = mul nsw i32 %vB1, %vA1\n"
 489       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 490       "%indvars.iv, i32 1\n"
 491       "  %vC1 = load i32, ptr %C1, align 4\n"
 492       "  %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 493       "%indvars.iv, i32 1\n"
 494       "  %vD1 = load i32, ptr %D1, align 4\n"
 495       "  %mul22 = mul nsw i32 %vD1, %vC1\n"
 496       "  %add1 = add nsw i32 %mul11, %mul12\n"
 497       "  %add2 = add nsw i32 %mul22, %mul21\n"
 498       "  %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 499       "%indvars.iv, i32 0\n"
 500       "  store i32 %add1, ptr %E0, align 4\n"
 501       "  %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 502       "%indvars.iv, i32 1\n"
 503       "  store i32 %add2, ptr %E1, align 4\n"
 504       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 505       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 506       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 507       "for.cond.cleanup:                                 ; preds = %for.body\n"
 508       "  ret void\n"
 509       "}\n";
 510
 511   Module &M = parseModule(ModuleString);
 512
 513   Function *F = M.getFunction("add_x3");
 514   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 515   auto Plan = buildHCFG(LoopHeader);
 516
 517   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 518   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 519   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 520
 521   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 522   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 523
 524   checkReorderExample(
 525       Store1, Store2, Body,
 526       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
 527 }
 528
 529 TEST_F(VPlanSlpTest, testSlpReorder_3) {
 530   LLVMContext Ctx;
 531   const char *ModuleString =
 532       "%struct.Test = type { i32, i32 }\n"
 533       "define void @add_x3(ptr %A, ptr %B, ptr "
 534       "%C,  ptr %D,  ptr %E)  {\n"
 535       "entry:\n"
 536       "  br label %for.body\n"
 537       "for.body:                                         ; preds = %for.body, "
 538       "%entry\n"
 539       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 540       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 541       "%indvars.iv, i32 1\n"
 542       "  %vA1 = load i32, ptr %A1, align 4\n"
 543       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 544       "%indvars.iv, i32 0\n"
 545       "  %vB0 = load i32, ptr %B0, align 4\n"
 546       "  %mul11 = mul nsw i32 %vA1, %vB0\n"
 547       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 548       "%indvars.iv, i32 0\n"
 549       "  %vC0 = load i32, ptr %C0, align 4\n"
 550       "  %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 551       "%indvars.iv, i32 0\n"
 552       "  %vD0 = load i32, ptr %D0, align 4\n"
 553       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 554       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 555       "%indvars.iv, i32 0\n"
 556       "  %vA0 = load i32, ptr %A0, align 4\n"
 557       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 558       "%indvars.iv, i32 1\n"
 559       "  %vB1 = load i32, ptr %B1, align 4\n"
 560       "  %mul21 = mul nsw i32 %vB1, %vA0\n"
 561       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 562       "%indvars.iv, i32 1\n"
 563       "  %vC1 = load i32, ptr %C1, align 4\n"
 564       "  %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 565       "%indvars.iv, i32 1\n"
 566       "  %vD1 = load i32, ptr %D1, align 4\n"
 567       "  %mul22 = mul nsw i32 %vD1, %vC1\n"
 568       "  %add1 = add nsw i32 %mul11, %mul12\n"
 569       "  %add2 = add nsw i32 %mul22, %mul21\n"
 570       "  %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 571       "%indvars.iv, i32 0\n"
 572       "  store i32 %add1, ptr %E0, align 4\n"
 573       "  %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 574       "%indvars.iv, i32 1\n"
 575       "  store i32 %add2, ptr %E1, align 4\n"
 576       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 577       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 578       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 579       "for.cond.cleanup:                                 ; preds = %for.body\n"
 580       "  ret void\n"
 581       "}\n";
 582
 583   Module &M = parseModule(ModuleString);
 584
 585   Function *F = M.getFunction("add_x3");
 586   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 587   auto Plan = buildHCFG(LoopHeader);
 588
 589   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 590   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 591   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 592
 593   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 594   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 595
 596   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 597   VPlanSlp Slp(VPIAI, *Body);
 598   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 599   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 600
 601   // FIXME Need to select better first value for lane0.
 602   EXPECT_FALSE(Slp.isCompletelySLP());
 603 }
 604
 605 TEST_F(VPlanSlpTest, testSlpReorder_4) {
 606   LLVMContext Ctx;
 607   const char *ModuleString =
 608       "%struct.Test = type { i32, i32 }\n"
 609       "define void @add_x3(ptr %A, ptr %B, ptr "
 610       "%C,  ptr %D,  ptr %E)  {\n"
 611       "entry:\n"
 612       "  br label %for.body\n"
 613       "for.body:                                         ; preds = %for.body, "
 614       "%entry\n"
 615       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 616       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 617       "%indvars.iv, i32 0\n"
 618       "  %vA0 = load i32, ptr %A0, align 4\n"
 619       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 620       "%indvars.iv, i32 0\n"
 621       "  %vB0 = load i32, ptr %B0, align 4\n"
 622       "  %mul11 = mul nsw i32 %vA0, %vB0\n"
 623       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 624       "%indvars.iv, i32 0\n"
 625       "  %vC0 = load i32, ptr %C0, align 4\n"
 626       "  %D0 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 627       "%indvars.iv, i32 0\n"
 628       "  %vD0 = load i32, ptr %D0, align 4\n"
 629       "  %mul12 = mul nsw i32 %vC0, %vD0\n"
 630       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 631       "%indvars.iv, i32 1\n"
 632       "  %vA1 = load i32, ptr %A1, align 4\n"
 633       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 634       "%indvars.iv, i32 1\n"
 635       "  %vB1 = load i32, ptr %B1, align 4\n"
 636       "  %mul21 = mul nsw i32 %vA1, %vB1\n"
 637       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 638       "%indvars.iv, i32 1\n"
 639       "  %vC1 = load i32, ptr %C1, align 4\n"
 640       "  %D1 = getelementptr inbounds %struct.Test, ptr %D, i64 "
 641       "%indvars.iv, i32 1\n"
 642       "  %vD1 = load i32, ptr %D1, align 4\n"
 643       "  %mul22 = mul nsw i32 %vC1, %vD1\n"
 644       "  %add1 = add nsw i32 %mul11, %mul12\n"
 645       "  %add2 = add nsw i32 %mul22, %mul21\n"
 646       "  %E0 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 647       "%indvars.iv, i32 0\n"
 648       "  store i32 %add1, ptr %E0, align 4\n"
 649       "  %E1 = getelementptr inbounds %struct.Test, ptr %E, i64 "
 650       "%indvars.iv, i32 1\n"
 651       "  store i32 %add2, ptr %E1, align 4\n"
 652       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 653       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 654       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 655       "for.cond.cleanup:                                 ; preds = %for.body\n"
 656       "  ret void\n"
 657       "}\n";
 658
 659   Module &M = parseModule(ModuleString);
 660
 661   Function *F = M.getFunction("add_x3");
 662   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 663   auto Plan = buildHCFG(LoopHeader);
 664
 665   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 666   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 667   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 668
 669   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 24));
 670   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 26));
 671
 672   checkReorderExample(
 673       Store1, Store2, Body,
 674       getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan));
 675 }
 676
 677 // Make sure we do not combine instructions with operands in different BBs.
 678 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs) {
 679   const char *ModuleString =
 680       "%struct.Test = type { i32, i32 }\n"
 681       "%struct.Test3 = type { i32, i32, i32 }\n"
 682       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 683       "define void @add_x2(ptr nocapture readonly %A, ptr "
 684       "nocapture readonly %B, ptr nocapture %C)  {\n"
 685       "entry:\n"
 686       "  br label %for.body\n"
 687       "for.body:                                         ; preds = %for.body, "
 688       "%entry\n"
 689       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %bb2 ]\n"
 690       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 691       "%indvars.iv, i32 0\n"
 692       "  %vA0 = load i32, ptr %A0, align 4\n"
 693       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 694       "%indvars.iv, i32 0\n"
 695       "  %vB0 = load i32, ptr %B0, align 4\n"
 696       "  %add0 = add nsw i32 %vA0, %vB0\n"
 697       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 698       "%indvars.iv, i32 1\n"
 699       "  %vA1 = load i32, ptr %A1, align 4\n"
 700       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 701       "%indvars.iv, i32 1\n"
 702       "  br label %bb2\n"
 703       "bb2:\n"
 704       "  %vB1 = load i32, ptr %B1, align 4\n"
 705       "  %add1 = add nsw i32 %vA1, %vB1\n"
 706       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 707       "%indvars.iv, i32 0\n"
 708       "  store i32 %add0, ptr %C0, align 4\n"
 709       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 710       "%indvars.iv, i32 1\n"
 711       "  store i32 %add1, ptr %C1, align 4\n"
 712       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 713       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 714       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 715       "for.cond.cleanup:                                 ; preds = %for.body\n"
 716       "  ret void\n"
 717       "}\n";
 718
 719   Module &M = parseModule(ModuleString);
 720
 721   Function *F = M.getFunction("add_x2");
 722   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 723   auto Plan = buildHCFG(LoopHeader);
 724   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 725
 726   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 727   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 728   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 729   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 730
 731   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
 732   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 5));
 733
 734   VPlanSlp Slp(VPIAI, *BB2);
 735   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 736   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 737   EXPECT_EQ(0u, Slp.getWidestBundleBits());
 738 }
 739
 740 // Make sure we do not combine instructions with operands in different BBs.
 741 TEST_F(VPlanSlpTest, testInstrsInDifferentBBs2) {
 742   const char *ModuleString =
 743       "%struct.Test = type { i32, i32 }\n"
 744       "%struct.Test3 = type { i32, i32, i32 }\n"
 745       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 746       "define void @add_x2(ptr nocapture readonly %A, ptr "
 747       "nocapture readonly %B, ptr nocapture %C)  {\n"
 748       "entry:\n"
 749       "  br label %for.body\n"
 750       "for.body:                                         ; preds = %for.body, "
 751       "%entry\n"
 752       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %bb2 ]\n"
 753       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 754       "%indvars.iv, i32 0\n"
 755       "  %vA0 = load i32, ptr %A0, align 4\n"
 756       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 757       "%indvars.iv, i32 0\n"
 758       "  %vB0 = load i32, ptr %B0, align 4\n"
 759       "  %add0 = add nsw i32 %vA0, %vB0\n"
 760       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 761       "%indvars.iv, i32 1\n"
 762       "  %vA1 = load i32, ptr %A1, align 4\n"
 763       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 764       "%indvars.iv, i32 1\n"
 765       "  %vB1 = load i32, ptr %B1, align 4\n"
 766       "  %add1 = add nsw i32 %vA1, %vB1\n"
 767       "  br label %bb2\n"
 768       "bb2:\n"
 769       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 770       "%indvars.iv, i32 0\n"
 771       "  store i32 %add0, ptr %C0, align 4\n"
 772       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 773       "%indvars.iv, i32 1\n"
 774       "  store i32 %add1, ptr %C1, align 4\n"
 775       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 776       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 777       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 778       "for.cond.cleanup:                                 ; preds = %for.body\n"
 779       "  ret void\n"
 780       "}\n";
 781
 782   Module &M = parseModule(ModuleString);
 783
 784   Function *F = M.getFunction("add_x2");
 785   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 786   auto Plan = buildHCFG(LoopHeader);
 787   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 788
 789   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 790   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 791   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 792   VPBasicBlock *BB2 = Body->getSingleSuccessor()->getEntryBasicBlock();
 793
 794   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(BB2->begin(), 1));
 795   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(BB2->begin(), 3));
 796
 797   VPlanSlp Slp(VPIAI, *BB2);
 798   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 799   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 800   EXPECT_EQ(0u, Slp.getWidestBundleBits());
 801 }
 802
 803 TEST_F(VPlanSlpTest, testSlpAtomicLoad) {
 804   const char *ModuleString =
 805       "%struct.Test = type { i32, i32 }\n"
 806       "%struct.Test3 = type { i32, i32, i32 }\n"
 807       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 808       "define void @add_x2(ptr nocapture readonly %A, ptr "
 809       "nocapture readonly %B, ptr nocapture %C)  {\n"
 810       "entry:\n"
 811       "  br label %for.body\n"
 812       "for.body:                                         ; preds = %for.body, "
 813       "%entry\n"
 814       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 815       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 816       "%indvars.iv, i32 0\n"
 817       "  %vA0 = load atomic i32, ptr %A0 monotonic, align 4\n"
 818       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 819       "%indvars.iv, i32 0\n"
 820       "  %vB0 = load i32, ptr %B0, align 4\n"
 821       "  %add0 = add nsw i32 %vA0, %vB0\n"
 822       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 823       "%indvars.iv, i32 1\n"
 824       "  %vA1 = load i32, ptr %A1, align 4\n"
 825       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 826       "%indvars.iv, i32 1\n"
 827       "  %vB1 = load i32, ptr %B1, align 4\n"
 828       "  %add1 = add nsw i32 %vA1, %vB1\n"
 829       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 830       "%indvars.iv, i32 0\n"
 831       "  store i32 %add0, ptr %C0, align 4\n"
 832       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 833       "%indvars.iv, i32 1\n"
 834       "  store i32 %add1, ptr %C1, align 4\n"
 835       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 836       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 837       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 838       "for.cond.cleanup:                                 ; preds = %for.body\n"
 839       "  ret void\n"
 840       "}\n";
 841
 842   Module &M = parseModule(ModuleString);
 843
 844   Function *F = M.getFunction("add_x2");
 845   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 846   auto Plan = buildHCFG(LoopHeader);
 847   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 848
 849   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 850   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 851   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 852
 853   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 854   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 855
 856   VPlanSlp Slp(VPIAI, *Body);
 857   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 858   EXPECT_EQ(nullptr, Slp.buildGraph(StoreRoot));
 859   EXPECT_FALSE(Slp.isCompletelySLP());
 860 }
 861
 862 TEST_F(VPlanSlpTest, testSlpAtomicStore) {
 863   const char *ModuleString =
 864       "%struct.Test = type { i32, i32 }\n"
 865       "%struct.Test3 = type { i32, i32, i32 }\n"
 866       "%struct.Test4xi8 = type { i8, i8, i8 }\n"
 867       "define void @add_x2(ptr nocapture readonly %A, ptr "
 868       "nocapture readonly %B, ptr nocapture %C)  {\n"
 869       "entry:\n"
 870       "  br label %for.body\n"
 871       "for.body:                                         ; preds = %for.body, "
 872       "%entry\n"
 873       "  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\n"
 874       "  %A0 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 875       "%indvars.iv, i32 0\n"
 876       "  %vA0 = load i32, ptr %A0, align 4\n"
 877       "  %B0 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 878       "%indvars.iv, i32 0\n"
 879       "  %vB0 = load i32, ptr %B0, align 4\n"
 880       "  %add0 = add nsw i32 %vA0, %vB0\n"
 881       "  %A1 = getelementptr inbounds %struct.Test, ptr %A, i64 "
 882       "%indvars.iv, i32 1\n"
 883       "  %vA1 = load i32, ptr %A1, align 4\n"
 884       "  %B1 = getelementptr inbounds %struct.Test, ptr %B, i64 "
 885       "%indvars.iv, i32 1\n"
 886       "  %vB1 = load i32, ptr %B1, align 4\n"
 887       "  %add1 = add nsw i32 %vA1, %vB1\n"
 888       "  %C0 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 889       "%indvars.iv, i32 0\n"
 890       "  store atomic i32 %add0, ptr %C0 monotonic, align 4\n"
 891       "  %C1 = getelementptr inbounds %struct.Test, ptr %C, i64 "
 892       "%indvars.iv, i32 1\n"
 893       "  store i32 %add1, ptr %C1, align 4\n"
 894       "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
 895       "  %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
 896       "  br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
 897       "for.cond.cleanup:                                 ; preds = %for.body\n"
 898       "  ret void\n"
 899       "}\n";
 900
 901   Module &M = parseModule(ModuleString);
 902
 903   Function *F = M.getFunction("add_x2");
 904   BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
 905   auto Plan = buildHCFG(LoopHeader);
 906   auto VPIAI = getInterleavedAccessInfo(*F, LI->getLoopFor(LoopHeader), *Plan);
 907
 908   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
 909   EXPECT_NE(nullptr, Entry->getSingleSuccessor());
 910   VPBasicBlock *Body = Plan->getVectorLoopRegion()->getEntryBasicBlock();
 911
 912   VPInstruction *Store1 = cast<VPInstruction>(&*std::next(Body->begin(), 12));
 913   VPInstruction *Store2 = cast<VPInstruction>(&*std::next(Body->begin(), 14));
 914
 915   VPlanSlp Slp(VPIAI, *Body);
 916   SmallVector<VPValue *, 4> StoreRoot = {Store1, Store2};
 917   Slp.buildGraph(StoreRoot);
 918   EXPECT_FALSE(Slp.isCompletelySLP());
 919 }
 920
 921 } // namespace
 922 } // namespace llvm