llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp

   1 //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Frontend/OpenMP/OMPConstants.h"
  10 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
  11 #include "llvm/IR/BasicBlock.h"
  12 #include "llvm/IR/DIBuilder.h"
  13 #include "llvm/IR/Function.h"
  14 #include "llvm/IR/InstIterator.h"
  15 #include "llvm/IR/LLVMContext.h"
  16 #include "llvm/IR/Module.h"
  17 #include "llvm/IR/Verifier.h"
  18 #include "llvm/Passes/PassBuilder.h"
  19 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  20 #include "gtest/gtest.h"
  21
  22 using namespace llvm;
  23 using namespace omp;
  24
  25 namespace {
  26
  27 /// Create an instruction that uses the values in \p Values. We use "printf"
  28 /// just because it is often used for this purpose in test code, but it is never
  29 /// executed here.
  30 static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr,
  31                                   ArrayRef<Value *> Values) {
  32   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
  33
  34   GlobalVariable *GV = Builder.CreateGlobalString(FormatStr, "", 0, M);
  35   Constant *Zero = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
  36   Constant *Indices[] = {Zero, Zero};
  37   Constant *FormatStrConst =
  38       ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, Indices);
  39
  40   Function *PrintfDecl = M->getFunction("printf");
  41   if (!PrintfDecl) {
  42     GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
  43     FunctionType *Ty = FunctionType::get(Builder.getInt32Ty(), true);
  44     PrintfDecl = Function::Create(Ty, Linkage, "printf", M);
  45   }
  46
  47   SmallVector<Value *, 4> Args;
  48   Args.push_back(FormatStrConst);
  49   Args.append(Values.begin(), Values.end());
  50   return Builder.CreateCall(PrintfDecl, Args);
  51 }
  52
  53 /// Verify that blocks in \p RefOrder are corresponds to the depth-first visit
  54 /// order the control flow of \p F.
  55 ///
  56 /// This is an easy way to verify the branching structure of the CFG without
  57 /// checking every branch instruction individually. For the CFG of a
  58 /// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering
  59 /// the body, i.e. the DFS order corresponds to the execution order with one
  60 /// loop iteration.
  61 static testing::AssertionResult
  62 verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
  63   ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
  64   ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
  65
  66   df_iterator_default_set<BasicBlock *, 16> Visited;
  67   auto DFS = llvm::depth_first_ext(&F->getEntryBlock(), Visited);
  68
  69   BasicBlock *Prev = nullptr;
  70   for (BasicBlock *BB : DFS) {
  71     if (It != E && BB == *It) {
  72       Prev = *It;
  73       ++It;
  74     }
  75   }
  76
  77   if (It == E)
  78     return testing::AssertionSuccess();
  79   if (!Prev)
  80     return testing::AssertionFailure()
  81            << "Did not find " << (*It)->getName() << " in control flow";
  82   return testing::AssertionFailure()
  83          << "Expected " << Prev->getName() << " before " << (*It)->getName()
  84          << " in control flow";
  85 }
  86
  87 /// Verify that blocks in \p RefOrder are in the same relative order in the
  88 /// linked lists of blocks in \p F. The linked list may contain additional
  89 /// blocks in-between.
  90 ///
  91 /// While the order in the linked list is not relevant for semantics, keeping
  92 /// the order roughly in execution order makes its printout easier to read.
  93 static testing::AssertionResult
  94 verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
  95   ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
  96   ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
  97
  98   BasicBlock *Prev = nullptr;
  99   for (BasicBlock &BB : *F) {
 100     if (It != E && &BB == *It) {
 101       Prev = *It;
 102       ++It;
 103     }
 104   }
 105
 106   if (It == E)
 107     return testing::AssertionSuccess();
 108   if (!Prev)
 109     return testing::AssertionFailure() << "Did not find " << (*It)->getName()
 110                                        << " in function " << F->getName();
 111   return testing::AssertionFailure()
 112          << "Expected " << Prev->getName() << " before " << (*It)->getName()
 113          << " in function " << F->getName();
 114 }
 115
 116 /// Populate Calls with call instructions calling the function with the given
 117 /// FnID from the given function F.
 118 static void findCalls(Function *F, omp::RuntimeFunction FnID,
 119                       OpenMPIRBuilder &OMPBuilder,
 120                       SmallVectorImpl<CallInst *> &Calls) {
 121   Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID);
 122   for (BasicBlock &BB : *F) {
 123     for (Instruction &I : BB) {
 124       auto *Call = dyn_cast<CallInst>(&I);
 125       if (Call && Call->getCalledFunction() == Fn)
 126         Calls.push_back(Call);
 127     }
 128   }
 129 }
 130
 131 /// Assuming \p F contains only one call to the function with the given \p FnID,
 132 /// return that call.
 133 static CallInst *findSingleCall(Function *F, omp::RuntimeFunction FnID,
 134                                 OpenMPIRBuilder &OMPBuilder) {
 135   SmallVector<CallInst *, 1> Calls;
 136   findCalls(F, FnID, OMPBuilder, Calls);
 137   EXPECT_EQ(1u, Calls.size());
 138   if (Calls.size() != 1)
 139     return nullptr;
 140   return Calls.front();
 141 }
 142
 143 static omp::ScheduleKind getSchedKind(omp::OMPScheduleType SchedType) {
 144   switch (SchedType & ~omp::OMPScheduleType::ModifierMask) {
 145   case omp::OMPScheduleType::BaseDynamicChunked:
 146     return omp::OMP_SCHEDULE_Dynamic;
 147   case omp::OMPScheduleType::BaseGuidedChunked:
 148     return omp::OMP_SCHEDULE_Guided;
 149   case omp::OMPScheduleType::BaseAuto:
 150     return omp::OMP_SCHEDULE_Auto;
 151   case omp::OMPScheduleType::BaseRuntime:
 152     return omp::OMP_SCHEDULE_Runtime;
 153   default:
 154     llvm_unreachable("unknown type for this test");
 155   }
 156 }
 157
 158 class OpenMPIRBuilderTest : public testing::Test {
 159 protected:
 160   void SetUp() override {
 161     Ctx.setOpaquePointers(false); // TODO: Update tests for opaque pointers.
 162     M.reset(new Module("MyModule", Ctx));
 163     FunctionType *FTy =
 164         FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
 165                           /*isVarArg=*/false);
 166     F = Function::Create(FTy, Function::ExternalLinkage, "", M.get());
 167     BB = BasicBlock::Create(Ctx, "", F);
 168
 169     DIBuilder DIB(*M);
 170     auto File = DIB.createFile("test.dbg", "/src", llvm::None,
 171                                Optional<StringRef>("/src/test.dbg"));
 172     auto CU =
 173         DIB.createCompileUnit(dwarf::DW_LANG_C, File, "llvm-C", true, "", 0);
 174     auto Type = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None));
 175     auto SP = DIB.createFunction(
 176         CU, "foo", "", File, 1, Type, 1, DINode::FlagZero,
 177         DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized);
 178     F->setSubprogram(SP);
 179     auto Scope = DIB.createLexicalBlockFile(SP, File, 0);
 180     DIB.finalize();
 181     DL = DILocation::get(Ctx, 3, 7, Scope);
 182   }
 183
 184   void TearDown() override {
 185     BB = nullptr;
 186     M.reset();
 187   }
 188
 189   /// Create a function with a simple loop that calls printf using the logical
 190   /// loop counter for use with tests that need a CanonicalLoopInfo object.
 191   CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL,
 192                                              OpenMPIRBuilder &OMPBuilder,
 193                                              int UseIVBits,
 194                                              CallInst **Call = nullptr,
 195                                              BasicBlock **BodyCode = nullptr) {
 196     OMPBuilder.initialize();
 197     F->setName("func");
 198
 199     IRBuilder<> Builder(BB);
 200     OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 201     Value *TripCount = F->getArg(0);
 202
 203     Type *IVType = Type::getIntNTy(Builder.getContext(), UseIVBits);
 204     Value *CastedTripCount =
 205         Builder.CreateZExtOrTrunc(TripCount, IVType, "tripcount");
 206
 207     auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP,
 208                              llvm::Value *LC) {
 209       Builder.restoreIP(CodeGenIP);
 210       if (BodyCode)
 211         *BodyCode = Builder.GetInsertBlock();
 212
 213       // Add something that consumes the induction variable to the body.
 214       CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC});
 215       if (Call)
 216         *Call = CallInst;
 217     };
 218     CanonicalLoopInfo *Loop =
 219         OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount);
 220
 221     // Finalize the function.
 222     Builder.restoreIP(Loop->getAfterIP());
 223     Builder.CreateRetVoid();
 224
 225     return Loop;
 226   }
 227
 228   LLVMContext Ctx;
 229   std::unique_ptr<Module> M;
 230   Function *F;
 231   BasicBlock *BB;
 232   DebugLoc DL;
 233 };
 234
 235 class OpenMPIRBuilderTestWithParams
 236     : public OpenMPIRBuilderTest,
 237       public ::testing::WithParamInterface<omp::OMPScheduleType> {};
 238
 239 class OpenMPIRBuilderTestWithIVBits
 240     : public OpenMPIRBuilderTest,
 241       public ::testing::WithParamInterface<int> {};
 242
 243 // Returns the value stored in the given allocation. Returns null if the given
 244 // value is not a result of an InstTy instruction, if no value is stored or if
 245 // there is more than one store.
 246 template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) {
 247   Instruction *Inst = dyn_cast<InstTy>(AllocaValue);
 248   if (!Inst)
 249     return nullptr;
 250   StoreInst *Store = nullptr;
 251   for (Use &U : Inst->uses()) {
 252     if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) {
 253       EXPECT_EQ(Store, nullptr);
 254       Store = CandidateStore;
 255     }
 256   }
 257   if (!Store)
 258     return nullptr;
 259   return Store->getValueOperand();
 260 }
 261
 262 // Returns the value stored in the aggregate argument of an outlined function,
 263 // or nullptr if it is not found.
 264 static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,
 265                                            unsigned Idx) {
 266   GetElementPtrInst *GEPAtIdx = nullptr;
 267   // Find GEP instruction at that index.
 268   for (User *Usr : Aggregate->users()) {
 269     GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Usr);
 270     if (!GEP)
 271       continue;
 272
 273     if (GEP->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx), Idx))
 274       continue;
 275
 276     EXPECT_EQ(GEPAtIdx, nullptr);
 277     GEPAtIdx = GEP;
 278   }
 279
 280   EXPECT_NE(GEPAtIdx, nullptr);
 281   EXPECT_EQ(GEPAtIdx->getNumUses(), 1U);
 282
 283   // Find the value stored to the aggregate.
 284   StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin());
 285   Value *StoredAggValue = StoreToAgg->getValueOperand();
 286
 287   Value *StoredValue = nullptr;
 288
 289   // Find the value stored to the value stored in the aggregate.
 290   for (User *Usr : StoredAggValue->users()) {
 291     StoreInst *Store = dyn_cast<StoreInst>(Usr);
 292     if (!Store)
 293       continue;
 294
 295     if (Store->getPointerOperand() != StoredAggValue)
 296       continue;
 297
 298     EXPECT_EQ(StoredValue, nullptr);
 299     StoredValue = Store->getValueOperand();
 300   }
 301
 302   return StoredValue;
 303 }
 304
 305 // Returns the aggregate that the value is originating from.
 306 static Value *findAggregateFromValue(Value *V) {
 307   // Expects a load instruction that loads from the aggregate.
 308   LoadInst *Load = dyn_cast<LoadInst>(V);
 309   EXPECT_NE(Load, nullptr);
 310   // Find the GEP instruction used in the load instruction.
 311   GetElementPtrInst *GEP =
 312       dyn_cast<GetElementPtrInst>(Load->getPointerOperand());
 313   EXPECT_NE(GEP, nullptr);
 314   // Find the aggregate used in the GEP instruction.
 315   Value *Aggregate = GEP->getPointerOperand();
 316
 317   return Aggregate;
 318 }
 319
 320 TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
 321   OpenMPIRBuilder OMPBuilder(*M);
 322   OMPBuilder.initialize();
 323
 324   IRBuilder<> Builder(BB);
 325
 326   OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
 327   EXPECT_TRUE(M->global_empty());
 328   EXPECT_EQ(M->size(), 1U);
 329   EXPECT_EQ(F->size(), 1U);
 330   EXPECT_EQ(BB->size(), 0U);
 331
 332   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
 333   OMPBuilder.createBarrier(Loc, OMPD_for);
 334   EXPECT_FALSE(M->global_empty());
 335   EXPECT_EQ(M->size(), 3U);
 336   EXPECT_EQ(F->size(), 1U);
 337   EXPECT_EQ(BB->size(), 2U);
 338
 339   CallInst *GTID = dyn_cast<CallInst>(&BB->front());
 340   EXPECT_NE(GTID, nullptr);
 341   EXPECT_EQ(GTID->arg_size(), 1U);
 342   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
 343   EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
 344   EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
 345
 346   CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
 347   EXPECT_NE(Barrier, nullptr);
 348   EXPECT_EQ(Barrier->arg_size(), 2U);
 349   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier");
 350   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
 351   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
 352
 353   EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
 354
 355   Builder.CreateUnreachable();
 356   EXPECT_FALSE(verifyModule(*M, &errs()));
 357 }
 358
 359 TEST_F(OpenMPIRBuilderTest, CreateCancel) {
 360   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 361   OpenMPIRBuilder OMPBuilder(*M);
 362   OMPBuilder.initialize();
 363
 364   BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
 365   new UnreachableInst(Ctx, CBB);
 366   auto FiniCB = [&](InsertPointTy IP) {
 367     ASSERT_NE(IP.getBlock(), nullptr);
 368     ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
 369     BranchInst::Create(CBB, IP.getBlock());
 370   };
 371   OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
 372
 373   IRBuilder<> Builder(BB);
 374
 375   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
 376   auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
 377   Builder.restoreIP(NewIP);
 378   EXPECT_FALSE(M->global_empty());
 379   EXPECT_EQ(M->size(), 4U);
 380   EXPECT_EQ(F->size(), 4U);
 381   EXPECT_EQ(BB->size(), 4U);
 382
 383   CallInst *GTID = dyn_cast<CallInst>(&BB->front());
 384   EXPECT_NE(GTID, nullptr);
 385   EXPECT_EQ(GTID->arg_size(), 1U);
 386   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
 387   EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
 388   EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
 389
 390   CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
 391   EXPECT_NE(Cancel, nullptr);
 392   EXPECT_EQ(Cancel->arg_size(), 3U);
 393   EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
 394   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
 395   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
 396   EXPECT_EQ(Cancel->getNumUses(), 1U);
 397   Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
 398   EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
 399   EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
 400   EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
 401   CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
 402   EXPECT_NE(GTID1, nullptr);
 403   EXPECT_EQ(GTID1->arg_size(), 1U);
 404   EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
 405   EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
 406   EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
 407   CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
 408   EXPECT_NE(Barrier, nullptr);
 409   EXPECT_EQ(Barrier->arg_size(), 2U);
 410   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
 411   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
 412   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
 413   EXPECT_EQ(Barrier->getNumUses(), 0U);
 414   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
 415             1U);
 416   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
 417
 418   EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
 419
 420   OMPBuilder.popFinalizationCB();
 421
 422   Builder.CreateUnreachable();
 423   EXPECT_FALSE(verifyModule(*M, &errs()));
 424 }
 425
 426 TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
 427   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 428   OpenMPIRBuilder OMPBuilder(*M);
 429   OMPBuilder.initialize();
 430
 431   BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
 432   new UnreachableInst(Ctx, CBB);
 433   auto FiniCB = [&](InsertPointTy IP) {
 434     ASSERT_NE(IP.getBlock(), nullptr);
 435     ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
 436     BranchInst::Create(CBB, IP.getBlock());
 437   };
 438   OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
 439
 440   IRBuilder<> Builder(BB);
 441
 442   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
 443   auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
 444   Builder.restoreIP(NewIP);
 445   EXPECT_FALSE(M->global_empty());
 446   EXPECT_EQ(M->size(), 4U);
 447   EXPECT_EQ(F->size(), 7U);
 448   EXPECT_EQ(BB->size(), 1U);
 449   ASSERT_TRUE(isa<BranchInst>(BB->getTerminator()));
 450   ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U);
 451   BB = BB->getTerminator()->getSuccessor(0);
 452   EXPECT_EQ(BB->size(), 4U);
 453
 454   CallInst *GTID = dyn_cast<CallInst>(&BB->front());
 455   EXPECT_NE(GTID, nullptr);
 456   EXPECT_EQ(GTID->arg_size(), 1U);
 457   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
 458   EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
 459   EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
 460
 461   CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
 462   EXPECT_NE(Cancel, nullptr);
 463   EXPECT_EQ(Cancel->arg_size(), 3U);
 464   EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
 465   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
 466   EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
 467   EXPECT_EQ(Cancel->getNumUses(), 1U);
 468   Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
 469   EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
 470   EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
 471   EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),
 472             NewIP.getBlock());
 473   EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
 474   CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
 475   EXPECT_NE(GTID1, nullptr);
 476   EXPECT_EQ(GTID1->arg_size(), 1U);
 477   EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
 478   EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
 479   EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
 480   CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
 481   EXPECT_NE(Barrier, nullptr);
 482   EXPECT_EQ(Barrier->arg_size(), 2U);
 483   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
 484   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
 485   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
 486   EXPECT_EQ(Barrier->getNumUses(), 0U);
 487   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
 488             1U);
 489   EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
 490
 491   EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
 492
 493   OMPBuilder.popFinalizationCB();
 494
 495   Builder.CreateUnreachable();
 496   EXPECT_FALSE(verifyModule(*M, &errs()));
 497 }
 498
 499 TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
 500   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 501   OpenMPIRBuilder OMPBuilder(*M);
 502   OMPBuilder.initialize();
 503
 504   BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
 505   new UnreachableInst(Ctx, CBB);
 506   auto FiniCB = [&](InsertPointTy IP) {
 507     ASSERT_NE(IP.getBlock(), nullptr);
 508     ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
 509     BranchInst::Create(CBB, IP.getBlock());
 510   };
 511   OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
 512
 513   IRBuilder<> Builder(BB);
 514
 515   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
 516   auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for);
 517   Builder.restoreIP(NewIP);
 518   EXPECT_FALSE(M->global_empty());
 519   EXPECT_EQ(M->size(), 3U);
 520   EXPECT_EQ(F->size(), 4U);
 521   EXPECT_EQ(BB->size(), 4U);
 522
 523   CallInst *GTID = dyn_cast<CallInst>(&BB->front());
 524   EXPECT_NE(GTID, nullptr);
 525   EXPECT_EQ(GTID->arg_size(), 1U);
 526   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
 527   EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
 528   EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
 529
 530   CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
 531   EXPECT_NE(Barrier, nullptr);
 532   EXPECT_EQ(Barrier->arg_size(), 2U);
 533   EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
 534   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
 535   EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
 536   EXPECT_EQ(Barrier->getNumUses(), 1U);
 537   Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
 538   EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
 539   EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
 540   EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);
 541   EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
 542             1U);
 543   EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0),
 544             CBB);
 545
 546   EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
 547
 548   OMPBuilder.popFinalizationCB();
 549
 550   Builder.CreateUnreachable();
 551   EXPECT_FALSE(verifyModule(*M, &errs()));
 552 }
 553
 554 TEST_F(OpenMPIRBuilderTest, DbgLoc) {
 555   OpenMPIRBuilder OMPBuilder(*M);
 556   OMPBuilder.initialize();
 557   F->setName("func");
 558
 559   IRBuilder<> Builder(BB);
 560
 561   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 562   OMPBuilder.createBarrier(Loc, OMPD_for);
 563   CallInst *GTID = dyn_cast<CallInst>(&BB->front());
 564   CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
 565   EXPECT_EQ(GTID->getDebugLoc(), DL);
 566   EXPECT_EQ(Barrier->getDebugLoc(), DL);
 567   EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0)));
 568   if (!isa<GlobalVariable>(Barrier->getOperand(0)))
 569     return;
 570   GlobalVariable *Ident = cast<GlobalVariable>(Barrier->getOperand(0));
 571   EXPECT_TRUE(Ident->hasInitializer());
 572   if (!Ident->hasInitializer())
 573     return;
 574   Constant *Initializer = Ident->getInitializer();
 575   EXPECT_TRUE(
 576       isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()));
 577   GlobalVariable *SrcStrGlob =
 578       cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
 579   if (!SrcStrGlob)
 580     return;
 581   EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer()));
 582   ConstantDataArray *SrcSrc =
 583       dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
 584   if (!SrcSrc)
 585     return;
 586   EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;");
 587 }
 588
 589 TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
 590   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 591   OpenMPIRBuilder OMPBuilder(*M);
 592   OMPBuilder.initialize();
 593   F->setName("func");
 594   IRBuilder<> Builder(BB);
 595
 596   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
 597   Builder.CreateBr(EnterBB);
 598   Builder.SetInsertPoint(EnterBB);
 599   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 600
 601   AllocaInst *PrivAI = nullptr;
 602
 603   unsigned NumBodiesGenerated = 0;
 604   unsigned NumPrivatizedVars = 0;
 605   unsigned NumFinalizationPoints = 0;
 606
 607   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
 608     ++NumBodiesGenerated;
 609
 610     Builder.restoreIP(AllocaIP);
 611     PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
 612     Builder.CreateStore(F->arg_begin(), PrivAI);
 613
 614     Builder.restoreIP(CodeGenIP);
 615     Value *PrivLoad =
 616         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
 617     Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
 618     Instruction *ThenTerm, *ElseTerm;
 619     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
 620                                   &ThenTerm, &ElseTerm);
 621   };
 622
 623   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
 624                     Value &Orig, Value &Inner,
 625                     Value *&ReplacementValue) -> InsertPointTy {
 626     ++NumPrivatizedVars;
 627
 628     if (!isa<AllocaInst>(Orig)) {
 629       EXPECT_EQ(&Orig, F->arg_begin());
 630       ReplacementValue = &Inner;
 631       return CodeGenIP;
 632     }
 633
 634     // Since the original value is an allocation, it has a pointer type and
 635     // therefore no additional wrapping should happen.
 636     EXPECT_EQ(&Orig, &Inner);
 637
 638     // Trivial copy (=firstprivate).
 639     Builder.restoreIP(AllocaIP);
 640     Type *VTy = ReplacementValue->getType();
 641     Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
 642     ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
 643     Builder.restoreIP(CodeGenIP);
 644     Builder.CreateStore(V, ReplacementValue);
 645     return CodeGenIP;
 646   };
 647
 648   auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
 649
 650   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
 651                                     F->getEntryBlock().getFirstInsertionPt());
 652   IRBuilder<>::InsertPoint AfterIP =
 653       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
 654                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
 655   EXPECT_EQ(NumBodiesGenerated, 1U);
 656   EXPECT_EQ(NumPrivatizedVars, 1U);
 657   EXPECT_EQ(NumFinalizationPoints, 1U);
 658
 659   Builder.restoreIP(AfterIP);
 660   Builder.CreateRetVoid();
 661
 662   OMPBuilder.finalize();
 663
 664   EXPECT_NE(PrivAI, nullptr);
 665   Function *OutlinedFn = PrivAI->getFunction();
 666   EXPECT_NE(F, OutlinedFn);
 667   EXPECT_FALSE(verifyModule(*M, &errs()));
 668   EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));
 669   EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse));
 670   EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));
 671   EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));
 672
 673   EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
 674   EXPECT_EQ(OutlinedFn->arg_size(), 3U);
 675
 676   EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
 677   EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
 678   User *Usr = OutlinedFn->user_back();
 679   ASSERT_TRUE(isa<ConstantExpr>(Usr));
 680   CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
 681   ASSERT_NE(ForkCI, nullptr);
 682
 683   EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
 684   EXPECT_EQ(ForkCI->arg_size(), 4U);
 685   EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
 686   EXPECT_EQ(ForkCI->getArgOperand(1),
 687             ConstantInt::get(Type::getInt32Ty(Ctx), 1U));
 688   EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
 689   Value *StoredValue =
 690       findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);
 691   EXPECT_EQ(StoredValue, F->arg_begin());
 692 }
 693
 694 TEST_F(OpenMPIRBuilderTest, ParallelNested) {
 695   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 696   OpenMPIRBuilder OMPBuilder(*M);
 697   OMPBuilder.initialize();
 698   F->setName("func");
 699   IRBuilder<> Builder(BB);
 700
 701   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
 702   Builder.CreateBr(EnterBB);
 703   Builder.SetInsertPoint(EnterBB);
 704   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 705
 706   unsigned NumInnerBodiesGenerated = 0;
 707   unsigned NumOuterBodiesGenerated = 0;
 708   unsigned NumFinalizationPoints = 0;
 709
 710   auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
 711     ++NumInnerBodiesGenerated;
 712   };
 713
 714   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
 715                     Value &Orig, Value &Inner,
 716                     Value *&ReplacementValue) -> InsertPointTy {
 717     // Trivial copy (=firstprivate).
 718     Builder.restoreIP(AllocaIP);
 719     Type *VTy = ReplacementValue->getType();
 720     Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
 721     ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
 722     Builder.restoreIP(CodeGenIP);
 723     Builder.CreateStore(V, ReplacementValue);
 724     return CodeGenIP;
 725   };
 726
 727   auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
 728
 729   auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
 730     ++NumOuterBodiesGenerated;
 731     Builder.restoreIP(CodeGenIP);
 732     BasicBlock *CGBB = CodeGenIP.getBlock();
 733     BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());
 734     CGBB->getTerminator()->eraseFromParent();
 735     ;
 736
 737     IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel(
 738         InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
 739         FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
 740
 741     Builder.restoreIP(AfterIP);
 742     Builder.CreateBr(NewBB);
 743   };
 744
 745   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
 746                                     F->getEntryBlock().getFirstInsertionPt());
 747   IRBuilder<>::InsertPoint AfterIP =
 748       OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
 749                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
 750
 751   EXPECT_EQ(NumInnerBodiesGenerated, 1U);
 752   EXPECT_EQ(NumOuterBodiesGenerated, 1U);
 753   EXPECT_EQ(NumFinalizationPoints, 2U);
 754
 755   Builder.restoreIP(AfterIP);
 756   Builder.CreateRetVoid();
 757
 758   OMPBuilder.finalize();
 759
 760   EXPECT_EQ(M->size(), 5U);
 761   for (Function &OutlinedFn : *M) {
 762     if (F == &OutlinedFn || OutlinedFn.isDeclaration())
 763       continue;
 764     EXPECT_FALSE(verifyModule(*M, &errs()));
 765     EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
 766     EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
 767     EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
 768     EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
 769
 770     EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
 771     EXPECT_EQ(OutlinedFn.arg_size(), 2U);
 772
 773     EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
 774     User *Usr = OutlinedFn.user_back();
 775     ASSERT_TRUE(isa<ConstantExpr>(Usr));
 776     CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
 777     ASSERT_NE(ForkCI, nullptr);
 778
 779     EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
 780     EXPECT_EQ(ForkCI->arg_size(), 3U);
 781     EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
 782     EXPECT_EQ(ForkCI->getArgOperand(1),
 783               ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
 784     EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
 785   }
 786 }
 787
 788 TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
 789   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 790   OpenMPIRBuilder OMPBuilder(*M);
 791   OMPBuilder.initialize();
 792   F->setName("func");
 793   IRBuilder<> Builder(BB);
 794
 795   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
 796   Builder.CreateBr(EnterBB);
 797   Builder.SetInsertPoint(EnterBB);
 798   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 799
 800   unsigned NumInnerBodiesGenerated = 0;
 801   unsigned NumOuterBodiesGenerated = 0;
 802   unsigned NumFinalizationPoints = 0;
 803
 804   auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
 805     ++NumInnerBodiesGenerated;
 806   };
 807
 808   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
 809                     Value &Orig, Value &Inner,
 810                     Value *&ReplacementValue) -> InsertPointTy {
 811     // Trivial copy (=firstprivate).
 812     Builder.restoreIP(AllocaIP);
 813     Type *VTy = ReplacementValue->getType();
 814     Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
 815     ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
 816     Builder.restoreIP(CodeGenIP);
 817     Builder.CreateStore(V, ReplacementValue);
 818     return CodeGenIP;
 819   };
 820
 821   auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
 822
 823   auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
 824     ++NumOuterBodiesGenerated;
 825     Builder.restoreIP(CodeGenIP);
 826     BasicBlock *CGBB = CodeGenIP.getBlock();
 827     BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint());
 828     BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt());
 829     CGBB->getTerminator()->eraseFromParent();
 830     ;
 831     NewBB1->getTerminator()->eraseFromParent();
 832     ;
 833
 834     IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel(
 835         InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
 836         FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
 837
 838     Builder.restoreIP(AfterIP1);
 839     Builder.CreateBr(NewBB1);
 840
 841     IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel(
 842         InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
 843         FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
 844
 845     Builder.restoreIP(AfterIP2);
 846     Builder.CreateBr(NewBB2);
 847   };
 848
 849   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
 850                                     F->getEntryBlock().getFirstInsertionPt());
 851   IRBuilder<>::InsertPoint AfterIP =
 852       OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
 853                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
 854
 855   EXPECT_EQ(NumInnerBodiesGenerated, 2U);
 856   EXPECT_EQ(NumOuterBodiesGenerated, 1U);
 857   EXPECT_EQ(NumFinalizationPoints, 3U);
 858
 859   Builder.restoreIP(AfterIP);
 860   Builder.CreateRetVoid();
 861
 862   OMPBuilder.finalize();
 863
 864   EXPECT_EQ(M->size(), 6U);
 865   for (Function &OutlinedFn : *M) {
 866     if (F == &OutlinedFn || OutlinedFn.isDeclaration())
 867       continue;
 868     EXPECT_FALSE(verifyModule(*M, &errs()));
 869     EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
 870     EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
 871     EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
 872     EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
 873
 874     EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
 875     EXPECT_EQ(OutlinedFn.arg_size(), 2U);
 876
 877     unsigned NumAllocas = 0;
 878     for (Instruction &I : instructions(OutlinedFn))
 879       NumAllocas += isa<AllocaInst>(I);
 880     EXPECT_EQ(NumAllocas, 1U);
 881
 882     EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
 883     User *Usr = OutlinedFn.user_back();
 884     ASSERT_TRUE(isa<ConstantExpr>(Usr));
 885     CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
 886     ASSERT_NE(ForkCI, nullptr);
 887
 888     EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
 889     EXPECT_EQ(ForkCI->arg_size(), 3U);
 890     EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
 891     EXPECT_EQ(ForkCI->getArgOperand(1),
 892               ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
 893     EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
 894   }
 895 }
 896
 897 TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
 898   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
 899   OpenMPIRBuilder OMPBuilder(*M);
 900   OMPBuilder.initialize();
 901   F->setName("func");
 902   IRBuilder<> Builder(BB);
 903
 904   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
 905   Builder.CreateBr(EnterBB);
 906   Builder.SetInsertPoint(EnterBB);
 907   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 908
 909   AllocaInst *PrivAI = nullptr;
 910
 911   unsigned NumBodiesGenerated = 0;
 912   unsigned NumPrivatizedVars = 0;
 913   unsigned NumFinalizationPoints = 0;
 914
 915   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
 916     ++NumBodiesGenerated;
 917
 918     Builder.restoreIP(AllocaIP);
 919     PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
 920     Builder.CreateStore(F->arg_begin(), PrivAI);
 921
 922     Builder.restoreIP(CodeGenIP);
 923     Value *PrivLoad =
 924         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
 925     Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
 926     Instruction *ThenTerm, *ElseTerm;
 927     SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm,
 928                                   &ElseTerm);
 929   };
 930
 931   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
 932                     Value &Orig, Value &Inner,
 933                     Value *&ReplacementValue) -> InsertPointTy {
 934     ++NumPrivatizedVars;
 935
 936     if (!isa<AllocaInst>(Orig)) {
 937       EXPECT_EQ(&Orig, F->arg_begin());
 938       ReplacementValue = &Inner;
 939       return CodeGenIP;
 940     }
 941
 942     // Since the original value is an allocation, it has a pointer type and
 943     // therefore no additional wrapping should happen.
 944     EXPECT_EQ(&Orig, &Inner);
 945
 946     // Trivial copy (=firstprivate).
 947     Builder.restoreIP(AllocaIP);
 948     Type *VTy = ReplacementValue->getType();
 949     Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
 950     ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
 951     Builder.restoreIP(CodeGenIP);
 952     Builder.CreateStore(V, ReplacementValue);
 953     return CodeGenIP;
 954   };
 955
 956   auto FiniCB = [&](InsertPointTy CodeGenIP) {
 957     ++NumFinalizationPoints;
 958     // No destructors.
 959   };
 960
 961   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
 962                                     F->getEntryBlock().getFirstInsertionPt());
 963   IRBuilder<>::InsertPoint AfterIP =
 964       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
 965                                 Builder.CreateIsNotNull(F->arg_begin()),
 966                                 nullptr, OMP_PROC_BIND_default, false);
 967
 968   EXPECT_EQ(NumBodiesGenerated, 1U);
 969   EXPECT_EQ(NumPrivatizedVars, 1U);
 970   EXPECT_EQ(NumFinalizationPoints, 1U);
 971
 972   Builder.restoreIP(AfterIP);
 973   Builder.CreateRetVoid();
 974   OMPBuilder.finalize();
 975
 976   EXPECT_NE(PrivAI, nullptr);
 977   Function *OutlinedFn = PrivAI->getFunction();
 978   EXPECT_NE(F, OutlinedFn);
 979   EXPECT_FALSE(verifyModule(*M, &errs()));
 980
 981   EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
 982   EXPECT_EQ(OutlinedFn->arg_size(), 3U);
 983
 984   EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
 985   ASSERT_EQ(OutlinedFn->getNumUses(), 2U);
 986
 987   CallInst *DirectCI = nullptr;
 988   CallInst *ForkCI = nullptr;
 989   for (User *Usr : OutlinedFn->users()) {
 990     if (isa<CallInst>(Usr)) {
 991       ASSERT_EQ(DirectCI, nullptr);
 992       DirectCI = cast<CallInst>(Usr);
 993     } else {
 994       ASSERT_TRUE(isa<ConstantExpr>(Usr));
 995       ASSERT_EQ(Usr->getNumUses(), 1U);
 996       ASSERT_TRUE(isa<CallInst>(Usr->user_back()));
 997       ForkCI = cast<CallInst>(Usr->user_back());
 998     }
 999   }
1000
1001   EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
1002   EXPECT_EQ(ForkCI->arg_size(), 4U);
1003   EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
1004   EXPECT_EQ(ForkCI->getArgOperand(1),
1005             ConstantInt::get(Type::getInt32Ty(Ctx), 1));
1006   Value *StoredForkArg =
1007       findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);
1008   EXPECT_EQ(StoredForkArg, F->arg_begin());
1009
1010   EXPECT_EQ(DirectCI->getCalledFunction(), OutlinedFn);
1011   EXPECT_EQ(DirectCI->arg_size(), 3U);
1012   EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(0)));
1013   EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(1)));
1014   Value *StoredDirectArg =
1015       findStoredValueInAggregateAt(Ctx, DirectCI->getArgOperand(2), 0);
1016   EXPECT_EQ(StoredDirectArg, F->arg_begin());
1017 }
1018
1019 TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
1020   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1021   OpenMPIRBuilder OMPBuilder(*M);
1022   OMPBuilder.initialize();
1023   F->setName("func");
1024   IRBuilder<> Builder(BB);
1025
1026   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
1027   Builder.CreateBr(EnterBB);
1028   Builder.SetInsertPoint(EnterBB);
1029   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1030
1031   unsigned NumBodiesGenerated = 0;
1032   unsigned NumPrivatizedVars = 0;
1033   unsigned NumFinalizationPoints = 0;
1034
1035   CallInst *CheckedBarrier = nullptr;
1036   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1037     ++NumBodiesGenerated;
1038
1039     Builder.restoreIP(CodeGenIP);
1040
1041     // Create three barriers, two cancel barriers but only one checked.
1042     Function *CBFn, *BFn;
1043
1044     Builder.restoreIP(
1045         OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));
1046
1047     CBFn = M->getFunction("__kmpc_cancel_barrier");
1048     BFn = M->getFunction("__kmpc_barrier");
1049     ASSERT_NE(CBFn, nullptr);
1050     ASSERT_EQ(BFn, nullptr);
1051     ASSERT_EQ(CBFn->getNumUses(), 1U);
1052     ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1053     ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
1054     CheckedBarrier = cast<CallInst>(CBFn->user_back());
1055
1056     Builder.restoreIP(
1057         OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));
1058     CBFn = M->getFunction("__kmpc_cancel_barrier");
1059     BFn = M->getFunction("__kmpc_barrier");
1060     ASSERT_NE(CBFn, nullptr);
1061     ASSERT_NE(BFn, nullptr);
1062     ASSERT_EQ(CBFn->getNumUses(), 1U);
1063     ASSERT_EQ(BFn->getNumUses(), 1U);
1064     ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
1065     ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
1066
1067     Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel,
1068                                                false, false));
1069     ASSERT_EQ(CBFn->getNumUses(), 2U);
1070     ASSERT_EQ(BFn->getNumUses(), 1U);
1071     ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
1072     ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1073     ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
1074   };
1075
1076   auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
1077                     Value *&) -> InsertPointTy {
1078     ++NumPrivatizedVars;
1079     llvm_unreachable("No privatization callback call expected!");
1080   };
1081
1082   FunctionType *FakeDestructorTy =
1083       FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
1084                         /*isVarArg=*/false);
1085   auto *FakeDestructor = Function::Create(
1086       FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get());
1087
1088   auto FiniCB = [&](InsertPointTy IP) {
1089     ++NumFinalizationPoints;
1090     Builder.restoreIP(IP);
1091     Builder.CreateCall(FakeDestructor,
1092                        {Builder.getInt32(NumFinalizationPoints)});
1093   };
1094
1095   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1096                                     F->getEntryBlock().getFirstInsertionPt());
1097   IRBuilder<>::InsertPoint AfterIP =
1098       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1099                                 Builder.CreateIsNotNull(F->arg_begin()),
1100                                 nullptr, OMP_PROC_BIND_default, true);
1101
1102   EXPECT_EQ(NumBodiesGenerated, 1U);
1103   EXPECT_EQ(NumPrivatizedVars, 0U);
1104   EXPECT_EQ(NumFinalizationPoints, 2U);
1105   EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
1106
1107   Builder.restoreIP(AfterIP);
1108   Builder.CreateRetVoid();
1109   OMPBuilder.finalize();
1110
1111   EXPECT_FALSE(verifyModule(*M, &errs()));
1112
1113   BasicBlock *ExitBB = nullptr;
1114   for (const User *Usr : FakeDestructor->users()) {
1115     const CallInst *CI = dyn_cast<CallInst>(Usr);
1116     ASSERT_EQ(CI->getCalledFunction(), FakeDestructor);
1117     ASSERT_TRUE(isa<BranchInst>(CI->getNextNode()));
1118     ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U);
1119     if (ExitBB)
1120       ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB);
1121     else
1122       ExitBB = CI->getNextNode()->getSuccessor(0);
1123     ASSERT_EQ(ExitBB->size(), 1U);
1124     if (!isa<ReturnInst>(ExitBB->front())) {
1125       ASSERT_TRUE(isa<BranchInst>(ExitBB->front()));
1126       ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U);
1127       ASSERT_TRUE(isa<ReturnInst>(
1128           cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front()));
1129     }
1130   }
1131 }
1132
1133 TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
1134   OpenMPIRBuilder OMPBuilder(*M);
1135   OMPBuilder.initialize();
1136   F->setName("func");
1137   IRBuilder<> Builder(BB);
1138   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1139   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1140
1141   Type *I32Ty = Type::getInt32Ty(M->getContext());
1142   Type *I32PtrTy = Type::getInt32PtrTy(M->getContext());
1143   Type *StructTy = StructType::get(I32Ty, I32PtrTy);
1144   Type *StructPtrTy = StructTy->getPointerTo();
1145   StructType *ArgStructTy =
1146       StructType::get(I32PtrTy, StructPtrTy, I32PtrTy, StructPtrTy);
1147   Type *VoidTy = Type::getVoidTy(M->getContext());
1148   FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty);
1149   FunctionCallee TakeI32Func =
1150       M->getOrInsertFunction("take_i32", VoidTy, I32Ty);
1151   FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", I32PtrTy);
1152   FunctionCallee TakeI32PtrFunc =
1153       M->getOrInsertFunction("take_i32ptr", VoidTy, I32PtrTy);
1154   FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy);
1155   FunctionCallee TakeStructFunc =
1156       M->getOrInsertFunction("take_struct", VoidTy, StructTy);
1157   FunctionCallee RetStructPtrFunc =
1158       M->getOrInsertFunction("ret_structptr", StructPtrTy);
1159   FunctionCallee TakeStructPtrFunc =
1160       M->getOrInsertFunction("take_structPtr", VoidTy, StructPtrTy);
1161   Value *I32Val = Builder.CreateCall(RetI32Func);
1162   Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc);
1163   Value *StructVal = Builder.CreateCall(RetStructFunc);
1164   Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);
1165
1166   Instruction *Internal;
1167   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1168     IRBuilder<>::InsertPointGuard Guard(Builder);
1169     Builder.restoreIP(CodeGenIP);
1170     Internal = Builder.CreateCall(TakeI32Func, I32Val);
1171     Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);
1172     Builder.CreateCall(TakeStructFunc, StructVal);
1173     Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);
1174   };
1175   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
1176                     Value &Inner, Value *&ReplacementValue) {
1177     ReplacementValue = &Inner;
1178     return CodeGenIP;
1179   };
1180   auto FiniCB = [](InsertPointTy) {};
1181
1182   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1183                                     F->getEntryBlock().getFirstInsertionPt());
1184   IRBuilder<>::InsertPoint AfterIP =
1185       OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1186                                 nullptr, nullptr, OMP_PROC_BIND_default, false);
1187   Builder.restoreIP(AfterIP);
1188   Builder.CreateRetVoid();
1189
1190   OMPBuilder.finalize();
1191
1192   EXPECT_FALSE(verifyModule(*M, &errs()));
1193   Function *OutlinedFn = Internal->getFunction();
1194
1195   Type *Arg2Type = OutlinedFn->getArg(2)->getType();
1196   EXPECT_TRUE(Arg2Type->isPointerTy());
1197   EXPECT_TRUE(
1198       cast<PointerType>(Arg2Type)->isOpaqueOrPointeeTypeMatches(ArgStructTy));
1199 }
1200
1201 TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
1202   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1203   OpenMPIRBuilder OMPBuilder(*M);
1204   OMPBuilder.initialize();
1205   IRBuilder<> Builder(BB);
1206   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1207   Value *TripCount = F->getArg(0);
1208
1209   unsigned NumBodiesGenerated = 0;
1210   auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
1211     NumBodiesGenerated += 1;
1212
1213     Builder.restoreIP(CodeGenIP);
1214
1215     Value *Cmp = Builder.CreateICmpEQ(LC, TripCount);
1216     Instruction *ThenTerm, *ElseTerm;
1217     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
1218                                   &ThenTerm, &ElseTerm);
1219   };
1220
1221   CanonicalLoopInfo *Loop =
1222       OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount);
1223
1224   Builder.restoreIP(Loop->getAfterIP());
1225   ReturnInst *RetInst = Builder.CreateRetVoid();
1226   OMPBuilder.finalize();
1227
1228   Loop->assertOK();
1229   EXPECT_FALSE(verifyModule(*M, &errs()));
1230
1231   EXPECT_EQ(NumBodiesGenerated, 1U);
1232
1233   // Verify control flow structure (in addition to Loop->assertOK()).
1234   EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock());
1235   EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock());
1236
1237   Instruction *IndVar = Loop->getIndVar();
1238   EXPECT_TRUE(isa<PHINode>(IndVar));
1239   EXPECT_EQ(IndVar->getType(), TripCount->getType());
1240   EXPECT_EQ(IndVar->getParent(), Loop->getHeader());
1241
1242   EXPECT_EQ(Loop->getTripCount(), TripCount);
1243
1244   BasicBlock *Body = Loop->getBody();
1245   Instruction *CmpInst = &Body->getInstList().front();
1246   EXPECT_TRUE(isa<ICmpInst>(CmpInst));
1247   EXPECT_EQ(CmpInst->getOperand(0), IndVar);
1248
1249   BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor();
1250   EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) {
1251     return SuccBB->getSingleSuccessor() == LatchPred;
1252   }));
1253
1254   EXPECT_EQ(&Loop->getAfter()->front(), RetInst);
1255 }
1256
1257 TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
1258   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1259   OpenMPIRBuilder OMPBuilder(*M);
1260   OMPBuilder.initialize();
1261   IRBuilder<> Builder(BB);
1262
1263   // Check the trip count is computed correctly. We generate the canonical loop
1264   // but rely on the IRBuilder's constant folder to compute the final result
1265   // since all inputs are constant. To verify overflow situations, limit the
1266   // trip count / loop counter widths to 16 bits.
1267   auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1268                            bool IsSigned, bool InclusiveStop) -> int64_t {
1269     OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1270     Type *LCTy = Type::getInt16Ty(Ctx);
1271     Value *StartVal = ConstantInt::get(LCTy, Start);
1272     Value *StopVal = ConstantInt::get(LCTy, Stop);
1273     Value *StepVal = ConstantInt::get(LCTy, Step);
1274     auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1275     CanonicalLoopInfo *Loop =
1276         OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1277                                        StepVal, IsSigned, InclusiveStop);
1278     Loop->assertOK();
1279     Builder.restoreIP(Loop->getAfterIP());
1280     Value *TripCount = Loop->getTripCount();
1281     return cast<ConstantInt>(TripCount)->getValue().getZExtValue();
1282   };
1283
1284   EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
1285   EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
1286   EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
1287   EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
1288   EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
1289   EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
1290   EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
1291   EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
1292   EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
1293   EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
1294   EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
1295   EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
1296   EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
1297
1298   EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
1299   EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
1300   EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
1301   EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
1302   EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
1303   EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
1304
1305   EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
1306   EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
1307   EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
1308   EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
1309   EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
1310   EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
1311   EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
1312
1313   EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
1314   EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
1315   EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
1316   EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
1317   EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
1318   EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
1319   EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
1320   EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
1321   EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
1322   EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
1323   EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
1324
1325   // Finalize the function and verify it.
1326   Builder.CreateRetVoid();
1327   OMPBuilder.finalize();
1328   EXPECT_FALSE(verifyModule(*M, &errs()));
1329 }
1330
1331 TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {
1332   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1333   OpenMPIRBuilder OMPBuilder(*M);
1334   OMPBuilder.initialize();
1335   F->setName("func");
1336
1337   IRBuilder<> Builder(BB);
1338
1339   Type *LCTy = F->getArg(0)->getType();
1340   Constant *One = ConstantInt::get(LCTy, 1);
1341   Constant *Two = ConstantInt::get(LCTy, 2);
1342   Value *OuterTripCount =
1343       Builder.CreateAdd(F->getArg(0), Two, "tripcount.outer");
1344   Value *InnerTripCount =
1345       Builder.CreateAdd(F->getArg(0), One, "tripcount.inner");
1346
1347   // Fix an insertion point for ComputeIP.
1348   BasicBlock *LoopNextEnter =
1349       BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1350                          Builder.GetInsertBlock()->getNextNode());
1351   BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1352   InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1353
1354   Builder.SetInsertPoint(LoopNextEnter);
1355   OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL);
1356
1357   CanonicalLoopInfo *InnerLoop = nullptr;
1358   CallInst *InbetweenLead = nullptr;
1359   CallInst *InbetweenTrail = nullptr;
1360   CallInst *Call = nullptr;
1361   auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) {
1362     Builder.restoreIP(OuterCodeGenIP);
1363     InbetweenLead =
1364         createPrintfCall(Builder, "In-between lead i=%d\\n", {OuterLC});
1365
1366     auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1367                                   Value *InnerLC) {
1368       Builder.restoreIP(InnerCodeGenIP);
1369       Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC});
1370     };
1371     InnerLoop = OMPBuilder.createCanonicalLoop(
1372         Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner");
1373
1374     Builder.restoreIP(InnerLoop->getAfterIP());
1375     InbetweenTrail =
1376         createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC});
1377   };
1378   CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1379       OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer");
1380
1381   // Finish the function.
1382   Builder.restoreIP(OuterLoop->getAfterIP());
1383   Builder.CreateRetVoid();
1384
1385   CanonicalLoopInfo *Collapsed =
1386       OMPBuilder.collapseLoops(DL, {OuterLoop, InnerLoop}, ComputeIP);
1387
1388   OMPBuilder.finalize();
1389   EXPECT_FALSE(verifyModule(*M, &errs()));
1390
1391   // Verify control flow and BB order.
1392   BasicBlock *RefOrder[] = {
1393       Collapsed->getPreheader(),   Collapsed->getHeader(),
1394       Collapsed->getCond(),        Collapsed->getBody(),
1395       InbetweenLead->getParent(),  Call->getParent(),
1396       InbetweenTrail->getParent(), Collapsed->getLatch(),
1397       Collapsed->getExit(),        Collapsed->getAfter(),
1398   };
1399   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1400   EXPECT_TRUE(verifyListOrder(F, RefOrder));
1401
1402   // Verify the total trip count.
1403   auto *TripCount = cast<MulOperator>(Collapsed->getTripCount());
1404   EXPECT_EQ(TripCount->getOperand(0), OuterTripCount);
1405   EXPECT_EQ(TripCount->getOperand(1), InnerTripCount);
1406
1407   // Verify the changed indvar.
1408   auto *OuterIV = cast<BinaryOperator>(Call->getOperand(1));
1409   EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv);
1410   EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody());
1411   EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount);
1412   EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar());
1413
1414   auto *InnerIV = cast<BinaryOperator>(Call->getOperand(2));
1415   EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem);
1416   EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody());
1417   EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar());
1418   EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount);
1419
1420   EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV);
1421   EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV);
1422 }
1423
1424 TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
1425   OpenMPIRBuilder OMPBuilder(*M);
1426   CallInst *Call;
1427   BasicBlock *BodyCode;
1428   CanonicalLoopInfo *Loop =
1429       buildSingleLoopFunction(DL, OMPBuilder, 32, &Call, &BodyCode);
1430
1431   Instruction *OrigIndVar = Loop->getIndVar();
1432   EXPECT_EQ(Call->getOperand(1), OrigIndVar);
1433
1434   // Tile the loop.
1435   Constant *TileSize = ConstantInt::get(Loop->getIndVarType(), APInt(32, 7));
1436   std::vector<CanonicalLoopInfo *> GenLoops =
1437       OMPBuilder.tileLoops(DL, {Loop}, {TileSize});
1438
1439   OMPBuilder.finalize();
1440   EXPECT_FALSE(verifyModule(*M, &errs()));
1441
1442   EXPECT_EQ(GenLoops.size(), 2u);
1443   CanonicalLoopInfo *Floor = GenLoops[0];
1444   CanonicalLoopInfo *Tile = GenLoops[1];
1445
1446   BasicBlock *RefOrder[] = {
1447       Floor->getPreheader(), Floor->getHeader(),   Floor->getCond(),
1448       Floor->getBody(),      Tile->getPreheader(), Tile->getHeader(),
1449       Tile->getCond(),       Tile->getBody(),      BodyCode,
1450       Tile->getLatch(),      Tile->getExit(),      Tile->getAfter(),
1451       Floor->getLatch(),     Floor->getExit(),     Floor->getAfter(),
1452   };
1453   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1454   EXPECT_TRUE(verifyListOrder(F, RefOrder));
1455
1456   // Check the induction variable.
1457   EXPECT_EQ(Call->getParent(), BodyCode);
1458   auto *Shift = cast<AddOperator>(Call->getOperand(1));
1459   EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody());
1460   EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar());
1461   auto *Scale = cast<MulOperator>(Shift->getOperand(0));
1462   EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody());
1463   EXPECT_EQ(Scale->getOperand(0), TileSize);
1464   EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar());
1465 }
1466
1467 TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
1468   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1469   OpenMPIRBuilder OMPBuilder(*M);
1470   OMPBuilder.initialize();
1471   F->setName("func");
1472
1473   IRBuilder<> Builder(BB);
1474   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1475   Value *TripCount = F->getArg(0);
1476   Type *LCTy = TripCount->getType();
1477
1478   BasicBlock *BodyCode = nullptr;
1479   CanonicalLoopInfo *InnerLoop = nullptr;
1480   auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1481                                 llvm::Value *OuterLC) {
1482     auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1483                                   llvm::Value *InnerLC) {
1484       Builder.restoreIP(InnerCodeGenIP);
1485       BodyCode = Builder.GetInsertBlock();
1486
1487       // Add something that consumes the induction variables to the body.
1488       createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1489     };
1490     InnerLoop = OMPBuilder.createCanonicalLoop(
1491         OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner");
1492   };
1493   CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1494       Loc, OuterLoopBodyGenCB, TripCount, "outer");
1495
1496   // Finalize the function.
1497   Builder.restoreIP(OuterLoop->getAfterIP());
1498   Builder.CreateRetVoid();
1499
1500   // Tile to loop nest.
1501   Constant *OuterTileSize = ConstantInt::get(LCTy, APInt(32, 11));
1502   Constant *InnerTileSize = ConstantInt::get(LCTy, APInt(32, 7));
1503   std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops(
1504       DL, {OuterLoop, InnerLoop}, {OuterTileSize, InnerTileSize});
1505
1506   OMPBuilder.finalize();
1507   EXPECT_FALSE(verifyModule(*M, &errs()));
1508
1509   EXPECT_EQ(GenLoops.size(), 4u);
1510   CanonicalLoopInfo *Floor1 = GenLoops[0];
1511   CanonicalLoopInfo *Floor2 = GenLoops[1];
1512   CanonicalLoopInfo *Tile1 = GenLoops[2];
1513   CanonicalLoopInfo *Tile2 = GenLoops[3];
1514
1515   BasicBlock *RefOrder[] = {
1516       Floor1->getPreheader(),
1517       Floor1->getHeader(),
1518       Floor1->getCond(),
1519       Floor1->getBody(),
1520       Floor2->getPreheader(),
1521       Floor2->getHeader(),
1522       Floor2->getCond(),
1523       Floor2->getBody(),
1524       Tile1->getPreheader(),
1525       Tile1->getHeader(),
1526       Tile1->getCond(),
1527       Tile1->getBody(),
1528       Tile2->getPreheader(),
1529       Tile2->getHeader(),
1530       Tile2->getCond(),
1531       Tile2->getBody(),
1532       BodyCode,
1533       Tile2->getLatch(),
1534       Tile2->getExit(),
1535       Tile2->getAfter(),
1536       Tile1->getLatch(),
1537       Tile1->getExit(),
1538       Tile1->getAfter(),
1539       Floor2->getLatch(),
1540       Floor2->getExit(),
1541       Floor2->getAfter(),
1542       Floor1->getLatch(),
1543       Floor1->getExit(),
1544       Floor1->getAfter(),
1545   };
1546   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1547   EXPECT_TRUE(verifyListOrder(F, RefOrder));
1548 }
1549
1550 TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
1551   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1552   OpenMPIRBuilder OMPBuilder(*M);
1553   OMPBuilder.initialize();
1554   F->setName("func");
1555
1556   IRBuilder<> Builder(BB);
1557   Value *TripCount = F->getArg(0);
1558   Type *LCTy = TripCount->getType();
1559
1560   Value *OuterStartVal = ConstantInt::get(LCTy, 2);
1561   Value *OuterStopVal = TripCount;
1562   Value *OuterStep = ConstantInt::get(LCTy, 5);
1563   Value *InnerStartVal = ConstantInt::get(LCTy, 13);
1564   Value *InnerStopVal = TripCount;
1565   Value *InnerStep = ConstantInt::get(LCTy, 3);
1566
1567   // Fix an insertion point for ComputeIP.
1568   BasicBlock *LoopNextEnter =
1569       BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1570                          Builder.GetInsertBlock()->getNextNode());
1571   BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1572   InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1573
1574   InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()};
1575   OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL});
1576
1577   BasicBlock *BodyCode = nullptr;
1578   CanonicalLoopInfo *InnerLoop = nullptr;
1579   CallInst *Call = nullptr;
1580   auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1581                                 llvm::Value *OuterLC) {
1582     auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1583                                   llvm::Value *InnerLC) {
1584       Builder.restoreIP(InnerCodeGenIP);
1585       BodyCode = Builder.GetInsertBlock();
1586
1587       // Add something that consumes the induction variable to the body.
1588       Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1589     };
1590     InnerLoop = OMPBuilder.createCanonicalLoop(
1591         OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal,
1592         InnerStep, false, false, ComputeIP, "inner");
1593   };
1594   CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1595       Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false,
1596       false, ComputeIP, "outer");
1597
1598   // Finalize the function
1599   Builder.restoreIP(OuterLoop->getAfterIP());
1600   Builder.CreateRetVoid();
1601
1602   // Tile the loop nest.
1603   Constant *TileSize0 = ConstantInt::get(LCTy, APInt(32, 11));
1604   Constant *TileSize1 = ConstantInt::get(LCTy, APInt(32, 7));
1605   std::vector<CanonicalLoopInfo *> GenLoops =
1606       OMPBuilder.tileLoops(DL, {OuterLoop, InnerLoop}, {TileSize0, TileSize1});
1607
1608   OMPBuilder.finalize();
1609   EXPECT_FALSE(verifyModule(*M, &errs()));
1610
1611   EXPECT_EQ(GenLoops.size(), 4u);
1612   CanonicalLoopInfo *Floor0 = GenLoops[0];
1613   CanonicalLoopInfo *Floor1 = GenLoops[1];
1614   CanonicalLoopInfo *Tile0 = GenLoops[2];
1615   CanonicalLoopInfo *Tile1 = GenLoops[3];
1616
1617   BasicBlock *RefOrder[] = {
1618       Floor0->getPreheader(),
1619       Floor0->getHeader(),
1620       Floor0->getCond(),
1621       Floor0->getBody(),
1622       Floor1->getPreheader(),
1623       Floor1->getHeader(),
1624       Floor1->getCond(),
1625       Floor1->getBody(),
1626       Tile0->getPreheader(),
1627       Tile0->getHeader(),
1628       Tile0->getCond(),
1629       Tile0->getBody(),
1630       Tile1->getPreheader(),
1631       Tile1->getHeader(),
1632       Tile1->getCond(),
1633       Tile1->getBody(),
1634       BodyCode,
1635       Tile1->getLatch(),
1636       Tile1->getExit(),
1637       Tile1->getAfter(),
1638       Tile0->getLatch(),
1639       Tile0->getExit(),
1640       Tile0->getAfter(),
1641       Floor1->getLatch(),
1642       Floor1->getExit(),
1643       Floor1->getAfter(),
1644       Floor0->getLatch(),
1645       Floor0->getExit(),
1646       Floor0->getAfter(),
1647   };
1648   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1649   EXPECT_TRUE(verifyListOrder(F, RefOrder));
1650
1651   EXPECT_EQ(Call->getParent(), BodyCode);
1652
1653   auto *RangeShift0 = cast<AddOperator>(Call->getOperand(1));
1654   EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal);
1655   auto *RangeScale0 = cast<MulOperator>(RangeShift0->getOperand(0));
1656   EXPECT_EQ(RangeScale0->getOperand(1), OuterStep);
1657   auto *TileShift0 = cast<AddOperator>(RangeScale0->getOperand(0));
1658   EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody());
1659   EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar());
1660   auto *TileScale0 = cast<MulOperator>(TileShift0->getOperand(0));
1661   EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody());
1662   EXPECT_EQ(TileScale0->getOperand(0), TileSize0);
1663   EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar());
1664
1665   auto *RangeShift1 = cast<AddOperator>(Call->getOperand(2));
1666   EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode);
1667   EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal);
1668   auto *RangeScale1 = cast<MulOperator>(RangeShift1->getOperand(0));
1669   EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode);
1670   EXPECT_EQ(RangeScale1->getOperand(1), InnerStep);
1671   auto *TileShift1 = cast<AddOperator>(RangeScale1->getOperand(0));
1672   EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody());
1673   EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar());
1674   auto *TileScale1 = cast<MulOperator>(TileShift1->getOperand(0));
1675   EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody());
1676   EXPECT_EQ(TileScale1->getOperand(0), TileSize1);
1677   EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar());
1678 }
1679
1680 TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {
1681   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1682   OpenMPIRBuilder OMPBuilder(*M);
1683   OMPBuilder.initialize();
1684   IRBuilder<> Builder(BB);
1685
1686   // Create a loop, tile it, and extract its trip count. All input values are
1687   // constant and IRBuilder evaluates all-constant arithmetic inplace, such that
1688   // the floor trip count itself will be a ConstantInt. Unfortunately we cannot
1689   // do the same for the tile loop.
1690   auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1691                            bool IsSigned, bool InclusiveStop,
1692                            int64_t TileSize) -> uint64_t {
1693     OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
1694     Type *LCTy = Type::getInt16Ty(Ctx);
1695     Value *StartVal = ConstantInt::get(LCTy, Start);
1696     Value *StopVal = ConstantInt::get(LCTy, Stop);
1697     Value *StepVal = ConstantInt::get(LCTy, Step);
1698
1699     // Generate a loop.
1700     auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1701     CanonicalLoopInfo *Loop =
1702         OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1703                                        StepVal, IsSigned, InclusiveStop);
1704     InsertPointTy AfterIP = Loop->getAfterIP();
1705
1706     // Tile the loop.
1707     Value *TileSizeVal = ConstantInt::get(LCTy, TileSize);
1708     std::vector<CanonicalLoopInfo *> GenLoops =
1709         OMPBuilder.tileLoops(Loc.DL, {Loop}, {TileSizeVal});
1710
1711     // Set the insertion pointer to after loop, where the next loop will be
1712     // emitted.
1713     Builder.restoreIP(AfterIP);
1714
1715     // Extract the trip count.
1716     CanonicalLoopInfo *FloorLoop = GenLoops[0];
1717     Value *FloorTripCount = FloorLoop->getTripCount();
1718     return cast<ConstantInt>(FloorTripCount)->getValue().getZExtValue();
1719   };
1720
1721   // Empty iteration domain.
1722   EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);
1723   EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);
1724   EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);
1725   EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);
1726   EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);
1727
1728   // Only complete tiles.
1729   EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1730   EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1731   EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);
1732   EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);
1733   EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);
1734   EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);
1735
1736   // Only a partial tile.
1737   EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);
1738   EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);
1739   EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);
1740   EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);
1741   EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);
1742
1743   // Complete and partial tiles.
1744   EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);
1745   EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);
1746   EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);
1747   EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);
1748   EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);
1749
1750   // Close to 16-bit integer range.
1751   EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu);
1752   EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1);
1753   EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1);
1754   EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1);
1755   EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1);
1756   EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);
1757   EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);
1758
1759   // Finalize the function.
1760   Builder.CreateRetVoid();
1761   OMPBuilder.finalize();
1762
1763   EXPECT_FALSE(verifyModule(*M, &errs()));
1764 }
1765
1766 TEST_F(OpenMPIRBuilderTest, ApplySimd) {
1767   OpenMPIRBuilder OMPBuilder(*M);
1768
1769   CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1770
1771   // Simd-ize the loop.
1772   OMPBuilder.applySimd(DL, CLI);
1773
1774   OMPBuilder.finalize();
1775   EXPECT_FALSE(verifyModule(*M, &errs()));
1776
1777   PassBuilder PB;
1778   FunctionAnalysisManager FAM;
1779   PB.registerFunctionAnalyses(FAM);
1780   LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1781
1782   const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1783   EXPECT_EQ(TopLvl.size(), 1u);
1784
1785   Loop *L = TopLvl.front();
1786   EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1787   EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1788
1789   // Check for llvm.access.group metadata attached to the printf
1790   // function in the loop body.
1791   BasicBlock *LoopBody = CLI->getBody();
1792   EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1793     return I.getMetadata("llvm.access.group") != nullptr;
1794   }));
1795 }
1796
1797 TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) {
1798   OpenMPIRBuilder OMPBuilder(*M);
1799
1800   CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1801
1802   // Unroll the loop.
1803   OMPBuilder.unrollLoopFull(DL, CLI);
1804
1805   OMPBuilder.finalize();
1806   EXPECT_FALSE(verifyModule(*M, &errs()));
1807
1808   PassBuilder PB;
1809   FunctionAnalysisManager FAM;
1810   PB.registerFunctionAnalyses(FAM);
1811   LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1812
1813   const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1814   EXPECT_EQ(TopLvl.size(), 1u);
1815
1816   Loop *L = TopLvl.front();
1817   EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
1818   EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full"));
1819 }
1820
1821 TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) {
1822   OpenMPIRBuilder OMPBuilder(*M);
1823   CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1824
1825   // Unroll the loop.
1826   CanonicalLoopInfo *UnrolledLoop = nullptr;
1827   OMPBuilder.unrollLoopPartial(DL, CLI, 5, &UnrolledLoop);
1828   ASSERT_NE(UnrolledLoop, nullptr);
1829
1830   OMPBuilder.finalize();
1831   EXPECT_FALSE(verifyModule(*M, &errs()));
1832   UnrolledLoop->assertOK();
1833
1834   PassBuilder PB;
1835   FunctionAnalysisManager FAM;
1836   PB.registerFunctionAnalyses(FAM);
1837   LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1838
1839   const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1840   EXPECT_EQ(TopLvl.size(), 1u);
1841   Loop *Outer = TopLvl.front();
1842   EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader());
1843   EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch());
1844   EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond());
1845   EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit());
1846
1847   EXPECT_EQ(Outer->getSubLoops().size(), 1u);
1848   Loop *Inner = Outer->getSubLoops().front();
1849
1850   EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable"));
1851   EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5);
1852 }
1853
1854 TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) {
1855   OpenMPIRBuilder OMPBuilder(*M);
1856
1857   CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1858
1859   // Unroll the loop.
1860   OMPBuilder.unrollLoopHeuristic(DL, CLI);
1861
1862   OMPBuilder.finalize();
1863   EXPECT_FALSE(verifyModule(*M, &errs()));
1864
1865   PassBuilder PB;
1866   FunctionAnalysisManager FAM;
1867   PB.registerFunctionAnalyses(FAM);
1868   LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1869
1870   const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1871   EXPECT_EQ(TopLvl.size(), 1u);
1872
1873   Loop *L = TopLvl.front();
1874   EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
1875 }
1876
1877 TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
1878   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1879   OpenMPIRBuilder OMPBuilder(*M);
1880   OMPBuilder.initialize();
1881   IRBuilder<> Builder(BB);
1882   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1883
1884   Type *LCTy = Type::getInt32Ty(Ctx);
1885   Value *StartVal = ConstantInt::get(LCTy, 10);
1886   Value *StopVal = ConstantInt::get(LCTy, 52);
1887   Value *StepVal = ConstantInt::get(LCTy, 2);
1888   auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
1889
1890   CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
1891       Loc, LoopBodyGen, StartVal, StopVal, StepVal,
1892       /*IsSigned=*/false, /*InclusiveStop=*/false);
1893   BasicBlock *Preheader = CLI->getPreheader();
1894   BasicBlock *Body = CLI->getBody();
1895   Value *IV = CLI->getIndVar();
1896   BasicBlock *ExitBlock = CLI->getExit();
1897
1898   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
1899   InsertPointTy AllocaIP = Builder.saveIP();
1900
1901   OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
1902                                 OMP_SCHEDULE_Static);
1903
1904   BasicBlock *Cond = Body->getSinglePredecessor();
1905   Instruction *Cmp = &*Cond->begin();
1906   Value *TripCount = Cmp->getOperand(1);
1907
1908   auto AllocaIter = BB->begin();
1909   ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
1910   AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
1911   AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
1912   AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
1913   AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
1914   EXPECT_NE(PLastIter, nullptr);
1915   EXPECT_NE(PLowerBound, nullptr);
1916   EXPECT_NE(PUpperBound, nullptr);
1917   EXPECT_NE(PStride, nullptr);
1918
1919   auto PreheaderIter = Preheader->begin();
1920   ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7);
1921   StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
1922   StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
1923   StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
1924   ASSERT_NE(LowerBoundStore, nullptr);
1925   ASSERT_NE(UpperBoundStore, nullptr);
1926   ASSERT_NE(StrideStore, nullptr);
1927
1928   auto *OrigLowerBound =
1929       dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
1930   auto *OrigUpperBound =
1931       dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
1932   auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand());
1933   ASSERT_NE(OrigLowerBound, nullptr);
1934   ASSERT_NE(OrigUpperBound, nullptr);
1935   ASSERT_NE(OrigStride, nullptr);
1936   EXPECT_EQ(OrigLowerBound->getValue(), 0);
1937   EXPECT_EQ(OrigUpperBound->getValue(), 20);
1938   EXPECT_EQ(OrigStride->getValue(), 1);
1939
1940   // Check that the loop IV is updated to account for the lower bound returned
1941   // by the OpenMP runtime call.
1942   BinaryOperator *Add = dyn_cast<BinaryOperator>(&Body->front());
1943   EXPECT_EQ(Add->getOperand(0), IV);
1944   auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1));
1945   ASSERT_NE(LoadedLowerBound, nullptr);
1946   EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);
1947
1948   // Check that the trip count is updated to account for the lower and upper
1949   // bounds return by the OpenMP runtime call.
1950   auto *AddOne = dyn_cast<Instruction>(TripCount);
1951   ASSERT_NE(AddOne, nullptr);
1952   ASSERT_TRUE(AddOne->isBinaryOp());
1953   auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1));
1954   ASSERT_NE(One, nullptr);
1955   EXPECT_EQ(One->getValue(), 1);
1956   auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0));
1957   ASSERT_NE(Difference, nullptr);
1958   ASSERT_TRUE(Difference->isBinaryOp());
1959   EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);
1960   auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0));
1961   ASSERT_NE(LoadedUpperBound, nullptr);
1962   EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);
1963
1964   // The original loop iterator should only be used in the condition, in the
1965   // increment and in the statement that adds the lower bound to it.
1966   EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
1967
1968   // The exit block should contain the "fini" call and the barrier call,
1969   // plus the call to obtain the thread ID.
1970   size_t NumCallsInExitBlock =
1971       count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
1972   EXPECT_EQ(NumCallsInExitBlock, 3u);
1973 }
1974
1975 TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {
1976   unsigned IVBits = GetParam();
1977
1978   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1979   OpenMPIRBuilder OMPBuilder(*M);
1980
1981   BasicBlock *Body;
1982   CallInst *Call;
1983   CanonicalLoopInfo *CLI =
1984       buildSingleLoopFunction(DL, OMPBuilder, IVBits, &Call, &Body);
1985
1986   Instruction *OrigIndVar = CLI->getIndVar();
1987   EXPECT_EQ(Call->getOperand(1), OrigIndVar);
1988
1989   Type *LCTy = Type::getInt32Ty(Ctx);
1990   Value *ChunkSize = ConstantInt::get(LCTy, 5);
1991   InsertPointTy AllocaIP{&F->getEntryBlock(),
1992                          F->getEntryBlock().getFirstInsertionPt()};
1993   OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
1994                                 OMP_SCHEDULE_Static, ChunkSize);
1995
1996   OMPBuilder.finalize();
1997   EXPECT_FALSE(verifyModule(*M, &errs()));
1998
1999   BasicBlock *Entry = &F->getEntryBlock();
2000   BasicBlock *Preheader = Entry->getSingleSuccessor();
2001
2002   BasicBlock *DispatchPreheader = Preheader->getSingleSuccessor();
2003   BasicBlock *DispatchHeader = DispatchPreheader->getSingleSuccessor();
2004   BasicBlock *DispatchCond = DispatchHeader->getSingleSuccessor();
2005   BasicBlock *DispatchBody = succ_begin(DispatchCond)[0];
2006   BasicBlock *DispatchExit = succ_begin(DispatchCond)[1];
2007   BasicBlock *DispatchAfter = DispatchExit->getSingleSuccessor();
2008   BasicBlock *Return = DispatchAfter->getSingleSuccessor();
2009
2010   BasicBlock *ChunkPreheader = DispatchBody->getSingleSuccessor();
2011   BasicBlock *ChunkHeader = ChunkPreheader->getSingleSuccessor();
2012   BasicBlock *ChunkCond = ChunkHeader->getSingleSuccessor();
2013   BasicBlock *ChunkBody = succ_begin(ChunkCond)[0];
2014   BasicBlock *ChunkExit = succ_begin(ChunkCond)[1];
2015   BasicBlock *ChunkInc = ChunkBody->getSingleSuccessor();
2016   BasicBlock *ChunkAfter = ChunkExit->getSingleSuccessor();
2017
2018   BasicBlock *DispatchInc = ChunkAfter;
2019
2020   EXPECT_EQ(ChunkBody, Body);
2021   EXPECT_EQ(ChunkInc->getSingleSuccessor(), ChunkHeader);
2022   EXPECT_EQ(DispatchInc->getSingleSuccessor(), DispatchHeader);
2023
2024   EXPECT_TRUE(isa<ReturnInst>(Return->front()));
2025
2026   Value *NewIV = Call->getOperand(1);
2027   EXPECT_EQ(NewIV->getType()->getScalarSizeInBits(), IVBits);
2028
2029   CallInst *InitCall = findSingleCall(
2030       F,
2031       (IVBits > 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u
2032                     : omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u,
2033       OMPBuilder);
2034   EXPECT_EQ(InitCall->getParent(), Preheader);
2035   EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(2))->getSExtValue(), 33);
2036   EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(7))->getSExtValue(), 1);
2037   EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(8))->getSExtValue(), 5);
2038
2039   CallInst *FiniCall = findSingleCall(
2040       F, omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini, OMPBuilder);
2041   EXPECT_EQ(FiniCall->getParent(), DispatchExit);
2042
2043   CallInst *BarrierCall = findSingleCall(
2044       F, omp::RuntimeFunction::OMPRTL___kmpc_barrier, OMPBuilder);
2045   EXPECT_EQ(BarrierCall->getParent(), DispatchExit);
2046 }
2047
2048 INSTANTIATE_TEST_SUITE_P(IVBits, OpenMPIRBuilderTestWithIVBits,
2049                          ::testing::Values(8, 16, 32, 64));
2050
2051 TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
2052   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2053   OpenMPIRBuilder OMPBuilder(*M);
2054   OMPBuilder.initialize();
2055   IRBuilder<> Builder(BB);
2056   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2057
2058   omp::OMPScheduleType SchedType = GetParam();
2059   uint32_t ChunkSize = 1;
2060   switch (SchedType & ~OMPScheduleType::ModifierMask) {
2061   case omp::OMPScheduleType::BaseDynamicChunked:
2062   case omp::OMPScheduleType::BaseGuidedChunked:
2063     ChunkSize = 7;
2064     break;
2065   case omp::OMPScheduleType::BaseAuto:
2066   case omp::OMPScheduleType::BaseRuntime:
2067     ChunkSize = 1;
2068     break;
2069   default:
2070     assert(0 && "unknown type for this test");
2071     break;
2072   }
2073
2074   Type *LCTy = Type::getInt32Ty(Ctx);
2075   Value *StartVal = ConstantInt::get(LCTy, 10);
2076   Value *StopVal = ConstantInt::get(LCTy, 52);
2077   Value *StepVal = ConstantInt::get(LCTy, 2);
2078   Value *ChunkVal =
2079       (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize);
2080   auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2081
2082   CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2083       Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2084       /*IsSigned=*/false, /*InclusiveStop=*/false);
2085
2086   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2087   InsertPointTy AllocaIP = Builder.saveIP();
2088
2089   // Collect all the info from CLI, as it isn't usable after the call to
2090   // createDynamicWorkshareLoop.
2091   InsertPointTy AfterIP = CLI->getAfterIP();
2092   BasicBlock *Preheader = CLI->getPreheader();
2093   BasicBlock *ExitBlock = CLI->getExit();
2094   BasicBlock *LatchBlock = CLI->getLatch();
2095   Value *IV = CLI->getIndVar();
2096
2097   InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2098       DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),
2099       ChunkVal, /*Simd=*/false,
2100       (SchedType & omp::OMPScheduleType::ModifierMonotonic) ==
2101           omp::OMPScheduleType::ModifierMonotonic,
2102       (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==
2103           omp::OMPScheduleType::ModifierNonmonotonic,
2104       /*Ordered=*/false);
2105
2106   // The returned value should be the "after" point.
2107   ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());
2108   ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());
2109
2110   auto AllocaIter = BB->begin();
2111   ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2112   AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
2113   AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2114   AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2115   AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
2116   EXPECT_NE(PLastIter, nullptr);
2117   EXPECT_NE(PLowerBound, nullptr);
2118   EXPECT_NE(PUpperBound, nullptr);
2119   EXPECT_NE(PStride, nullptr);
2120
2121   auto PreheaderIter = Preheader->begin();
2122   ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6);
2123   StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2124   StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2125   StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2126   ASSERT_NE(LowerBoundStore, nullptr);
2127   ASSERT_NE(UpperBoundStore, nullptr);
2128   ASSERT_NE(StrideStore, nullptr);
2129
2130   CallInst *ThreadIdCall = dyn_cast<CallInst>(&*(PreheaderIter++));
2131   ASSERT_NE(ThreadIdCall, nullptr);
2132   EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(),
2133             "__kmpc_global_thread_num");
2134
2135   CallInst *InitCall = dyn_cast<CallInst>(&*PreheaderIter);
2136
2137   ASSERT_NE(InitCall, nullptr);
2138   EXPECT_EQ(InitCall->getCalledFunction()->getName(),
2139             "__kmpc_dispatch_init_4u");
2140   EXPECT_EQ(InitCall->arg_size(), 7U);
2141   EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize));
2142   ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2143   if ((SchedType & OMPScheduleType::MonotonicityMask) ==
2144       OMPScheduleType::None) {
2145     // Implementation is allowed to add default nonmonotonicity flag
2146     EXPECT_EQ(
2147         static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()) |
2148             OMPScheduleType::ModifierNonmonotonic,
2149         SchedType | OMPScheduleType::ModifierNonmonotonic);
2150   } else {
2151     EXPECT_EQ(static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()),
2152               SchedType);
2153   }
2154
2155   ConstantInt *OrigLowerBound =
2156       dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
2157   ConstantInt *OrigUpperBound =
2158       dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
2159   ConstantInt *OrigStride =
2160       dyn_cast<ConstantInt>(StrideStore->getValueOperand());
2161   ASSERT_NE(OrigLowerBound, nullptr);
2162   ASSERT_NE(OrigUpperBound, nullptr);
2163   ASSERT_NE(OrigStride, nullptr);
2164   EXPECT_EQ(OrigLowerBound->getValue(), 1);
2165   EXPECT_EQ(OrigUpperBound->getValue(), 21);
2166   EXPECT_EQ(OrigStride->getValue(), 1);
2167
2168   CallInst *FiniCall = dyn_cast<CallInst>(
2169       &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2170   EXPECT_EQ(FiniCall, nullptr);
2171
2172   // The original loop iterator should only be used in the condition, in the
2173   // increment and in the statement that adds the lower bound to it.
2174   EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2175
2176   // The exit block should contain the barrier call, plus the call to obtain
2177   // the thread ID.
2178   size_t NumCallsInExitBlock =
2179       count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2180   EXPECT_EQ(NumCallsInExitBlock, 2u);
2181
2182   // Add a termination to our block and check that it is internally consistent.
2183   Builder.restoreIP(EndIP);
2184   Builder.CreateRetVoid();
2185   OMPBuilder.finalize();
2186   EXPECT_FALSE(verifyModule(*M, &errs()));
2187 }
2188
2189 INSTANTIATE_TEST_SUITE_P(
2190     OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams,
2191     ::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked,
2192                       omp::OMPScheduleType::UnorderedGuidedChunked,
2193                       omp::OMPScheduleType::UnorderedAuto,
2194                       omp::OMPScheduleType::UnorderedRuntime,
2195                       omp::OMPScheduleType::UnorderedDynamicChunked |
2196                           omp::OMPScheduleType::ModifierMonotonic,
2197                       omp::OMPScheduleType::UnorderedDynamicChunked |
2198                           omp::OMPScheduleType::ModifierNonmonotonic,
2199                       omp::OMPScheduleType::UnorderedGuidedChunked |
2200                           omp::OMPScheduleType::ModifierMonotonic,
2201                       omp::OMPScheduleType::UnorderedGuidedChunked |
2202                           omp::OMPScheduleType::ModifierNonmonotonic,
2203                       omp::OMPScheduleType::UnorderedAuto |
2204                           omp::OMPScheduleType::ModifierMonotonic,
2205                       omp::OMPScheduleType::UnorderedRuntime |
2206                           omp::OMPScheduleType::ModifierMonotonic));
2207
2208 TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
2209   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2210   OpenMPIRBuilder OMPBuilder(*M);
2211   OMPBuilder.initialize();
2212   IRBuilder<> Builder(BB);
2213   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2214
2215   uint32_t ChunkSize = 1;
2216   Type *LCTy = Type::getInt32Ty(Ctx);
2217   Value *StartVal = ConstantInt::get(LCTy, 10);
2218   Value *StopVal = ConstantInt::get(LCTy, 52);
2219   Value *StepVal = ConstantInt::get(LCTy, 2);
2220   Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize);
2221   auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2222
2223   CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2224       Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2225       /*IsSigned=*/false, /*InclusiveStop=*/false);
2226
2227   Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2228   InsertPointTy AllocaIP = Builder.saveIP();
2229
2230   // Collect all the info from CLI, as it isn't usable after the call to
2231   // createDynamicWorkshareLoop.
2232   BasicBlock *Preheader = CLI->getPreheader();
2233   BasicBlock *ExitBlock = CLI->getExit();
2234   BasicBlock *LatchBlock = CLI->getLatch();
2235   Value *IV = CLI->getIndVar();
2236
2237   InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2238       DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal,
2239       /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
2240       /*HasNonmonotonicModifier=*/false,
2241       /*HasOrderedClause=*/true);
2242
2243   // Add a termination to our block and check that it is internally consistent.
2244   Builder.restoreIP(EndIP);
2245   Builder.CreateRetVoid();
2246   OMPBuilder.finalize();
2247   EXPECT_FALSE(verifyModule(*M, &errs()));
2248
2249   CallInst *InitCall = nullptr;
2250   for (Instruction &EI : *Preheader) {
2251     Instruction *Cur = &EI;
2252     if (isa<CallInst>(Cur)) {
2253       InitCall = cast<CallInst>(Cur);
2254       if (InitCall->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")
2255         break;
2256       InitCall = nullptr;
2257     }
2258   }
2259   EXPECT_NE(InitCall, nullptr);
2260   EXPECT_EQ(InitCall->arg_size(), 7U);
2261   ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2262   EXPECT_EQ(SchedVal->getValue(),
2263             static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked));
2264
2265   CallInst *FiniCall = dyn_cast<CallInst>(
2266       &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2267   ASSERT_NE(FiniCall, nullptr);
2268   EXPECT_EQ(FiniCall->getCalledFunction()->getName(),
2269             "__kmpc_dispatch_fini_4u");
2270   EXPECT_EQ(FiniCall->arg_size(), 2U);
2271   EXPECT_EQ(InitCall->getArgOperand(0), FiniCall->getArgOperand(0));
2272   EXPECT_EQ(InitCall->getArgOperand(1), FiniCall->getArgOperand(1));
2273
2274   // The original loop iterator should only be used in the condition, in the
2275   // increment and in the statement that adds the lower bound to it.
2276   EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2277
2278   // The exit block should contain the barrier call, plus the call to obtain
2279   // the thread ID.
2280   size_t NumCallsInExitBlock =
2281       count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2282   EXPECT_EQ(NumCallsInExitBlock, 2u);
2283 }
2284
2285 TEST_F(OpenMPIRBuilderTest, MasterDirective) {
2286   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2287   OpenMPIRBuilder OMPBuilder(*M);
2288   OMPBuilder.initialize();
2289   F->setName("func");
2290   IRBuilder<> Builder(BB);
2291
2292   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2293
2294   AllocaInst *PrivAI = nullptr;
2295
2296   BasicBlock *EntryBB = nullptr;
2297   BasicBlock *ThenBB = nullptr;
2298
2299   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2300     if (AllocaIP.isSet())
2301       Builder.restoreIP(AllocaIP);
2302     else
2303       Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2304     PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2305     Builder.CreateStore(F->arg_begin(), PrivAI);
2306
2307     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2308     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2309     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2310
2311     Builder.restoreIP(CodeGenIP);
2312
2313     // collect some info for checks later
2314     ThenBB = Builder.GetInsertBlock();
2315     EntryBB = ThenBB->getUniquePredecessor();
2316
2317     // simple instructions for body
2318     Value *PrivLoad =
2319         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2320     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2321   };
2322
2323   auto FiniCB = [&](InsertPointTy IP) {
2324     BasicBlock *IPBB = IP.getBlock();
2325     EXPECT_NE(IPBB->end(), IP.getPoint());
2326   };
2327
2328   Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
2329   Value *EntryBBTI = EntryBB->getTerminator();
2330   EXPECT_NE(EntryBBTI, nullptr);
2331   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2332   BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2333   EXPECT_TRUE(EntryBr->isConditional());
2334   EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2335   BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2336   EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2337
2338   CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2339   EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2340
2341   CallInst *MasterEntryCI = cast<CallInst>(CondInst->getOperand(0));
2342   EXPECT_EQ(MasterEntryCI->arg_size(), 2U);
2343   EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master");
2344   EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0)));
2345
2346   CallInst *MasterEndCI = nullptr;
2347   for (auto &FI : *ThenBB) {
2348     Instruction *cur = &FI;
2349     if (isa<CallInst>(cur)) {
2350       MasterEndCI = cast<CallInst>(cur);
2351       if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master")
2352         break;
2353       MasterEndCI = nullptr;
2354     }
2355   }
2356   EXPECT_NE(MasterEndCI, nullptr);
2357   EXPECT_EQ(MasterEndCI->arg_size(), 2U);
2358   EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0)));
2359   EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1));
2360 }
2361
2362 TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
2363   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2364   OpenMPIRBuilder OMPBuilder(*M);
2365   OMPBuilder.initialize();
2366   F->setName("func");
2367   IRBuilder<> Builder(BB);
2368
2369   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2370
2371   AllocaInst *PrivAI = nullptr;
2372
2373   BasicBlock *EntryBB = nullptr;
2374   BasicBlock *ThenBB = nullptr;
2375
2376   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2377     if (AllocaIP.isSet())
2378       Builder.restoreIP(AllocaIP);
2379     else
2380       Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2381     PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2382     Builder.CreateStore(F->arg_begin(), PrivAI);
2383
2384     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2385     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2386     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2387
2388     Builder.restoreIP(CodeGenIP);
2389
2390     // collect some info for checks later
2391     ThenBB = Builder.GetInsertBlock();
2392     EntryBB = ThenBB->getUniquePredecessor();
2393
2394     // simple instructions for body
2395     Value *PrivLoad =
2396         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2397     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2398   };
2399
2400   auto FiniCB = [&](InsertPointTy IP) {
2401     BasicBlock *IPBB = IP.getBlock();
2402     EXPECT_NE(IPBB->end(), IP.getPoint());
2403   };
2404
2405   Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
2406   Builder.restoreIP(
2407       OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter));
2408   Value *EntryBBTI = EntryBB->getTerminator();
2409   EXPECT_NE(EntryBBTI, nullptr);
2410   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2411   BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2412   EXPECT_TRUE(EntryBr->isConditional());
2413   EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2414   BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2415   EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2416
2417   CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2418   EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2419
2420   CallInst *MaskedEntryCI = cast<CallInst>(CondInst->getOperand(0));
2421   EXPECT_EQ(MaskedEntryCI->arg_size(), 3U);
2422   EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked");
2423   EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0)));
2424
2425   CallInst *MaskedEndCI = nullptr;
2426   for (auto &FI : *ThenBB) {
2427     Instruction *cur = &FI;
2428     if (isa<CallInst>(cur)) {
2429       MaskedEndCI = cast<CallInst>(cur);
2430       if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked")
2431         break;
2432       MaskedEndCI = nullptr;
2433     }
2434   }
2435   EXPECT_NE(MaskedEndCI, nullptr);
2436   EXPECT_EQ(MaskedEndCI->arg_size(), 2U);
2437   EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0)));
2438   EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1));
2439 }
2440
2441 TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
2442   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2443   OpenMPIRBuilder OMPBuilder(*M);
2444   OMPBuilder.initialize();
2445   F->setName("func");
2446   IRBuilder<> Builder(BB);
2447
2448   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2449
2450   AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2451
2452   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2453     // actual start for bodyCB
2454     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2455     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2456     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2457
2458     // body begin
2459     Builder.restoreIP(CodeGenIP);
2460     Builder.CreateStore(F->arg_begin(), PrivAI);
2461     Value *PrivLoad =
2462         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2463     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2464   };
2465
2466   auto FiniCB = [&](InsertPointTy IP) {
2467     BasicBlock *IPBB = IP.getBlock();
2468     EXPECT_NE(IPBB->end(), IP.getPoint());
2469   };
2470   BasicBlock *EntryBB = Builder.GetInsertBlock();
2471
2472   Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
2473                                               "testCRT", nullptr));
2474
2475   CallInst *CriticalEntryCI = nullptr;
2476   for (auto &EI : *EntryBB) {
2477     Instruction *cur = &EI;
2478     if (isa<CallInst>(cur)) {
2479       CriticalEntryCI = cast<CallInst>(cur);
2480       if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical")
2481         break;
2482       CriticalEntryCI = nullptr;
2483     }
2484   }
2485   EXPECT_NE(CriticalEntryCI, nullptr);
2486   EXPECT_EQ(CriticalEntryCI->arg_size(), 3U);
2487   EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical");
2488   EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0)));
2489
2490   CallInst *CriticalEndCI = nullptr;
2491   for (auto &FI : *EntryBB) {
2492     Instruction *cur = &FI;
2493     if (isa<CallInst>(cur)) {
2494       CriticalEndCI = cast<CallInst>(cur);
2495       if (CriticalEndCI->getCalledFunction()->getName() ==
2496           "__kmpc_end_critical")
2497         break;
2498       CriticalEndCI = nullptr;
2499     }
2500   }
2501   EXPECT_NE(CriticalEndCI, nullptr);
2502   EXPECT_EQ(CriticalEndCI->arg_size(), 3U);
2503   EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0)));
2504   EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1));
2505   PointerType *CriticalNamePtrTy =
2506       PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx), 8));
2507   EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2));
2508   EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy);
2509 }
2510
2511 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) {
2512   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2513   OpenMPIRBuilder OMPBuilder(*M);
2514   OMPBuilder.initialize();
2515   F->setName("func");
2516   IRBuilder<> Builder(BB);
2517   LLVMContext &Ctx = M->getContext();
2518
2519   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2520
2521   InsertPointTy AllocaIP(&F->getEntryBlock(),
2522                          F->getEntryBlock().getFirstInsertionPt());
2523
2524   unsigned NumLoops = 2;
2525   SmallVector<Value *, 2> StoreValues;
2526   Type *LCTy = Type::getInt64Ty(Ctx);
2527   StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2528   StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2529
2530   // Test for "#omp ordered depend(source)"
2531   Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2532                                                    StoreValues, ".cnt.addr",
2533                                                    /*IsDependSource=*/true));
2534
2535   Builder.CreateRetVoid();
2536   OMPBuilder.finalize();
2537   EXPECT_FALSE(verifyModule(*M, &errs()));
2538
2539   AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2540   ASSERT_NE(AllocInst, nullptr);
2541   ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2542   EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2543   EXPECT_TRUE(
2544       AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2545
2546   Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2547   for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2548     GetElementPtrInst *DependAddrGEPIter =
2549         dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2550     ASSERT_NE(DependAddrGEPIter, nullptr);
2551     EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2552     EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2553     auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2554     auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2555     ASSERT_NE(FirstIdx, nullptr);
2556     ASSERT_NE(SecondIdx, nullptr);
2557     EXPECT_EQ(FirstIdx->getValue(), 0);
2558     EXPECT_EQ(SecondIdx->getValue(), Iter);
2559     StoreInst *StoreValue =
2560         dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2561     ASSERT_NE(StoreValue, nullptr);
2562     EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2563     EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2564     EXPECT_EQ(StoreValue->getAlign(), Align(8));
2565     IterInst = dyn_cast<Instruction>(StoreValue);
2566   }
2567
2568   GetElementPtrInst *DependBaseAddrGEP =
2569       dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2570   ASSERT_NE(DependBaseAddrGEP, nullptr);
2571   EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2572   EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2573   auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2574   auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2575   ASSERT_NE(FirstIdx, nullptr);
2576   ASSERT_NE(SecondIdx, nullptr);
2577   EXPECT_EQ(FirstIdx->getValue(), 0);
2578   EXPECT_EQ(SecondIdx->getValue(), 0);
2579
2580   CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2581   ASSERT_NE(GTID, nullptr);
2582   EXPECT_EQ(GTID->arg_size(), 1U);
2583   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2584   EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2585   EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2586
2587   CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2588   ASSERT_NE(Depend, nullptr);
2589   EXPECT_EQ(Depend->arg_size(), 3U);
2590   EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post");
2591   EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2592   EXPECT_EQ(Depend->getArgOperand(1), GTID);
2593   EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2594 }
2595
2596 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) {
2597   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2598   OpenMPIRBuilder OMPBuilder(*M);
2599   OMPBuilder.initialize();
2600   F->setName("func");
2601   IRBuilder<> Builder(BB);
2602   LLVMContext &Ctx = M->getContext();
2603
2604   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2605
2606   InsertPointTy AllocaIP(&F->getEntryBlock(),
2607                          F->getEntryBlock().getFirstInsertionPt());
2608
2609   unsigned NumLoops = 2;
2610   SmallVector<Value *, 2> StoreValues;
2611   Type *LCTy = Type::getInt64Ty(Ctx);
2612   StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2613   StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2614
2615   // Test for "#omp ordered depend(sink: vec)"
2616   Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2617                                                    StoreValues, ".cnt.addr",
2618                                                    /*IsDependSource=*/false));
2619
2620   Builder.CreateRetVoid();
2621   OMPBuilder.finalize();
2622   EXPECT_FALSE(verifyModule(*M, &errs()));
2623
2624   AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2625   ASSERT_NE(AllocInst, nullptr);
2626   ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2627   EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2628   EXPECT_TRUE(
2629       AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2630
2631   Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2632   for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2633     GetElementPtrInst *DependAddrGEPIter =
2634         dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2635     ASSERT_NE(DependAddrGEPIter, nullptr);
2636     EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2637     EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2638     auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2639     auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2640     ASSERT_NE(FirstIdx, nullptr);
2641     ASSERT_NE(SecondIdx, nullptr);
2642     EXPECT_EQ(FirstIdx->getValue(), 0);
2643     EXPECT_EQ(SecondIdx->getValue(), Iter);
2644     StoreInst *StoreValue =
2645         dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2646     ASSERT_NE(StoreValue, nullptr);
2647     EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2648     EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2649     EXPECT_EQ(StoreValue->getAlign(), Align(8));
2650     IterInst = dyn_cast<Instruction>(StoreValue);
2651   }
2652
2653   GetElementPtrInst *DependBaseAddrGEP =
2654       dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2655   ASSERT_NE(DependBaseAddrGEP, nullptr);
2656   EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2657   EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2658   auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2659   auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2660   ASSERT_NE(FirstIdx, nullptr);
2661   ASSERT_NE(SecondIdx, nullptr);
2662   EXPECT_EQ(FirstIdx->getValue(), 0);
2663   EXPECT_EQ(SecondIdx->getValue(), 0);
2664
2665   CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2666   ASSERT_NE(GTID, nullptr);
2667   EXPECT_EQ(GTID->arg_size(), 1U);
2668   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2669   EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2670   EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2671
2672   CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2673   ASSERT_NE(Depend, nullptr);
2674   EXPECT_EQ(Depend->arg_size(), 3U);
2675   EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait");
2676   EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2677   EXPECT_EQ(Depend->getArgOperand(1), GTID);
2678   EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2679 }
2680
2681 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
2682   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2683   OpenMPIRBuilder OMPBuilder(*M);
2684   OMPBuilder.initialize();
2685   F->setName("func");
2686   IRBuilder<> Builder(BB);
2687
2688   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2689
2690   AllocaInst *PrivAI =
2691       Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
2692
2693   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2694     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2695     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2696     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2697
2698     Builder.restoreIP(CodeGenIP);
2699     Builder.CreateStore(F->arg_begin(), PrivAI);
2700     Value *PrivLoad =
2701         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2702     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2703   };
2704
2705   auto FiniCB = [&](InsertPointTy IP) {
2706     BasicBlock *IPBB = IP.getBlock();
2707     EXPECT_NE(IPBB->end(), IP.getPoint());
2708   };
2709
2710   // Test for "#omp ordered [threads]"
2711   BasicBlock *EntryBB = Builder.GetInsertBlock();
2712   Builder.restoreIP(
2713       OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true));
2714
2715   Builder.CreateRetVoid();
2716   OMPBuilder.finalize();
2717   EXPECT_FALSE(verifyModule(*M, &errs()));
2718
2719   EXPECT_NE(EntryBB->getTerminator(), nullptr);
2720
2721   CallInst *OrderedEntryCI = nullptr;
2722   for (auto &EI : *EntryBB) {
2723     Instruction *Cur = &EI;
2724     if (isa<CallInst>(Cur)) {
2725       OrderedEntryCI = cast<CallInst>(Cur);
2726       if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
2727         break;
2728       OrderedEntryCI = nullptr;
2729     }
2730   }
2731   EXPECT_NE(OrderedEntryCI, nullptr);
2732   EXPECT_EQ(OrderedEntryCI->arg_size(), 2U);
2733   EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered");
2734   EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0)));
2735
2736   CallInst *OrderedEndCI = nullptr;
2737   for (auto &FI : *EntryBB) {
2738     Instruction *Cur = &FI;
2739     if (isa<CallInst>(Cur)) {
2740       OrderedEndCI = cast<CallInst>(Cur);
2741       if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
2742         break;
2743       OrderedEndCI = nullptr;
2744     }
2745   }
2746   EXPECT_NE(OrderedEndCI, nullptr);
2747   EXPECT_EQ(OrderedEndCI->arg_size(), 2U);
2748   EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0)));
2749   EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1));
2750 }
2751
2752 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
2753   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2754   OpenMPIRBuilder OMPBuilder(*M);
2755   OMPBuilder.initialize();
2756   F->setName("func");
2757   IRBuilder<> Builder(BB);
2758
2759   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2760
2761   AllocaInst *PrivAI =
2762       Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
2763
2764   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2765     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2766     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2767     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2768
2769     Builder.restoreIP(CodeGenIP);
2770     Builder.CreateStore(F->arg_begin(), PrivAI);
2771     Value *PrivLoad =
2772         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2773     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2774   };
2775
2776   auto FiniCB = [&](InsertPointTy IP) {
2777     BasicBlock *IPBB = IP.getBlock();
2778     EXPECT_NE(IPBB->end(), IP.getPoint());
2779   };
2780
2781   // Test for "#omp ordered simd"
2782   BasicBlock *EntryBB = Builder.GetInsertBlock();
2783   Builder.restoreIP(
2784       OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false));
2785
2786   Builder.CreateRetVoid();
2787   OMPBuilder.finalize();
2788   EXPECT_FALSE(verifyModule(*M, &errs()));
2789
2790   EXPECT_NE(EntryBB->getTerminator(), nullptr);
2791
2792   CallInst *OrderedEntryCI = nullptr;
2793   for (auto &EI : *EntryBB) {
2794     Instruction *Cur = &EI;
2795     if (isa<CallInst>(Cur)) {
2796       OrderedEntryCI = cast<CallInst>(Cur);
2797       if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
2798         break;
2799       OrderedEntryCI = nullptr;
2800     }
2801   }
2802   EXPECT_EQ(OrderedEntryCI, nullptr);
2803
2804   CallInst *OrderedEndCI = nullptr;
2805   for (auto &FI : *EntryBB) {
2806     Instruction *Cur = &FI;
2807     if (isa<CallInst>(Cur)) {
2808       OrderedEndCI = cast<CallInst>(Cur);
2809       if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
2810         break;
2811       OrderedEndCI = nullptr;
2812     }
2813   }
2814   EXPECT_EQ(OrderedEndCI, nullptr);
2815 }
2816
2817 TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {
2818   OpenMPIRBuilder OMPBuilder(*M);
2819   OMPBuilder.initialize();
2820   F->setName("func");
2821   IRBuilder<> Builder(BB);
2822
2823   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2824
2825   IntegerType *Int32 = Type::getInt32Ty(M->getContext());
2826   AllocaInst *MasterAddress = Builder.CreateAlloca(Int32->getPointerTo());
2827   AllocaInst *PrivAddress = Builder.CreateAlloca(Int32->getPointerTo());
2828
2829   BasicBlock *EntryBB = BB;
2830
2831   OMPBuilder.createCopyinClauseBlocks(Builder.saveIP(), MasterAddress,
2832                                       PrivAddress, Int32, /*BranchtoEnd*/ true);
2833
2834   BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator());
2835
2836   EXPECT_NE(EntryBr, nullptr);
2837   EXPECT_TRUE(EntryBr->isConditional());
2838
2839   BasicBlock *NotMasterBB = EntryBr->getSuccessor(0);
2840   BasicBlock *CopyinEnd = EntryBr->getSuccessor(1);
2841   CmpInst *CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition());
2842
2843   EXPECT_NE(CMP, nullptr);
2844   EXPECT_NE(NotMasterBB, nullptr);
2845   EXPECT_NE(CopyinEnd, nullptr);
2846
2847   BranchInst *NotMasterBr =
2848       dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator());
2849   EXPECT_NE(NotMasterBr, nullptr);
2850   EXPECT_FALSE(NotMasterBr->isConditional());
2851   EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0));
2852 }
2853
2854 TEST_F(OpenMPIRBuilderTest, SingleDirective) {
2855   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2856   OpenMPIRBuilder OMPBuilder(*M);
2857   OMPBuilder.initialize();
2858   F->setName("func");
2859   IRBuilder<> Builder(BB);
2860
2861   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2862
2863   AllocaInst *PrivAI = nullptr;
2864
2865   BasicBlock *EntryBB = nullptr;
2866   BasicBlock *ThenBB = nullptr;
2867
2868   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2869     if (AllocaIP.isSet())
2870       Builder.restoreIP(AllocaIP);
2871     else
2872       Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2873     PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2874     Builder.CreateStore(F->arg_begin(), PrivAI);
2875
2876     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2877     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2878     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2879
2880     Builder.restoreIP(CodeGenIP);
2881
2882     // collect some info for checks later
2883     ThenBB = Builder.GetInsertBlock();
2884     EntryBB = ThenBB->getUniquePredecessor();
2885
2886     // simple instructions for body
2887     Value *PrivLoad =
2888         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2889     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2890   };
2891
2892   auto FiniCB = [&](InsertPointTy IP) {
2893     BasicBlock *IPBB = IP.getBlock();
2894     EXPECT_NE(IPBB->end(), IP.getPoint());
2895   };
2896
2897   Builder.restoreIP(OMPBuilder.createSingle(
2898       Builder, BodyGenCB, FiniCB, /*IsNowait*/ false, /*DidIt*/ nullptr));
2899   Value *EntryBBTI = EntryBB->getTerminator();
2900   EXPECT_NE(EntryBBTI, nullptr);
2901   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2902   BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2903   EXPECT_TRUE(EntryBr->isConditional());
2904   EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2905   BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2906   EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2907
2908   CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2909   EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2910
2911   CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
2912   EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
2913   EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
2914   EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
2915
2916   CallInst *SingleEndCI = nullptr;
2917   for (auto &FI : *ThenBB) {
2918     Instruction *cur = &FI;
2919     if (isa<CallInst>(cur)) {
2920       SingleEndCI = cast<CallInst>(cur);
2921       if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
2922         break;
2923       SingleEndCI = nullptr;
2924     }
2925   }
2926   EXPECT_NE(SingleEndCI, nullptr);
2927   EXPECT_EQ(SingleEndCI->arg_size(), 2U);
2928   EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
2929   EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
2930
2931   bool FoundBarrier = false;
2932   for (auto &FI : *ExitBB) {
2933     Instruction *cur = &FI;
2934     if (auto CI = dyn_cast<CallInst>(cur)) {
2935       if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
2936         FoundBarrier = true;
2937         break;
2938       }
2939     }
2940   }
2941   EXPECT_TRUE(FoundBarrier);
2942 }
2943
2944 TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
2945   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2946   OpenMPIRBuilder OMPBuilder(*M);
2947   OMPBuilder.initialize();
2948   F->setName("func");
2949   IRBuilder<> Builder(BB);
2950
2951   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2952
2953   AllocaInst *PrivAI = nullptr;
2954
2955   BasicBlock *EntryBB = nullptr;
2956   BasicBlock *ThenBB = nullptr;
2957
2958   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2959     if (AllocaIP.isSet())
2960       Builder.restoreIP(AllocaIP);
2961     else
2962       Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2963     PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2964     Builder.CreateStore(F->arg_begin(), PrivAI);
2965
2966     llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2967     llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2968     EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2969
2970     Builder.restoreIP(CodeGenIP);
2971
2972     // collect some info for checks later
2973     ThenBB = Builder.GetInsertBlock();
2974     EntryBB = ThenBB->getUniquePredecessor();
2975
2976     // simple instructions for body
2977     Value *PrivLoad =
2978         Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2979     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2980   };
2981
2982   auto FiniCB = [&](InsertPointTy IP) {
2983     BasicBlock *IPBB = IP.getBlock();
2984     EXPECT_NE(IPBB->end(), IP.getPoint());
2985   };
2986
2987   Builder.restoreIP(OMPBuilder.createSingle(
2988       Builder, BodyGenCB, FiniCB, /*IsNowait*/ true, /*DidIt*/ nullptr));
2989   Value *EntryBBTI = EntryBB->getTerminator();
2990   EXPECT_NE(EntryBBTI, nullptr);
2991   EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2992   BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2993   EXPECT_TRUE(EntryBr->isConditional());
2994   EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2995   BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2996   EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2997
2998   CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2999   EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3000
3001   CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
3002   EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3003   EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3004   EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3005
3006   CallInst *SingleEndCI = nullptr;
3007   for (auto &FI : *ThenBB) {
3008     Instruction *cur = &FI;
3009     if (isa<CallInst>(cur)) {
3010       SingleEndCI = cast<CallInst>(cur);
3011       if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3012         break;
3013       SingleEndCI = nullptr;
3014     }
3015   }
3016   EXPECT_NE(SingleEndCI, nullptr);
3017   EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3018   EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3019   EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3020
3021   CallInst *ExitBarrier = nullptr;
3022   for (auto &FI : *ExitBB) {
3023     Instruction *cur = &FI;
3024     if (auto CI = dyn_cast<CallInst>(cur)) {
3025       if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3026         ExitBarrier = CI;
3027         break;
3028       }
3029     }
3030   }
3031   EXPECT_EQ(ExitBarrier, nullptr);
3032 }
3033
3034 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) {
3035   OpenMPIRBuilder OMPBuilder(*M);
3036   OMPBuilder.initialize();
3037   F->setName("func");
3038   IRBuilder<> Builder(BB);
3039
3040   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3041
3042   Type *Float32 = Type::getFloatTy(M->getContext());
3043   AllocaInst *XVal = Builder.CreateAlloca(Float32);
3044   XVal->setName("AtomicVar");
3045   AllocaInst *VVal = Builder.CreateAlloca(Float32);
3046   VVal->setName("AtomicRead");
3047   AtomicOrdering AO = AtomicOrdering::Monotonic;
3048   OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3049   OpenMPIRBuilder::AtomicOpValue V = {VVal, Float32, false, false};
3050
3051   Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3052
3053   IntegerType *IntCastTy =
3054       IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3055
3056   BitCastInst *CastFrmFlt = cast<BitCastInst>(VVal->getNextNode());
3057   EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo());
3058   EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo());
3059   EXPECT_EQ(CastFrmFlt->getOperand(0), XVal);
3060
3061   LoadInst *AtomicLoad = cast<LoadInst>(CastFrmFlt->getNextNode());
3062   EXPECT_TRUE(AtomicLoad->isAtomic());
3063   EXPECT_EQ(AtomicLoad->getPointerOperand(), CastFrmFlt);
3064
3065   BitCastInst *CastToFlt = cast<BitCastInst>(AtomicLoad->getNextNode());
3066   EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy);
3067   EXPECT_EQ(CastToFlt->getDestTy(), Float32);
3068   EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad);
3069
3070   StoreInst *StoreofAtomic = cast<StoreInst>(CastToFlt->getNextNode());
3071   EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt);
3072   EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal);
3073
3074   Builder.CreateRetVoid();
3075   OMPBuilder.finalize();
3076   EXPECT_FALSE(verifyModule(*M, &errs()));
3077 }
3078
3079 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) {
3080   OpenMPIRBuilder OMPBuilder(*M);
3081   OMPBuilder.initialize();
3082   F->setName("func");
3083   IRBuilder<> Builder(BB);
3084
3085   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3086
3087   IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3088   AllocaInst *XVal = Builder.CreateAlloca(Int32);
3089   XVal->setName("AtomicVar");
3090   AllocaInst *VVal = Builder.CreateAlloca(Int32);
3091   VVal->setName("AtomicRead");
3092   AtomicOrdering AO = AtomicOrdering::Monotonic;
3093   OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3094   OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3095
3096   BasicBlock *EntryBB = BB;
3097
3098   Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3099   LoadInst *AtomicLoad = nullptr;
3100   StoreInst *StoreofAtomic = nullptr;
3101
3102   for (Instruction &Cur : *EntryBB) {
3103     if (isa<LoadInst>(Cur)) {
3104       AtomicLoad = cast<LoadInst>(&Cur);
3105       if (AtomicLoad->getPointerOperand() == XVal)
3106         continue;
3107       AtomicLoad = nullptr;
3108     } else if (isa<StoreInst>(Cur)) {
3109       StoreofAtomic = cast<StoreInst>(&Cur);
3110       if (StoreofAtomic->getPointerOperand() == VVal)
3111         continue;
3112       StoreofAtomic = nullptr;
3113     }
3114   }
3115
3116   EXPECT_NE(AtomicLoad, nullptr);
3117   EXPECT_TRUE(AtomicLoad->isAtomic());
3118
3119   EXPECT_NE(StoreofAtomic, nullptr);
3120   EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad);
3121
3122   Builder.CreateRetVoid();
3123   OMPBuilder.finalize();
3124
3125   EXPECT_FALSE(verifyModule(*M, &errs()));
3126 }
3127
3128 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) {
3129   OpenMPIRBuilder OMPBuilder(*M);
3130   OMPBuilder.initialize();
3131   F->setName("func");
3132   IRBuilder<> Builder(BB);
3133
3134   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3135
3136   LLVMContext &Ctx = M->getContext();
3137   Type *Float32 = Type::getFloatTy(Ctx);
3138   AllocaInst *XVal = Builder.CreateAlloca(Float32);
3139   XVal->setName("AtomicVar");
3140   OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3141   AtomicOrdering AO = AtomicOrdering::Monotonic;
3142   Constant *ValToWrite = ConstantFP::get(Float32, 1.0);
3143
3144   Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3145
3146   IntegerType *IntCastTy =
3147       IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3148
3149   BitCastInst *CastFrmFlt = cast<BitCastInst>(XVal->getNextNode());
3150   EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo());
3151   EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo());
3152   EXPECT_EQ(CastFrmFlt->getOperand(0), XVal);
3153
3154   Value *ExprCast = Builder.CreateBitCast(ValToWrite, IntCastTy);
3155
3156   StoreInst *StoreofAtomic = cast<StoreInst>(CastFrmFlt->getNextNode());
3157   EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast);
3158   EXPECT_EQ(StoreofAtomic->getPointerOperand(), CastFrmFlt);
3159   EXPECT_TRUE(StoreofAtomic->isAtomic());
3160
3161   Builder.CreateRetVoid();
3162   OMPBuilder.finalize();
3163   EXPECT_FALSE(verifyModule(*M, &errs()));
3164 }
3165
3166 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) {
3167   OpenMPIRBuilder OMPBuilder(*M);
3168   OMPBuilder.initialize();
3169   F->setName("func");
3170   IRBuilder<> Builder(BB);
3171
3172   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3173
3174   LLVMContext &Ctx = M->getContext();
3175   IntegerType *Int32 = Type::getInt32Ty(Ctx);
3176   AllocaInst *XVal = Builder.CreateAlloca(Int32);
3177   XVal->setName("AtomicVar");
3178   OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3179   AtomicOrdering AO = AtomicOrdering::Monotonic;
3180   ConstantInt *ValToWrite = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3181
3182   BasicBlock *EntryBB = BB;
3183
3184   Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3185
3186   StoreInst *StoreofAtomic = nullptr;
3187
3188   for (Instruction &Cur : *EntryBB) {
3189     if (isa<StoreInst>(Cur)) {
3190       StoreofAtomic = cast<StoreInst>(&Cur);
3191       if (StoreofAtomic->getPointerOperand() == XVal)
3192         continue;
3193       StoreofAtomic = nullptr;
3194     }
3195   }
3196
3197   EXPECT_NE(StoreofAtomic, nullptr);
3198   EXPECT_TRUE(StoreofAtomic->isAtomic());
3199   EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite);
3200
3201   Builder.CreateRetVoid();
3202   OMPBuilder.finalize();
3203   EXPECT_FALSE(verifyModule(*M, &errs()));
3204 }
3205
3206 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
3207   OpenMPIRBuilder OMPBuilder(*M);
3208   OMPBuilder.initialize();
3209   F->setName("func");
3210   IRBuilder<> Builder(BB);
3211
3212   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3213
3214   IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3215   AllocaInst *XVal = Builder.CreateAlloca(Int32);
3216   XVal->setName("AtomicVar");
3217   Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3218   OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3219   AtomicOrdering AO = AtomicOrdering::Monotonic;
3220   ConstantInt *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3221   Value *Expr = nullptr;
3222   AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub;
3223   bool IsXLHSInRHSPart = false;
3224
3225   BasicBlock *EntryBB = BB;
3226   OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3227                                           EntryBB->getFirstInsertionPt());
3228   Value *Sub = nullptr;
3229
3230   auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3231     Sub = IRB.CreateSub(ConstVal, Atomic);
3232     return Sub;
3233   };
3234   Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3235       Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3236   BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3237   BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3238   EXPECT_NE(ContTI, nullptr);
3239   BasicBlock *EndBB = ContTI->getSuccessor(0);
3240   EXPECT_TRUE(ContTI->isConditional());
3241   EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3242   EXPECT_NE(EndBB, nullptr);
3243
3244   PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3245   EXPECT_NE(Phi, nullptr);
3246   EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3247   EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3248   EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3249
3250   EXPECT_EQ(Sub->getNumUses(), 1U);
3251   StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3252   AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3253
3254   ExtractValueInst *ExVI1 =
3255       dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3256   EXPECT_NE(ExVI1, nullptr);
3257   AtomicCmpXchgInst *CmpExchg =
3258       dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3259   EXPECT_NE(CmpExchg, nullptr);
3260   EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3261   EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3262   EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3263
3264   LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3265   EXPECT_NE(Ld, nullptr);
3266   EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3267
3268   Builder.CreateRetVoid();
3269   OMPBuilder.finalize();
3270   EXPECT_FALSE(verifyModule(*M, &errs()));
3271 }
3272
3273 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
3274   OpenMPIRBuilder OMPBuilder(*M);
3275   OMPBuilder.initialize();
3276   F->setName("func");
3277   IRBuilder<> Builder(BB);
3278
3279   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3280
3281   Type *FloatTy = Type::getFloatTy(M->getContext());
3282   AllocaInst *XVal = Builder.CreateAlloca(FloatTy);
3283   XVal->setName("AtomicVar");
3284   Builder.CreateStore(ConstantFP::get(Type::getFloatTy(Ctx), 0.0), XVal);
3285   OpenMPIRBuilder::AtomicOpValue X = {XVal, FloatTy, false, false};
3286   AtomicOrdering AO = AtomicOrdering::Monotonic;
3287   Constant *ConstVal = ConstantFP::get(Type::getFloatTy(Ctx), 1.0);
3288   Value *Expr = nullptr;
3289   AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::FSub;
3290   bool IsXLHSInRHSPart = false;
3291
3292   BasicBlock *EntryBB = BB;
3293   OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3294                                           EntryBB->getFirstInsertionPt());
3295   Value *Sub = nullptr;
3296
3297   auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3298     Sub = IRB.CreateFSub(ConstVal, Atomic);
3299     return Sub;
3300   };
3301   Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3302       Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3303   BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3304   BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3305   EXPECT_NE(ContTI, nullptr);
3306   BasicBlock *EndBB = ContTI->getSuccessor(0);
3307   EXPECT_TRUE(ContTI->isConditional());
3308   EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3309   EXPECT_NE(EndBB, nullptr);
3310
3311   PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3312   EXPECT_NE(Phi, nullptr);
3313   EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3314   EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3315   EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3316
3317   EXPECT_EQ(Sub->getNumUses(), 1U);
3318   StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3319   AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3320
3321   ExtractValueInst *ExVI1 =
3322       dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3323   EXPECT_NE(ExVI1, nullptr);
3324   AtomicCmpXchgInst *CmpExchg =
3325       dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3326   EXPECT_NE(CmpExchg, nullptr);
3327   BitCastInst *BitCastNew =
3328       dyn_cast<BitCastInst>(CmpExchg->getPointerOperand());
3329   EXPECT_NE(BitCastNew, nullptr);
3330   EXPECT_EQ(BitCastNew->getOperand(0), XVal);
3331   EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3332   EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3333
3334   LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3335   EXPECT_NE(Ld, nullptr);
3336   BitCastInst *BitCastOld = dyn_cast<BitCastInst>(Ld->getPointerOperand());
3337   EXPECT_NE(BitCastOld, nullptr);
3338   EXPECT_EQ(UpdateTemp, BitCastOld->getOperand(0));
3339
3340   Builder.CreateRetVoid();
3341   OMPBuilder.finalize();
3342   EXPECT_FALSE(verifyModule(*M, &errs()));
3343 }
3344
3345 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
3346   OpenMPIRBuilder OMPBuilder(*M);
3347   OMPBuilder.initialize();
3348   F->setName("func");
3349   IRBuilder<> Builder(BB);
3350
3351   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3352
3353   Type *IntTy = Type::getInt32Ty(M->getContext());
3354   AllocaInst *XVal = Builder.CreateAlloca(IntTy);
3355   XVal->setName("AtomicVar");
3356   Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);
3357   OpenMPIRBuilder::AtomicOpValue X = {XVal, IntTy, false, false};
3358   AtomicOrdering AO = AtomicOrdering::Monotonic;
3359   Constant *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3360   Value *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3361   AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::UMax;
3362   bool IsXLHSInRHSPart = false;
3363
3364   BasicBlock *EntryBB = BB;
3365   OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3366                                           EntryBB->getFirstInsertionPt());
3367   Value *Sub = nullptr;
3368
3369   auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3370     Sub = IRB.CreateSub(ConstVal, Atomic);
3371     return Sub;
3372   };
3373   Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3374       Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3375   BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3376   BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3377   EXPECT_NE(ContTI, nullptr);
3378   BasicBlock *EndBB = ContTI->getSuccessor(0);
3379   EXPECT_TRUE(ContTI->isConditional());
3380   EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3381   EXPECT_NE(EndBB, nullptr);
3382
3383   PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3384   EXPECT_NE(Phi, nullptr);
3385   EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3386   EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3387   EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3388
3389   EXPECT_EQ(Sub->getNumUses(), 1U);
3390   StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3391   AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3392
3393   ExtractValueInst *ExVI1 =
3394       dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3395   EXPECT_NE(ExVI1, nullptr);
3396   AtomicCmpXchgInst *CmpExchg =
3397       dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3398   EXPECT_NE(CmpExchg, nullptr);
3399   EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3400   EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3401   EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3402
3403   LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3404   EXPECT_NE(Ld, nullptr);
3405   EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3406
3407   Builder.CreateRetVoid();
3408   OMPBuilder.finalize();
3409   EXPECT_FALSE(verifyModule(*M, &errs()));
3410 }
3411
3412 TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {
3413   OpenMPIRBuilder OMPBuilder(*M);
3414   OMPBuilder.initialize();
3415   F->setName("func");
3416   IRBuilder<> Builder(BB);
3417
3418   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3419
3420   LLVMContext &Ctx = M->getContext();
3421   IntegerType *Int32 = Type::getInt32Ty(Ctx);
3422   AllocaInst *XVal = Builder.CreateAlloca(Int32);
3423   XVal->setName("AtomicVar");
3424   AllocaInst *VVal = Builder.CreateAlloca(Int32);
3425   VVal->setName("AtomicCapTar");
3426   StoreInst *Init =
3427       Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3428
3429   OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3430   OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3431   AtomicOrdering AO = AtomicOrdering::Monotonic;
3432   ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3433   AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add;
3434   bool IsXLHSInRHSPart = true;
3435   bool IsPostfixUpdate = true;
3436   bool UpdateExpr = true;
3437
3438   BasicBlock *EntryBB = BB;
3439   OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3440                                           EntryBB->getFirstInsertionPt());
3441
3442   // integer update - not used
3443   auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };
3444
3445   Builder.restoreIP(OMPBuilder.createAtomicCapture(
3446       Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr,
3447       IsPostfixUpdate, IsXLHSInRHSPart));
3448   EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3449   AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3450   EXPECT_NE(ARWM, nullptr);
3451   EXPECT_EQ(ARWM->getPointerOperand(), XVal);
3452   EXPECT_EQ(ARWM->getOperation(), RMWOp);
3453   StoreInst *St = dyn_cast<StoreInst>(ARWM->user_back());
3454   EXPECT_NE(St, nullptr);
3455   EXPECT_EQ(St->getPointerOperand(), VVal);
3456
3457   Builder.CreateRetVoid();
3458   OMPBuilder.finalize();
3459   EXPECT_FALSE(verifyModule(*M, &errs()));
3460 }
3461
3462 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompare) {
3463   OpenMPIRBuilder OMPBuilder(*M);
3464   OMPBuilder.initialize();
3465   F->setName("func");
3466   IRBuilder<> Builder(BB);
3467
3468   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3469
3470   LLVMContext &Ctx = M->getContext();
3471   IntegerType *Int32 = Type::getInt32Ty(Ctx);
3472   AllocaInst *XVal = Builder.CreateAlloca(Int32);
3473   XVal->setName("x");
3474   StoreInst *Init =
3475       Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3476
3477   OpenMPIRBuilder::AtomicOpValue XSigned = {XVal, Int32, true, false};
3478   OpenMPIRBuilder::AtomicOpValue XUnsigned = {XVal, Int32, false, false};
3479   // V and R are not used in atomic compare
3480   OpenMPIRBuilder::AtomicOpValue V = {nullptr, nullptr, false, false};
3481   OpenMPIRBuilder::AtomicOpValue R = {nullptr, nullptr, false, false};
3482   AtomicOrdering AO = AtomicOrdering::Monotonic;
3483   ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3484   ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3485   OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3486   OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3487
3488   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3489       Builder, XSigned, V, R, Expr, nullptr, AO, OpMax, true, false, false));
3490   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3491       Builder, XUnsigned, V, R, Expr, nullptr, AO, OpMax, false, false, false));
3492   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3493       Builder, XSigned, V, R, Expr, D, AO, OpEQ, true, false, false));
3494
3495   BasicBlock *EntryBB = BB;
3496   EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3497   EXPECT_EQ(EntryBB->size(), 5U);
3498
3499   AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3500   EXPECT_NE(ARWM1, nullptr);
3501   EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3502   EXPECT_EQ(ARWM1->getValOperand(), Expr);
3503   EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3504
3505   AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(ARWM1->getNextNode());
3506   EXPECT_NE(ARWM2, nullptr);
3507   EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3508   EXPECT_EQ(ARWM2->getValOperand(), Expr);
3509   EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::UMax);
3510
3511   AtomicCmpXchgInst *AXCHG = dyn_cast<AtomicCmpXchgInst>(ARWM2->getNextNode());
3512   EXPECT_NE(AXCHG, nullptr);
3513   EXPECT_EQ(AXCHG->getPointerOperand(), XVal);
3514   EXPECT_EQ(AXCHG->getCompareOperand(), Expr);
3515   EXPECT_EQ(AXCHG->getNewValOperand(), D);
3516
3517   Builder.CreateRetVoid();
3518   OMPBuilder.finalize();
3519   EXPECT_FALSE(verifyModule(*M, &errs()));
3520 }
3521
3522 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) {
3523   OpenMPIRBuilder OMPBuilder(*M);
3524   OMPBuilder.initialize();
3525   F->setName("func");
3526   IRBuilder<> Builder(BB);
3527
3528   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3529
3530   LLVMContext &Ctx = M->getContext();
3531   IntegerType *Int32 = Type::getInt32Ty(Ctx);
3532   AllocaInst *XVal = Builder.CreateAlloca(Int32);
3533   XVal->setName("x");
3534   AllocaInst *VVal = Builder.CreateAlloca(Int32);
3535   VVal->setName("v");
3536   AllocaInst *RVal = Builder.CreateAlloca(Int32);
3537   RVal->setName("r");
3538
3539   StoreInst *Init =
3540       Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3541
3542   OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, true, false};
3543   OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3544   OpenMPIRBuilder::AtomicOpValue NoV = {nullptr, nullptr, false, false};
3545   OpenMPIRBuilder::AtomicOpValue R = {RVal, Int32, false, false};
3546   OpenMPIRBuilder::AtomicOpValue NoR = {nullptr, nullptr, false, false};
3547
3548   AtomicOrdering AO = AtomicOrdering::Monotonic;
3549   ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3550   ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3551   OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3552   OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3553
3554   // { cond-update-stmt v = x; }
3555   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3556       Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3557       /* IsPostfixUpdate */ false,
3558       /* IsFailOnly */ false));
3559   // { v = x; cond-update-stmt }
3560   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3561       Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3562       /* IsPostfixUpdate */ true,
3563       /* IsFailOnly */ false));
3564   // if(x == e) { x = d; } else { v = x; }
3565   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3566       Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3567       /* IsPostfixUpdate */ false,
3568       /* IsFailOnly */ true));
3569   // { r = x == e; if(r) { x = d; } }
3570   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3571       Builder, X, NoV, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3572       /* IsPostfixUpdate */ false,
3573       /* IsFailOnly */ false));
3574   // { r = x == e; if(r) { x = d; } else { v = x; } }
3575   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3576       Builder, X, V, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3577       /* IsPostfixUpdate */ false,
3578       /* IsFailOnly */ true));
3579
3580   // { v = x; cond-update-stmt }
3581   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3582       Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ true,
3583       /* IsPostfixUpdate */ true,
3584       /* IsFailOnly */ false));
3585   // { cond-update-stmt v = x; }
3586   Builder.restoreIP(OMPBuilder.createAtomicCompare(
3587       Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ false,
3588       /* IsPostfixUpdate */ false,
3589       /* IsFailOnly */ false));
3590
3591   BasicBlock *EntryBB = BB;
3592   EXPECT_EQ(EntryBB->getParent()->size(), 5U);
3593   BasicBlock *Cont1 = dyn_cast<BasicBlock>(EntryBB->getNextNode());
3594   EXPECT_NE(Cont1, nullptr);
3595   BasicBlock *Exit1 = dyn_cast<BasicBlock>(Cont1->getNextNode());
3596   EXPECT_NE(Exit1, nullptr);
3597   BasicBlock *Cont2 = dyn_cast<BasicBlock>(Exit1->getNextNode());
3598   EXPECT_NE(Cont2, nullptr);
3599   BasicBlock *Exit2 = dyn_cast<BasicBlock>(Cont2->getNextNode());
3600   EXPECT_NE(Exit2, nullptr);
3601
3602   AtomicCmpXchgInst *CmpXchg1 =
3603       dyn_cast<AtomicCmpXchgInst>(Init->getNextNode());
3604   EXPECT_NE(CmpXchg1, nullptr);
3605   EXPECT_EQ(CmpXchg1->getPointerOperand(), XVal);
3606   EXPECT_EQ(CmpXchg1->getCompareOperand(), Expr);
3607   EXPECT_EQ(CmpXchg1->getNewValOperand(), D);
3608   ExtractValueInst *ExtVal1 =
3609       dyn_cast<ExtractValueInst>(CmpXchg1->getNextNode());
3610   EXPECT_NE(ExtVal1, nullptr);
3611   EXPECT_EQ(ExtVal1->getAggregateOperand(), CmpXchg1);
3612   EXPECT_EQ(ExtVal1->getIndices(), ArrayRef<unsigned int>(0U));
3613   ExtractValueInst *ExtVal2 =
3614       dyn_cast<ExtractValueInst>(ExtVal1->getNextNode());
3615   EXPECT_NE(ExtVal2, nullptr);
3616   EXPECT_EQ(ExtVal2->getAggregateOperand(), CmpXchg1);
3617   EXPECT_EQ(ExtVal2->getIndices(), ArrayRef<unsigned int>(1U));
3618   SelectInst *Sel1 = dyn_cast<SelectInst>(ExtVal2->getNextNode());
3619   EXPECT_NE(Sel1, nullptr);
3620   EXPECT_EQ(Sel1->getCondition(), ExtVal2);
3621   EXPECT_EQ(Sel1->getTrueValue(), Expr);
3622   EXPECT_EQ(Sel1->getFalseValue(), ExtVal1);
3623   StoreInst *Store1 = dyn_cast<StoreInst>(Sel1->getNextNode());
3624   EXPECT_NE(Store1, nullptr);
3625   EXPECT_EQ(Store1->getPointerOperand(), VVal);
3626   EXPECT_EQ(Store1->getValueOperand(), Sel1);
3627
3628   AtomicCmpXchgInst *CmpXchg2 =
3629       dyn_cast<AtomicCmpXchgInst>(Store1->getNextNode());
3630   EXPECT_NE(CmpXchg2, nullptr);
3631   EXPECT_EQ(CmpXchg2->getPointerOperand(), XVal);
3632   EXPECT_EQ(CmpXchg2->getCompareOperand(), Expr);
3633   EXPECT_EQ(CmpXchg2->getNewValOperand(), D);
3634   ExtractValueInst *ExtVal3 =
3635       dyn_cast<ExtractValueInst>(CmpXchg2->getNextNode());
3636   EXPECT_NE(ExtVal3, nullptr);
3637   EXPECT_EQ(ExtVal3->getAggregateOperand(), CmpXchg2);
3638   EXPECT_EQ(ExtVal3->getIndices(), ArrayRef<unsigned int>(0U));
3639   StoreInst *Store2 = dyn_cast<StoreInst>(ExtVal3->getNextNode());
3640   EXPECT_NE(Store2, nullptr);
3641   EXPECT_EQ(Store2->getPointerOperand(), VVal);
3642   EXPECT_EQ(Store2->getValueOperand(), ExtVal3);
3643
3644   AtomicCmpXchgInst *CmpXchg3 =
3645       dyn_cast<AtomicCmpXchgInst>(Store2->getNextNode());
3646   EXPECT_NE(CmpXchg3, nullptr);
3647   EXPECT_EQ(CmpXchg3->getPointerOperand(), XVal);
3648   EXPECT_EQ(CmpXchg3->getCompareOperand(), Expr);
3649   EXPECT_EQ(CmpXchg3->getNewValOperand(), D);
3650   ExtractValueInst *ExtVal4 =
3651       dyn_cast<ExtractValueInst>(CmpXchg3->getNextNode());
3652   EXPECT_NE(ExtVal4, nullptr);
3653   EXPECT_EQ(ExtVal4->getAggregateOperand(), CmpXchg3);
3654   EXPECT_EQ(ExtVal4->getIndices(), ArrayRef<unsigned int>(0U));
3655   ExtractValueInst *ExtVal5 =
3656       dyn_cast<ExtractValueInst>(ExtVal4->getNextNode());
3657   EXPECT_NE(ExtVal5, nullptr);
3658   EXPECT_EQ(ExtVal5->getAggregateOperand(), CmpXchg3);
3659   EXPECT_EQ(ExtVal5->getIndices(), ArrayRef<unsigned int>(1U));
3660   BranchInst *Br1 = dyn_cast<BranchInst>(ExtVal5->getNextNode());
3661   EXPECT_NE(Br1, nullptr);
3662   EXPECT_EQ(Br1->isConditional(), true);
3663   EXPECT_EQ(Br1->getCondition(), ExtVal5);
3664   EXPECT_EQ(Br1->getSuccessor(0), Exit1);
3665   EXPECT_EQ(Br1->getSuccessor(1), Cont1);
3666
3667   StoreInst *Store3 = dyn_cast<StoreInst>(&Cont1->front());
3668   EXPECT_NE(Store3, nullptr);
3669   EXPECT_EQ(Store3->getPointerOperand(), VVal);
3670   EXPECT_EQ(Store3->getValueOperand(), ExtVal4);
3671   BranchInst *Br2 = dyn_cast<BranchInst>(Store3->getNextNode());
3672   EXPECT_NE(Br2, nullptr);
3673   EXPECT_EQ(Br2->isUnconditional(), true);
3674   EXPECT_EQ(Br2->getSuccessor(0), Exit1);
3675
3676   AtomicCmpXchgInst *CmpXchg4 = dyn_cast<AtomicCmpXchgInst>(&Exit1->front());
3677   EXPECT_NE(CmpXchg4, nullptr);
3678   EXPECT_EQ(CmpXchg4->getPointerOperand(), XVal);
3679   EXPECT_EQ(CmpXchg4->getCompareOperand(), Expr);
3680   EXPECT_EQ(CmpXchg4->getNewValOperand(), D);
3681   ExtractValueInst *ExtVal6 =
3682       dyn_cast<ExtractValueInst>(CmpXchg4->getNextNode());
3683   EXPECT_NE(ExtVal6, nullptr);
3684   EXPECT_EQ(ExtVal6->getAggregateOperand(), CmpXchg4);
3685   EXPECT_EQ(ExtVal6->getIndices(), ArrayRef<unsigned int>(1U));
3686   ZExtInst *ZExt1 = dyn_cast<ZExtInst>(ExtVal6->getNextNode());
3687   EXPECT_NE(ZExt1, nullptr);
3688   EXPECT_EQ(ZExt1->getDestTy(), Int32);
3689   StoreInst *Store4 = dyn_cast<StoreInst>(ZExt1->getNextNode());
3690   EXPECT_NE(Store4, nullptr);
3691   EXPECT_EQ(Store4->getPointerOperand(), RVal);
3692   EXPECT_EQ(Store4->getValueOperand(), ZExt1);
3693
3694   AtomicCmpXchgInst *CmpXchg5 =
3695       dyn_cast<AtomicCmpXchgInst>(Store4->getNextNode());
3696   EXPECT_NE(CmpXchg5, nullptr);
3697   EXPECT_EQ(CmpXchg5->getPointerOperand(), XVal);
3698   EXPECT_EQ(CmpXchg5->getCompareOperand(), Expr);
3699   EXPECT_EQ(CmpXchg5->getNewValOperand(), D);
3700   ExtractValueInst *ExtVal7 =
3701       dyn_cast<ExtractValueInst>(CmpXchg5->getNextNode());
3702   EXPECT_NE(ExtVal7, nullptr);
3703   EXPECT_EQ(ExtVal7->getAggregateOperand(), CmpXchg5);
3704   EXPECT_EQ(ExtVal7->getIndices(), ArrayRef<unsigned int>(0U));
3705   ExtractValueInst *ExtVal8 =
3706       dyn_cast<ExtractValueInst>(ExtVal7->getNextNode());
3707   EXPECT_NE(ExtVal8, nullptr);
3708   EXPECT_EQ(ExtVal8->getAggregateOperand(), CmpXchg5);
3709   EXPECT_EQ(ExtVal8->getIndices(), ArrayRef<unsigned int>(1U));
3710   BranchInst *Br3 = dyn_cast<BranchInst>(ExtVal8->getNextNode());
3711   EXPECT_NE(Br3, nullptr);
3712   EXPECT_EQ(Br3->isConditional(), true);
3713   EXPECT_EQ(Br3->getCondition(), ExtVal8);
3714   EXPECT_EQ(Br3->getSuccessor(0), Exit2);
3715   EXPECT_EQ(Br3->getSuccessor(1), Cont2);
3716
3717   StoreInst *Store5 = dyn_cast<StoreInst>(&Cont2->front());
3718   EXPECT_NE(Store5, nullptr);
3719   EXPECT_EQ(Store5->getPointerOperand(), VVal);
3720   EXPECT_EQ(Store5->getValueOperand(), ExtVal7);
3721   BranchInst *Br4 = dyn_cast<BranchInst>(Store5->getNextNode());
3722   EXPECT_NE(Br4, nullptr);
3723   EXPECT_EQ(Br4->isUnconditional(), true);
3724   EXPECT_EQ(Br4->getSuccessor(0), Exit2);
3725
3726   ExtractValueInst *ExtVal9 = dyn_cast<ExtractValueInst>(&Exit2->front());
3727   EXPECT_NE(ExtVal9, nullptr);
3728   EXPECT_EQ(ExtVal9->getAggregateOperand(), CmpXchg5);
3729   EXPECT_EQ(ExtVal9->getIndices(), ArrayRef<unsigned int>(1U));
3730   ZExtInst *ZExt2 = dyn_cast<ZExtInst>(ExtVal9->getNextNode());
3731   EXPECT_NE(ZExt2, nullptr);
3732   EXPECT_EQ(ZExt2->getDestTy(), Int32);
3733   StoreInst *Store6 = dyn_cast<StoreInst>(ZExt2->getNextNode());
3734   EXPECT_NE(Store6, nullptr);
3735   EXPECT_EQ(Store6->getPointerOperand(), RVal);
3736   EXPECT_EQ(Store6->getValueOperand(), ZExt2);
3737
3738   AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Store6->getNextNode());
3739   EXPECT_NE(ARWM1, nullptr);
3740   EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3741   EXPECT_EQ(ARWM1->getValOperand(), Expr);
3742   EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3743   StoreInst *Store7 = dyn_cast<StoreInst>(ARWM1->getNextNode());
3744   EXPECT_NE(Store7, nullptr);
3745   EXPECT_EQ(Store7->getPointerOperand(), VVal);
3746   EXPECT_EQ(Store7->getValueOperand(), ARWM1);
3747
3748   AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Store7->getNextNode());
3749   EXPECT_NE(ARWM2, nullptr);
3750   EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3751   EXPECT_EQ(ARWM2->getValOperand(), Expr);
3752   EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::Max);
3753   CmpInst *Cmp1 = dyn_cast<CmpInst>(ARWM2->getNextNode());
3754   EXPECT_NE(Cmp1, nullptr);
3755   EXPECT_EQ(Cmp1->getPredicate(), CmpInst::ICMP_SGT);
3756   EXPECT_EQ(Cmp1->getOperand(0), ARWM2);
3757   EXPECT_EQ(Cmp1->getOperand(1), Expr);
3758   SelectInst *Sel2 = dyn_cast<SelectInst>(Cmp1->getNextNode());
3759   EXPECT_NE(Sel2, nullptr);
3760   EXPECT_EQ(Sel2->getCondition(), Cmp1);
3761   EXPECT_EQ(Sel2->getTrueValue(), Expr);
3762   EXPECT_EQ(Sel2->getFalseValue(), ARWM2);
3763   StoreInst *Store8 = dyn_cast<StoreInst>(Sel2->getNextNode());
3764   EXPECT_NE(Store8, nullptr);
3765   EXPECT_EQ(Store8->getPointerOperand(), VVal);
3766   EXPECT_EQ(Store8->getValueOperand(), Sel2);
3767
3768   Builder.CreateRetVoid();
3769   OMPBuilder.finalize();
3770   EXPECT_FALSE(verifyModule(*M, &errs()));
3771 }
3772
3773 /// Returns the single instruction of InstTy type in BB that uses the value V.
3774 /// If there is more than one such instruction, returns null.
3775 template <typename InstTy>
3776 static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) {
3777   InstTy *Result = nullptr;
3778   for (User *U : V->users()) {
3779     auto *Inst = dyn_cast<InstTy>(U);
3780     if (!Inst || Inst->getParent() != BB)
3781       continue;
3782     if (Result)
3783       return nullptr;
3784     Result = Inst;
3785   }
3786   return Result;
3787 }
3788
3789 /// Returns true if BB contains a simple binary reduction that loads a value
3790 /// from Accum, performs some binary operation with it, and stores it back to
3791 /// Accum.
3792 static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB,
3793                                     Instruction::BinaryOps *OpCode = nullptr) {
3794   StoreInst *Store = findSingleUserInBlock<StoreInst>(Accum, BB);
3795   if (!Store)
3796     return false;
3797   auto *Stored = dyn_cast<BinaryOperator>(Store->getOperand(0));
3798   if (!Stored)
3799     return false;
3800   if (OpCode && *OpCode != Stored->getOpcode())
3801     return false;
3802   auto *Load = dyn_cast<LoadInst>(Stored->getOperand(0));
3803   return Load && Load->getOperand(0) == Accum;
3804 }
3805
3806 /// Returns true if BB contains a binary reduction that reduces V using a binary
3807 /// operator into an accumulator that is a function argument.
3808 static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) {
3809   auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB);
3810   if (!ReductionOp)
3811     return false;
3812
3813   auto *GlobalLoad = dyn_cast<LoadInst>(ReductionOp->getOperand(0));
3814   if (!GlobalLoad)
3815     return false;
3816
3817   auto *Store = findSingleUserInBlock<StoreInst>(ReductionOp, BB);
3818   if (!Store)
3819     return false;
3820
3821   return Store->getPointerOperand() == GlobalLoad->getPointerOperand() &&
3822          isa<Argument>(findAggregateFromValue(GlobalLoad->getPointerOperand()));
3823 }
3824
3825 /// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and
3826 /// [0, 1], respectively, and assigns results of these instructions to Zero and
3827 /// One. Returns true on success, false on failure or if such instructions are
3828 /// not unique among the users of Ptr.
3829 static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) {
3830   Zero = nullptr;
3831   One = nullptr;
3832   for (User *U : Ptr->users()) {
3833     if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
3834       if (GEP->getNumIndices() != 2)
3835         continue;
3836       auto *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
3837       auto *SecondIdx = dyn_cast<ConstantInt>(GEP->getOperand(2));
3838       EXPECT_NE(FirstIdx, nullptr);
3839       EXPECT_NE(SecondIdx, nullptr);
3840
3841       EXPECT_TRUE(FirstIdx->isZero());
3842       if (SecondIdx->isZero()) {
3843         if (Zero)
3844           return false;
3845         Zero = GEP;
3846       } else if (SecondIdx->isOne()) {
3847         if (One)
3848           return false;
3849         One = GEP;
3850       } else {
3851         return false;
3852       }
3853     }
3854   }
3855   return Zero != nullptr && One != nullptr;
3856 }
3857
3858 static OpenMPIRBuilder::InsertPointTy
3859 sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
3860              Value *&Result) {
3861   IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3862   Result = Builder.CreateFAdd(LHS, RHS, "red.add");
3863   return Builder.saveIP();
3864 }
3865
3866 static OpenMPIRBuilder::InsertPointTy
3867 sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
3868                    Value *RHS) {
3869   IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3870   Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
3871   Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, LHS, Partial, None,
3872                           AtomicOrdering::Monotonic);
3873   return Builder.saveIP();
3874 }
3875
3876 static OpenMPIRBuilder::InsertPointTy
3877 xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
3878              Value *&Result) {
3879   IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3880   Result = Builder.CreateXor(LHS, RHS, "red.xor");
3881   return Builder.saveIP();
3882 }
3883
3884 static OpenMPIRBuilder::InsertPointTy
3885 xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
3886                    Value *RHS) {
3887   IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3888   Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
3889   Builder.CreateAtomicRMW(AtomicRMWInst::Xor, LHS, Partial, None,
3890                           AtomicOrdering::Monotonic);
3891   return Builder.saveIP();
3892 }
3893
3894 TEST_F(OpenMPIRBuilderTest, CreateReductions) {
3895   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3896   OpenMPIRBuilder OMPBuilder(*M);
3897   OMPBuilder.initialize();
3898   F->setName("func");
3899   IRBuilder<> Builder(BB);
3900
3901   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
3902   Builder.CreateBr(EnterBB);
3903   Builder.SetInsertPoint(EnterBB);
3904   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3905
3906   // Create variables to be reduced.
3907   InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
3908                               F->getEntryBlock().getFirstInsertionPt());
3909   Type *SumType = Builder.getFloatTy();
3910   Type *XorType = Builder.getInt32Ty();
3911   Value *SumReduced;
3912   Value *XorReduced;
3913   {
3914     IRBuilderBase::InsertPointGuard Guard(Builder);
3915     Builder.restoreIP(OuterAllocaIP);
3916     SumReduced = Builder.CreateAlloca(SumType);
3917     XorReduced = Builder.CreateAlloca(XorType);
3918   }
3919
3920   // Store initial values of reductions into global variables.
3921   Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
3922   Builder.CreateStore(Builder.getInt32(1), XorReduced);
3923
3924   // The loop body computes two reductions:
3925   //   sum of (float) thread-id;
3926   //   xor of thread-id;
3927   // and store the result in global variables.
3928   InsertPointTy BodyIP, BodyAllocaIP;
3929   auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {
3930     IRBuilderBase::InsertPointGuard Guard(Builder);
3931     Builder.restoreIP(CodeGenIP);
3932
3933     uint32_t StrSize;
3934     Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
3935     Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
3936     Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
3937     Value *SumLocal =
3938         Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
3939     Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
3940     Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
3941     Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
3942     Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
3943     Builder.CreateStore(Sum, SumReduced);
3944     Builder.CreateStore(Xor, XorReduced);
3945
3946     BodyIP = Builder.saveIP();
3947     BodyAllocaIP = InnerAllocaIP;
3948   };
3949
3950   // Privatization for reduction creates local copies of reduction variables and
3951   // initializes them to reduction-neutral values.
3952   Value *SumPrivatized;
3953   Value *XorPrivatized;
3954   auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
3955                     Value &Original, Value &Inner, Value *&ReplVal) {
3956     IRBuilderBase::InsertPointGuard Guard(Builder);
3957     Builder.restoreIP(InnerAllocaIP);
3958     if (&Original == SumReduced) {
3959       SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
3960       ReplVal = SumPrivatized;
3961     } else if (&Original == XorReduced) {
3962       XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
3963       ReplVal = XorPrivatized;
3964     } else {
3965       ReplVal = &Inner;
3966       return CodeGenIP;
3967     }
3968
3969     Builder.restoreIP(CodeGenIP);
3970     if (&Original == SumReduced)
3971       Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
3972                           SumPrivatized);
3973     else if (&Original == XorReduced)
3974       Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
3975
3976     return Builder.saveIP();
3977   };
3978
3979   // Do nothing in finalization.
3980   auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
3981
3982   InsertPointTy AfterIP =
3983       OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
3984                                 /* IfCondition */ nullptr,
3985                                 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
3986                                 /* IsCancellable */ false);
3987   Builder.restoreIP(AfterIP);
3988
3989   OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
3990       {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
3991       {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
3992
3993   OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
3994
3995   Builder.restoreIP(AfterIP);
3996   Builder.CreateRetVoid();
3997
3998   OMPBuilder.finalize(F);
3999
4000   // The IR must be valid.
4001   EXPECT_FALSE(verifyModule(*M));
4002
4003   // Outlining must have happened.
4004   SmallVector<CallInst *> ForkCalls;
4005   findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4006             ForkCalls);
4007   ASSERT_EQ(ForkCalls.size(), 1u);
4008   Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0);
4009   Function *Outlined = dyn_cast<Function>(CalleeVal);
4010   EXPECT_NE(Outlined, nullptr);
4011
4012   // Check that the lock variable was created with the expected name.
4013   GlobalVariable *LockVar =
4014       M->getGlobalVariable(".gomp_critical_user_.reduction.var");
4015   EXPECT_NE(LockVar, nullptr);
4016
4017   // Find the allocation of a local array that will be used to call the runtime
4018   // reduciton function.
4019   BasicBlock &AllocBlock = Outlined->getEntryBlock();
4020   Value *LocalArray = nullptr;
4021   for (Instruction &I : AllocBlock) {
4022     if (AllocaInst *Alloc = dyn_cast<AllocaInst>(&I)) {
4023       if (!Alloc->getAllocatedType()->isArrayTy() ||
4024           !Alloc->getAllocatedType()->getArrayElementType()->isPointerTy())
4025         continue;
4026       LocalArray = Alloc;
4027       break;
4028     }
4029   }
4030   ASSERT_NE(LocalArray, nullptr);
4031
4032   // Find the call to the runtime reduction function.
4033   BasicBlock *BB = AllocBlock.getUniqueSuccessor();
4034   Value *LocalArrayPtr = nullptr;
4035   Value *ReductionFnVal = nullptr;
4036   Value *SwitchArg = nullptr;
4037   for (Instruction &I : *BB) {
4038     if (CallInst *Call = dyn_cast<CallInst>(&I)) {
4039       if (Call->getCalledFunction() !=
4040           OMPBuilder.getOrCreateRuntimeFunctionPtr(
4041               RuntimeFunction::OMPRTL___kmpc_reduce))
4042         continue;
4043       LocalArrayPtr = Call->getOperand(4);
4044       ReductionFnVal = Call->getOperand(5);
4045       SwitchArg = Call;
4046       break;
4047     }
4048   }
4049
4050   // Check that the local array is passed to the function.
4051   ASSERT_NE(LocalArrayPtr, nullptr);
4052   BitCastInst *BitCast = dyn_cast<BitCastInst>(LocalArrayPtr);
4053   ASSERT_NE(BitCast, nullptr);
4054   EXPECT_EQ(BitCast->getOperand(0), LocalArray);
4055
4056   // Find the GEP instructions preceding stores to the local array.
4057   Value *FirstArrayElemPtr = nullptr;
4058   Value *SecondArrayElemPtr = nullptr;
4059   EXPECT_EQ(LocalArray->getNumUses(), 3u);
4060   ASSERT_TRUE(
4061       findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));
4062
4063   // Check that the values stored into the local array are privatized reduction
4064   // variables.
4065   auto *FirstStored = dyn_cast_or_null<BitCastInst>(
4066       findStoredValue<GetElementPtrInst>(FirstArrayElemPtr));
4067   auto *SecondStored = dyn_cast_or_null<BitCastInst>(
4068       findStoredValue<GetElementPtrInst>(SecondArrayElemPtr));
4069   ASSERT_NE(FirstStored, nullptr);
4070   ASSERT_NE(SecondStored, nullptr);
4071   Value *FirstPrivatized = FirstStored->getOperand(0);
4072   Value *SecondPrivatized = SecondStored->getOperand(0);
4073   EXPECT_TRUE(
4074       isSimpleBinaryReduction(FirstPrivatized, FirstStored->getParent()));
4075   EXPECT_TRUE(
4076       isSimpleBinaryReduction(SecondPrivatized, SecondStored->getParent()));
4077
4078   // Check that the result of the runtime reduction call is used for further
4079   // dispatch.
4080   ASSERT_EQ(SwitchArg->getNumUses(), 1u);
4081   SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin());
4082   ASSERT_NE(Switch, nullptr);
4083   EXPECT_EQ(Switch->getNumSuccessors(), 3u);
4084   BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor();
4085   BasicBlock *AtomicBB = std::next(Switch->case_begin())->getCaseSuccessor();
4086
4087   // Non-atomic block contains reductions to the global reduction variable,
4088   // which is passed into the outlined function as an argument.
4089   Value *FirstLoad =
4090       findSingleUserInBlock<LoadInst>(FirstPrivatized, NonAtomicBB);
4091   Value *SecondLoad =
4092       findSingleUserInBlock<LoadInst>(SecondPrivatized, NonAtomicBB);
4093   EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB));
4094   EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB));
4095
4096   // Atomic block also constains reductions to the global reduction variable.
4097   FirstLoad = findSingleUserInBlock<LoadInst>(FirstPrivatized, AtomicBB);
4098   SecondLoad = findSingleUserInBlock<LoadInst>(SecondPrivatized, AtomicBB);
4099   auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(FirstLoad, AtomicBB);
4100   auto *SecondAtomic =
4101       findSingleUserInBlock<AtomicRMWInst>(SecondLoad, AtomicBB);
4102   ASSERT_NE(FirstAtomic, nullptr);
4103   Value *AtomicStorePointer = FirstAtomic->getPointerOperand();
4104   EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4105   ASSERT_NE(SecondAtomic, nullptr);
4106   AtomicStorePointer = SecondAtomic->getPointerOperand();
4107   EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4108
4109   // Check that the separate reduction function also performs (non-atomic)
4110   // reductions after extracting reduction variables from its arguments.
4111   Function *ReductionFn = cast<Function>(ReductionFnVal);
4112   BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock();
4113   auto *Bitcast =
4114       findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(0), FnReductionBB);
4115   Value *FirstLHSPtr;
4116   Value *SecondLHSPtr;
4117   ASSERT_TRUE(findGEPZeroOne(Bitcast, FirstLHSPtr, SecondLHSPtr));
4118   Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4119   ASSERT_NE(Opaque, nullptr);
4120   Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4121   ASSERT_NE(Bitcast, nullptr);
4122   EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB));
4123   Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4124   ASSERT_NE(Opaque, nullptr);
4125   Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4126   ASSERT_NE(Bitcast, nullptr);
4127   EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB));
4128
4129   Bitcast =
4130       findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(1), FnReductionBB);
4131   Value *FirstRHS;
4132   Value *SecondRHS;
4133   EXPECT_TRUE(findGEPZeroOne(Bitcast, FirstRHS, SecondRHS));
4134 }
4135
4136 TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
4137   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4138   OpenMPIRBuilder OMPBuilder(*M);
4139   OMPBuilder.initialize();
4140   F->setName("func");
4141   IRBuilder<> Builder(BB);
4142
4143   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
4144   Builder.CreateBr(EnterBB);
4145   Builder.SetInsertPoint(EnterBB);
4146   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4147
4148   // Create variables to be reduced.
4149   InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4150                               F->getEntryBlock().getFirstInsertionPt());
4151   Type *SumType = Builder.getFloatTy();
4152   Type *XorType = Builder.getInt32Ty();
4153   Value *SumReduced;
4154   Value *XorReduced;
4155   {
4156     IRBuilderBase::InsertPointGuard Guard(Builder);
4157     Builder.restoreIP(OuterAllocaIP);
4158     SumReduced = Builder.CreateAlloca(SumType);
4159     XorReduced = Builder.CreateAlloca(XorType);
4160   }
4161
4162   // Store initial values of reductions into global variables.
4163   Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
4164   Builder.CreateStore(Builder.getInt32(1), XorReduced);
4165
4166   InsertPointTy FirstBodyIP, FirstBodyAllocaIP;
4167   auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4168                             InsertPointTy CodeGenIP) {
4169     IRBuilderBase::InsertPointGuard Guard(Builder);
4170     Builder.restoreIP(CodeGenIP);
4171
4172     uint32_t StrSize;
4173     Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4174     Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4175     Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4176     Value *SumLocal =
4177         Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
4178     Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
4179     Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
4180     Builder.CreateStore(Sum, SumReduced);
4181
4182     FirstBodyIP = Builder.saveIP();
4183     FirstBodyAllocaIP = InnerAllocaIP;
4184   };
4185
4186   InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
4187   auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4188                              InsertPointTy CodeGenIP) {
4189     IRBuilderBase::InsertPointGuard Guard(Builder);
4190     Builder.restoreIP(CodeGenIP);
4191
4192     uint32_t StrSize;
4193     Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4194     Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4195     Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4196     Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
4197     Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
4198     Builder.CreateStore(Xor, XorReduced);
4199
4200     SecondBodyIP = Builder.saveIP();
4201     SecondBodyAllocaIP = InnerAllocaIP;
4202   };
4203
4204   // Privatization for reduction creates local copies of reduction variables and
4205   // initializes them to reduction-neutral values. The same privatization
4206   // callback is used for both loops, with dispatch based on the value being
4207   // privatized.
4208   Value *SumPrivatized;
4209   Value *XorPrivatized;
4210   auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4211                     Value &Original, Value &Inner, Value *&ReplVal) {
4212     IRBuilderBase::InsertPointGuard Guard(Builder);
4213     Builder.restoreIP(InnerAllocaIP);
4214     if (&Original == SumReduced) {
4215       SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
4216       ReplVal = SumPrivatized;
4217     } else if (&Original == XorReduced) {
4218       XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
4219       ReplVal = XorPrivatized;
4220     } else {
4221       ReplVal = &Inner;
4222       return CodeGenIP;
4223     }
4224
4225     Builder.restoreIP(CodeGenIP);
4226     if (&Original == SumReduced)
4227       Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4228                           SumPrivatized);
4229     else if (&Original == XorReduced)
4230       Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4231
4232     return Builder.saveIP();
4233   };
4234
4235   // Do nothing in finalization.
4236   auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4237
4238   Builder.restoreIP(
4239       OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
4240                                 FiniCB, /* IfCondition */ nullptr,
4241                                 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4242                                 /* IsCancellable */ false));
4243   InsertPointTy AfterIP = OMPBuilder.createParallel(
4244       {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB,
4245       /* IfCondition */ nullptr,
4246       /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4247       /* IsCancellable */ false);
4248
4249   OMPBuilder.createReductions(
4250       FirstBodyIP, FirstBodyAllocaIP,
4251       {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
4252   OMPBuilder.createReductions(
4253       SecondBodyIP, SecondBodyAllocaIP,
4254       {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
4255
4256   Builder.restoreIP(AfterIP);
4257   Builder.CreateRetVoid();
4258
4259   OMPBuilder.finalize(F);
4260
4261   // The IR must be valid.
4262   EXPECT_FALSE(verifyModule(*M));
4263
4264   // Two different outlined functions must have been created.
4265   SmallVector<CallInst *> ForkCalls;
4266   findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4267             ForkCalls);
4268   ASSERT_EQ(ForkCalls.size(), 2u);
4269   Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0);
4270   Function *FirstCallee = cast<Function>(CalleeVal);
4271   CalleeVal = cast<Constant>(ForkCalls[1]->getOperand(2))->getOperand(0);
4272   Function *SecondCallee = cast<Function>(CalleeVal);
4273   EXPECT_NE(FirstCallee, SecondCallee);
4274
4275   // Two different reduction functions must have been created.
4276   SmallVector<CallInst *> ReduceCalls;
4277   findCalls(FirstCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder,
4278             ReduceCalls);
4279   ASSERT_EQ(ReduceCalls.size(), 1u);
4280   auto *AddReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4281   ReduceCalls.clear();
4282   findCalls(SecondCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce,
4283             OMPBuilder, ReduceCalls);
4284   auto *XorReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4285   EXPECT_NE(AddReduction, XorReduction);
4286
4287   // Each reduction function does its own kind of reduction.
4288   BasicBlock *FnReductionBB = &AddReduction->getEntryBlock();
4289   auto *Bitcast = findSingleUserInBlock<BitCastInst>(AddReduction->getArg(0),
4290                                                      FnReductionBB);
4291   ASSERT_NE(Bitcast, nullptr);
4292   Value *FirstLHSPtr =
4293       findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB);
4294   ASSERT_NE(FirstLHSPtr, nullptr);
4295   Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4296   ASSERT_NE(Opaque, nullptr);
4297   Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4298   ASSERT_NE(Bitcast, nullptr);
4299   Instruction::BinaryOps Opcode = Instruction::FAdd;
4300   EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode));
4301
4302   FnReductionBB = &XorReduction->getEntryBlock();
4303   Bitcast = findSingleUserInBlock<BitCastInst>(XorReduction->getArg(0),
4304                                                FnReductionBB);
4305   ASSERT_NE(Bitcast, nullptr);
4306   Value *SecondLHSPtr =
4307       findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB);
4308   ASSERT_NE(FirstLHSPtr, nullptr);
4309   Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4310   ASSERT_NE(Opaque, nullptr);
4311   Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4312   ASSERT_NE(Bitcast, nullptr);
4313   Opcode = Instruction::Xor;
4314   EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode));
4315 }
4316
4317 TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
4318   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4319   using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4320   OpenMPIRBuilder OMPBuilder(*M);
4321   OMPBuilder.initialize();
4322   F->setName("func");
4323   IRBuilder<> Builder(BB);
4324
4325   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4326   Builder.CreateBr(EnterBB);
4327   Builder.SetInsertPoint(EnterBB);
4328   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4329
4330   llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4331   llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4332
4333   auto FiniCB = [&](InsertPointTy IP) {};
4334   auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4335   SectionCBVector.push_back(SectionCB);
4336
4337   auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4338                    llvm::Value &, llvm::Value &Val,
4339                    llvm::Value *&ReplVal) { return CodeGenIP; };
4340   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4341                                     F->getEntryBlock().getFirstInsertionPt());
4342   Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4343                                               PrivCB, FiniCB, false, false));
4344   Builder.CreateRetVoid(); // Required at the end of the function
4345   EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);
4346   EXPECT_FALSE(verifyModule(*M, &errs()));
4347 }
4348
4349 TEST_F(OpenMPIRBuilderTest, CreateSections) {
4350   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4351   using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4352   OpenMPIRBuilder OMPBuilder(*M);
4353   OMPBuilder.initialize();
4354   F->setName("func");
4355   IRBuilder<> Builder(BB);
4356
4357   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4358   llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4359   llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4360
4361   BasicBlock *SwitchBB = nullptr;
4362   AllocaInst *PrivAI = nullptr;
4363   SwitchInst *Switch = nullptr;
4364
4365   unsigned NumBodiesGenerated = 0;
4366   unsigned NumFiniCBCalls = 0;
4367   PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
4368
4369   auto FiniCB = [&](InsertPointTy IP) {
4370     ++NumFiniCBCalls;
4371     BasicBlock *IPBB = IP.getBlock();
4372     EXPECT_NE(IPBB->end(), IP.getPoint());
4373   };
4374
4375   auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4376     ++NumBodiesGenerated;
4377     CaseBBs.push_back(CodeGenIP.getBlock());
4378     SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();
4379     Builder.restoreIP(CodeGenIP);
4380     Builder.CreateStore(F->arg_begin(), PrivAI);
4381     Value *PrivLoad =
4382         Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca");
4383     Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
4384   };
4385   auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4386                    llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4387     // TODO: Privatization not implemented yet
4388     return CodeGenIP;
4389   };
4390
4391   SectionCBVector.push_back(SectionCB);
4392   SectionCBVector.push_back(SectionCB);
4393
4394   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4395                                     F->getEntryBlock().getFirstInsertionPt());
4396   Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4397                                               PrivCB, FiniCB, false, false));
4398   Builder.CreateRetVoid(); // Required at the end of the function
4399
4400   // Switch BB's predecessor is loop condition BB, whose successor at index 1 is
4401   // loop's exit BB
4402   BasicBlock *ForExitBB =
4403       SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(1);
4404   EXPECT_NE(ForExitBB, nullptr);
4405
4406   EXPECT_NE(PrivAI, nullptr);
4407   Function *OutlinedFn = PrivAI->getFunction();
4408   EXPECT_EQ(F, OutlinedFn);
4409   EXPECT_FALSE(verifyModule(*M, &errs()));
4410   EXPECT_EQ(OutlinedFn->arg_size(), 1U);
4411
4412   BasicBlock *LoopPreheaderBB =
4413       OutlinedFn->getEntryBlock().getSingleSuccessor();
4414   // loop variables are 5 - lower bound, upper bound, stride, islastiter, and
4415   // iterator/counter
4416   bool FoundForInit = false;
4417   for (Instruction &Inst : *LoopPreheaderBB) {
4418     if (isa<CallInst>(Inst)) {
4419       if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4420           "__kmpc_for_static_init_4u") {
4421         FoundForInit = true;
4422       }
4423     }
4424   }
4425   EXPECT_EQ(FoundForInit, true);
4426
4427   bool FoundForExit = false;
4428   bool FoundBarrier = false;
4429   for (Instruction &Inst : *ForExitBB) {
4430     if (isa<CallInst>(Inst)) {
4431       if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4432           "__kmpc_for_static_fini") {
4433         FoundForExit = true;
4434       }
4435       if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4436           "__kmpc_barrier") {
4437         FoundBarrier = true;
4438       }
4439       if (FoundForExit && FoundBarrier)
4440         break;
4441     }
4442   }
4443   EXPECT_EQ(FoundForExit, true);
4444   EXPECT_EQ(FoundBarrier, true);
4445
4446   EXPECT_NE(SwitchBB, nullptr);
4447   EXPECT_NE(SwitchBB->getTerminator(), nullptr);
4448   EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true);
4449   Switch = cast<SwitchInst>(SwitchBB->getTerminator());
4450   EXPECT_EQ(Switch->getNumCases(), 2U);
4451
4452   EXPECT_EQ(CaseBBs.size(), 2U);
4453   for (auto *&CaseBB : CaseBBs) {
4454     EXPECT_EQ(CaseBB->getParent(), OutlinedFn);
4455   }
4456
4457   ASSERT_EQ(NumBodiesGenerated, 2U);
4458   ASSERT_EQ(NumFiniCBCalls, 1U);
4459   EXPECT_FALSE(verifyModule(*M, &errs()));
4460 }
4461
4462 TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {
4463   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4464   using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4465   OpenMPIRBuilder OMPBuilder(*M);
4466   OMPBuilder.initialize();
4467   F->setName("func");
4468   IRBuilder<> Builder(BB);
4469
4470   BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4471   Builder.CreateBr(EnterBB);
4472   Builder.SetInsertPoint(EnterBB);
4473   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4474
4475   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4476                                     F->getEntryBlock().getFirstInsertionPt());
4477   llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4478   auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4479                    llvm::Value &, llvm::Value &Val,
4480                    llvm::Value *&ReplVal) { return CodeGenIP; };
4481   auto FiniCB = [&](InsertPointTy IP) {};
4482
4483   Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4484                                               PrivCB, FiniCB, false, true));
4485   Builder.CreateRetVoid(); // Required at the end of the function
4486   for (auto &Inst : instructions(*F)) {
4487     EXPECT_FALSE(isa<CallInst>(Inst) &&
4488                  cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4489                      "__kmpc_barrier" &&
4490                  "call to function __kmpc_barrier found with nowait");
4491   }
4492 }
4493
4494 TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) {
4495   OpenMPIRBuilder OMPBuilder(*M);
4496   OMPBuilder.initialize();
4497
4498   IRBuilder<> Builder(BB);
4499
4500   SmallVector<uint64_t> Mappings = {0, 1};
4501   GlobalVariable *OffloadMaptypesGlobal =
4502       OMPBuilder.createOffloadMaptypes(Mappings, "offload_maptypes");
4503   EXPECT_FALSE(M->global_empty());
4504   EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes");
4505   EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
4506   EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
4507   EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
4508   EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
4509   Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
4510   EXPECT_TRUE(isa<ConstantDataArray>(Initializer));
4511   ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Initializer);
4512   EXPECT_EQ(MappingInit->getNumElements(), Mappings.size());
4513   EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64));
4514   Constant *CA = ConstantDataArray::get(Builder.getContext(), Mappings);
4515   EXPECT_EQ(MappingInit, CA);
4516 }
4517
4518 TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) {
4519   OpenMPIRBuilder OMPBuilder(*M);
4520   OMPBuilder.initialize();
4521
4522   IRBuilder<> Builder(BB);
4523
4524   uint32_t StrSize;
4525   Constant *Cst1 =
4526       OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
4527   Constant *Cst2 =
4528       OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
4529   SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
4530
4531   GlobalVariable *OffloadMaptypesGlobal =
4532       OMPBuilder.createOffloadMapnames(Names, "offload_mapnames");
4533   EXPECT_FALSE(M->global_empty());
4534   EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames");
4535   EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
4536   EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
4537   EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
4538   EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
4539   Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
4540   EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts()));
4541   EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts()));
4542
4543   GlobalVariable *Name1Gbl =
4544       cast<GlobalVariable>(Initializer->getOperand(0)->stripPointerCasts());
4545   EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer()));
4546   ConstantDataArray *Name1GblCA =
4547       dyn_cast<ConstantDataArray>(Name1Gbl->getInitializer());
4548   EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;");
4549
4550   GlobalVariable *Name2Gbl =
4551       cast<GlobalVariable>(Initializer->getOperand(1)->stripPointerCasts());
4552   EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer()));
4553   ConstantDataArray *Name2GblCA =
4554       dyn_cast<ConstantDataArray>(Name2Gbl->getInitializer());
4555   EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;");
4556
4557   EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy());
4558   EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size());
4559 }
4560
4561 TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) {
4562   OpenMPIRBuilder OMPBuilder(*M);
4563   OMPBuilder.initialize();
4564   F->setName("func");
4565   IRBuilder<> Builder(BB);
4566
4567   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4568
4569   unsigned TotalNbOperand = 2;
4570
4571   OpenMPIRBuilder::MapperAllocas MapperAllocas;
4572   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4573                                     F->getEntryBlock().getFirstInsertionPt());
4574   OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
4575   EXPECT_NE(MapperAllocas.ArgsBase, nullptr);
4576   EXPECT_NE(MapperAllocas.Args, nullptr);
4577   EXPECT_NE(MapperAllocas.ArgSizes, nullptr);
4578   EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy());
4579   ArrayType *ArrType =
4580       dyn_cast<ArrayType>(MapperAllocas.ArgsBase->getAllocatedType());
4581   EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4582   EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()
4583                   ->getArrayElementType()
4584                   ->isPointerTy());
4585   EXPECT_TRUE(
4586       cast<PointerType>(
4587           MapperAllocas.ArgsBase->getAllocatedType()->getArrayElementType())
4588           ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty()));
4589
4590   EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy());
4591   ArrType = dyn_cast<ArrayType>(MapperAllocas.Args->getAllocatedType());
4592   EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4593   EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()
4594                   ->getArrayElementType()
4595                   ->isPointerTy());
4596   EXPECT_TRUE(cast<PointerType>(
4597                   MapperAllocas.Args->getAllocatedType()->getArrayElementType())
4598                   ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty()));
4599
4600   EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy());
4601   ArrType = dyn_cast<ArrayType>(MapperAllocas.ArgSizes->getAllocatedType());
4602   EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4603   EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()
4604                   ->getArrayElementType()
4605                   ->isIntegerTy(64));
4606 }
4607
4608 TEST_F(OpenMPIRBuilderTest, EmitMapperCall) {
4609   OpenMPIRBuilder OMPBuilder(*M);
4610   OMPBuilder.initialize();
4611   F->setName("func");
4612   IRBuilder<> Builder(BB);
4613   LLVMContext &Ctx = M->getContext();
4614
4615   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4616
4617   unsigned TotalNbOperand = 2;
4618
4619   OpenMPIRBuilder::MapperAllocas MapperAllocas;
4620   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4621                                     F->getEntryBlock().getFirstInsertionPt());
4622   OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
4623
4624   auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
4625       omp::OMPRTL___tgt_target_data_begin_mapper);
4626
4627   SmallVector<uint64_t> Flags = {0, 2};
4628
4629   uint32_t StrSize;
4630   Constant *SrcLocCst =
4631       OMPBuilder.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize);
4632   Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocCst, StrSize);
4633
4634   Constant *Cst1 =
4635       OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
4636   Constant *Cst2 =
4637       OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
4638   SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
4639
4640   GlobalVariable *Maptypes =
4641       OMPBuilder.createOffloadMaptypes(Flags, ".offload_maptypes");
4642   Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32(
4643       ArrayType::get(Type::getInt64Ty(Ctx), TotalNbOperand), Maptypes,
4644       /*Idx0=*/0, /*Idx1=*/0);
4645
4646   GlobalVariable *Mapnames =
4647       OMPBuilder.createOffloadMapnames(Names, ".offload_mapnames");
4648   Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32(
4649       ArrayType::get(Type::getInt8PtrTy(Ctx), TotalNbOperand), Mapnames,
4650       /*Idx0=*/0, /*Idx1=*/0);
4651
4652   OMPBuilder.emitMapperCall(Builder.saveIP(), BeginMapperFunc, SrcLocInfo,
4653                             MaptypesArg, MapnamesArg, MapperAllocas, -1,
4654                             TotalNbOperand);
4655
4656   CallInst *MapperCall = dyn_cast<CallInst>(&BB->back());
4657   EXPECT_NE(MapperCall, nullptr);
4658   EXPECT_EQ(MapperCall->arg_size(), 9U);
4659   EXPECT_EQ(MapperCall->getCalledFunction()->getName(),
4660             "__tgt_target_data_begin_mapper");
4661   EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo);
4662   EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64));
4663   EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32));
4664
4665   EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg);
4666   EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg);
4667   EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy());
4668 }
4669
4670 TEST_F(OpenMPIRBuilderTest, CreateTask) {
4671   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4672   OpenMPIRBuilder OMPBuilder(*M);
4673   OMPBuilder.initialize();
4674   F->setName("func");
4675   IRBuilder<> Builder(BB);
4676
4677   AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
4678   AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
4679   Value *Val128 =
4680       Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
4681
4682   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4683     Builder.restoreIP(AllocaIP);
4684     AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
4685                                                 "bodygen.alloca128");
4686
4687     Builder.restoreIP(CodeGenIP);
4688     // Loading and storing captured pointer and values
4689     Builder.CreateStore(Val128, Local128);
4690     Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
4691                                       "bodygen.load32");
4692
4693     LoadInst *PrivLoad128 = Builder.CreateLoad(
4694         Local128->getAllocatedType(), Local128, "bodygen.local.load128");
4695     Value *Cmp = Builder.CreateICmpNE(
4696         Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));
4697     Instruction *ThenTerm, *ElseTerm;
4698     SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
4699                                   &ThenTerm, &ElseTerm);
4700   };
4701
4702   BasicBlock *AllocaBB = Builder.GetInsertBlock();
4703   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4704   OpenMPIRBuilder::LocationDescription Loc(
4705       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4706   Builder.restoreIP(OMPBuilder.createTask(
4707       Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
4708       BodyGenCB));
4709   OMPBuilder.finalize();
4710   Builder.CreateRetVoid();
4711
4712   EXPECT_FALSE(verifyModule(*M, &errs()));
4713
4714   CallInst *TaskAllocCall = dyn_cast<CallInst>(
4715       OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4716           ->user_back());
4717
4718   // Verify the Ident argument
4719   GlobalVariable *Ident = cast<GlobalVariable>(TaskAllocCall->getArgOperand(0));
4720   ASSERT_NE(Ident, nullptr);
4721   EXPECT_TRUE(Ident->hasInitializer());
4722   Constant *Initializer = Ident->getInitializer();
4723   GlobalVariable *SrcStrGlob =
4724       cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
4725   ASSERT_NE(SrcStrGlob, nullptr);
4726   ConstantDataArray *SrcSrc =
4727       dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
4728   ASSERT_NE(SrcSrc, nullptr);
4729
4730   // Verify the num_threads argument.
4731   CallInst *GTID = dyn_cast<CallInst>(TaskAllocCall->getArgOperand(1));
4732   ASSERT_NE(GTID, nullptr);
4733   EXPECT_EQ(GTID->arg_size(), 1U);
4734   EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
4735
4736   // Verify the flags
4737   // TODO: Check for others flags. Currently testing only for tiedness.
4738   ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
4739   ASSERT_NE(Flags, nullptr);
4740   EXPECT_EQ(Flags->getSExtValue(), 1);
4741
4742   // Verify the data size
4743   ConstantInt *DataSize =
4744       dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));
4745   ASSERT_NE(DataSize, nullptr);
4746   EXPECT_EQ(DataSize->getSExtValue(), 24); // 64-bit pointer + 128-bit integer
4747
4748   // TODO: Verify size of shared clause variables
4749
4750   // Verify Wrapper function
4751   Function *WrapperFunc =
4752       dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());
4753   ASSERT_NE(WrapperFunc, nullptr);
4754   EXPECT_FALSE(WrapperFunc->isDeclaration());
4755   CallInst *OutlinedFnCall = dyn_cast<CallInst>(WrapperFunc->begin()->begin());
4756   ASSERT_NE(OutlinedFnCall, nullptr);
4757   EXPECT_EQ(WrapperFunc->getArg(0)->getType(), Builder.getInt32Ty());
4758   EXPECT_EQ(OutlinedFnCall->getArgOperand(0), WrapperFunc->getArg(1));
4759
4760   // Verify the presence of `trunc` and `icmp` instructions in Outlined function
4761   Function *OutlinedFn = OutlinedFnCall->getCalledFunction();
4762   ASSERT_NE(OutlinedFn, nullptr);
4763   EXPECT_TRUE(any_of(instructions(OutlinedFn),
4764                      [](Instruction &inst) { return isa<TruncInst>(&inst); }));
4765   EXPECT_TRUE(any_of(instructions(OutlinedFn),
4766                      [](Instruction &inst) { return isa<ICmpInst>(&inst); }));
4767
4768   // Verify the execution of the task
4769   CallInst *TaskCall = dyn_cast<CallInst>(
4770       OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)
4771           ->user_back());
4772   ASSERT_NE(TaskCall, nullptr);
4773   EXPECT_EQ(TaskCall->getArgOperand(0), Ident);
4774   EXPECT_EQ(TaskCall->getArgOperand(1), GTID);
4775   EXPECT_EQ(TaskCall->getArgOperand(2), TaskAllocCall);
4776
4777   // Verify that the argument data has been copied
4778   for (User *in : TaskAllocCall->users()) {
4779     if (MemCpyInst *memCpyInst = dyn_cast<MemCpyInst>(in)) {
4780       EXPECT_EQ(memCpyInst->getDest(), TaskAllocCall);
4781     }
4782   }
4783 }
4784
4785 TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
4786   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4787   OpenMPIRBuilder OMPBuilder(*M);
4788   OMPBuilder.initialize();
4789   F->setName("func");
4790   IRBuilder<> Builder(BB);
4791
4792   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4793
4794   BasicBlock *AllocaBB = Builder.GetInsertBlock();
4795   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4796   OpenMPIRBuilder::LocationDescription Loc(
4797       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4798   Builder.restoreIP(OMPBuilder.createTask(
4799       Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
4800       BodyGenCB));
4801   OMPBuilder.finalize();
4802   Builder.CreateRetVoid();
4803
4804   EXPECT_FALSE(verifyModule(*M, &errs()));
4805 }
4806
4807 TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
4808   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4809   OpenMPIRBuilder OMPBuilder(*M);
4810   OMPBuilder.initialize();
4811   F->setName("func");
4812   IRBuilder<> Builder(BB);
4813   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4814   BasicBlock *AllocaBB = Builder.GetInsertBlock();
4815   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4816   OpenMPIRBuilder::LocationDescription Loc(
4817       InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4818   Builder.restoreIP(OMPBuilder.createTask(
4819       Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
4820       /*Tied=*/false));
4821   OMPBuilder.finalize();
4822   Builder.CreateRetVoid();
4823
4824   // Check for the `Tied` argument
4825   CallInst *TaskAllocCall = dyn_cast<CallInst>(
4826       OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4827           ->user_back());
4828   ASSERT_NE(TaskAllocCall, nullptr);
4829   ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
4830   ASSERT_NE(Flags, nullptr);
4831   EXPECT_EQ(Flags->getZExtValue() & 1U, 0U);
4832
4833   EXPECT_FALSE(verifyModule(*M, &errs()));
4834 }
4835
4836 TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
4837   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4838   OpenMPIRBuilder OMPBuilder(*M);
4839   OMPBuilder.initialize();
4840   F->setName("func");
4841   IRBuilder<> Builder(BB);
4842   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4843   IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
4844   BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4845   Builder.SetInsertPoint(BodyBB);
4846   Value *Final = Builder.CreateICmp(
4847       CmpInst::Predicate::ICMP_EQ, F->getArg(0),
4848       ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
4849   OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
4850   Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
4851                                           /*Tied=*/false, Final));
4852   OMPBuilder.finalize();
4853   Builder.CreateRetVoid();
4854
4855   // Check for the `Tied` argument
4856   CallInst *TaskAllocCall = dyn_cast<CallInst>(
4857       OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4858           ->user_back());
4859   ASSERT_NE(TaskAllocCall, nullptr);
4860   BinaryOperator *OrInst =
4861       dyn_cast<BinaryOperator>(TaskAllocCall->getArgOperand(2));
4862   ASSERT_NE(OrInst, nullptr);
4863   EXPECT_EQ(OrInst->getOpcode(), BinaryOperator::BinaryOps::Or);
4864
4865   // One of the arguments to `or` instruction is the tied flag, which is equal
4866   // to zero.
4867   EXPECT_TRUE(any_of(OrInst->operands(), [](Value *op) {
4868     if (ConstantInt *TiedValue = dyn_cast<ConstantInt>(op))
4869       return TiedValue->getSExtValue() == 0;
4870     return false;
4871   }));
4872
4873   // One of the arguments to `or` instruction is the final condition.
4874   EXPECT_TRUE(any_of(OrInst->operands(), [Final](Value *op) {
4875     if (SelectInst *Select = dyn_cast<SelectInst>(op)) {
4876       ConstantInt *TrueValue = dyn_cast<ConstantInt>(Select->getTrueValue());
4877       ConstantInt *FalseValue = dyn_cast<ConstantInt>(Select->getFalseValue());
4878       if (!TrueValue || !FalseValue)
4879         return false;
4880       return Select->getCondition() == Final &&
4881              TrueValue->getSExtValue() == 2 && FalseValue->getSExtValue() == 0;
4882     }
4883     return false;
4884   }));
4885
4886   EXPECT_FALSE(verifyModule(*M, &errs()));
4887 }
4888
4889 } // namespace