?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Run DCE after a LoopFlatten test to reduce spurious output [nfc]
[llvm-project.git] / llvm / unittests / Frontend / OpenMPIRBuilderTest.cpp
blobb9e9655b89108d23e594b1e732a6b702726c7046
1 //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Frontend/OpenMP/OMPConstants.h"
10 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
11 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
12 #include "llvm/IR/BasicBlock.h"
13 #include "llvm/IR/DIBuilder.h"
14 #include "llvm/IR/Function.h"
15 #include "llvm/IR/InstIterator.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/IR/LLVMContext.h"
18 #include "llvm/IR/Module.h"
19 #include "llvm/IR/Verifier.h"
20 #include "llvm/Passes/PassBuilder.h"
21 #include "llvm/Support/Casting.h"
22 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
23 #include "gmock/gmock.h"
24 #include "gtest/gtest.h"
25 #include <optional>
26
27 using namespace llvm;
28 using namespace omp;
29
30 namespace {
31
32 /// Create an instruction that uses the values in \p Values. We use "printf"
33 /// just because it is often used for this purpose in test code, but it is never
34 /// executed here.
35 static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr,
36 ArrayRef<Value *> Values) {
37 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
38
39 GlobalVariable *GV = Builder.CreateGlobalString(FormatStr, "", 0, M);
40 Constant *Zero = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
41 Constant *Indices[] = {Zero, Zero};
42 Constant *FormatStrConst =
43 ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, Indices);
44
45 Function *PrintfDecl = M->getFunction("printf");
46 if (!PrintfDecl) {
47 GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
48 FunctionType *Ty = FunctionType::get(Builder.getInt32Ty(), true);
49 PrintfDecl = Function::Create(Ty, Linkage, "printf", M);
50 }
51
52 SmallVector<Value *, 4> Args;
53 Args.push_back(FormatStrConst);
54 Args.append(Values.begin(), Values.end());
55 return Builder.CreateCall(PrintfDecl, Args);
56 }
57
58 /// Verify that blocks in \p RefOrder are corresponds to the depth-first visit
59 /// order the control flow of \p F.
60 ///
61 /// This is an easy way to verify the branching structure of the CFG without
62 /// checking every branch instruction individually. For the CFG of a
63 /// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering
64 /// the body, i.e. the DFS order corresponds to the execution order with one
65 /// loop iteration.
66 static testing::AssertionResult
67 verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
68 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
69 ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
70
71 df_iterator_default_set<BasicBlock *, 16> Visited;
72 auto DFS = llvm::depth_first_ext(&F->getEntryBlock(), Visited);
73
74 BasicBlock *Prev = nullptr;
75 for (BasicBlock *BB : DFS) {
76 if (It != E && BB == *It) {
77 Prev = *It;
78 ++It;
79 }
80 }
81
82 if (It == E)
83 return testing::AssertionSuccess();
84 if (!Prev)
85 return testing::AssertionFailure()
86 << "Did not find " << (*It)->getName() << " in control flow";
87 return testing::AssertionFailure()
88 << "Expected " << Prev->getName() << " before " << (*It)->getName()
89 << " in control flow";
90 }
91
92 /// Verify that blocks in \p RefOrder are in the same relative order in the
93 /// linked lists of blocks in \p F. The linked list may contain additional
94 /// blocks in-between.
95 ///
96 /// While the order in the linked list is not relevant for semantics, keeping
97 /// the order roughly in execution order makes its printout easier to read.
98 static testing::AssertionResult
99 verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
100 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
101 ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
102
103 BasicBlock *Prev = nullptr;
104 for (BasicBlock &BB : *F) {
105 if (It != E && &BB == *It) {
106 Prev = *It;
107 ++It;
108 }
109 }
110
111 if (It == E)
112 return testing::AssertionSuccess();
113 if (!Prev)
114 return testing::AssertionFailure() << "Did not find " << (*It)->getName()
115 << " in function " << F->getName();
116 return testing::AssertionFailure()
117 << "Expected " << Prev->getName() << " before " << (*It)->getName()
118 << " in function " << F->getName();
119 }
120
121 /// Populate Calls with call instructions calling the function with the given
122 /// FnID from the given function F.
123 static void findCalls(Function *F, omp::RuntimeFunction FnID,
124 OpenMPIRBuilder &OMPBuilder,
125 SmallVectorImpl<CallInst *> &Calls) {
126 Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID);
127 for (BasicBlock &BB : *F) {
128 for (Instruction &I : BB) {
129 auto *Call = dyn_cast<CallInst>(&I);
130 if (Call && Call->getCalledFunction() == Fn)
131 Calls.push_back(Call);
132 }
133 }
134 }
135
136 /// Assuming \p F contains only one call to the function with the given \p FnID,
137 /// return that call.
138 static CallInst *findSingleCall(Function *F, omp::RuntimeFunction FnID,
139 OpenMPIRBuilder &OMPBuilder) {
140 SmallVector<CallInst *, 1> Calls;
141 findCalls(F, FnID, OMPBuilder, Calls);
142 EXPECT_EQ(1u, Calls.size());
143 if (Calls.size() != 1)
144 return nullptr;
145 return Calls.front();
146 }
147
148 static omp::ScheduleKind getSchedKind(omp::OMPScheduleType SchedType) {
149 switch (SchedType & ~omp::OMPScheduleType::ModifierMask) {
150 case omp::OMPScheduleType::BaseDynamicChunked:
151 return omp::OMP_SCHEDULE_Dynamic;
152 case omp::OMPScheduleType::BaseGuidedChunked:
153 return omp::OMP_SCHEDULE_Guided;
154 case omp::OMPScheduleType::BaseAuto:
155 return omp::OMP_SCHEDULE_Auto;
156 case omp::OMPScheduleType::BaseRuntime:
157 return omp::OMP_SCHEDULE_Runtime;
158 default:
159 llvm_unreachable("unknown type for this test");
160 }
161 }
162
163 class OpenMPIRBuilderTest : public testing::Test {
164 protected:
165 void SetUp() override {
166 M.reset(new Module("MyModule", Ctx));
167 FunctionType *FTy =
168 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
169 /*isVarArg=*/false);
170 F = Function::Create(FTy, Function::ExternalLinkage, "", M.get());
171 BB = BasicBlock::Create(Ctx, "", F);
172
173 DIBuilder DIB(*M);
174 auto File = DIB.createFile("test.dbg", "/src", std::nullopt,
175 std::optional<StringRef>("/src/test.dbg"));
176 auto CU =
177 DIB.createCompileUnit(dwarf::DW_LANG_C, File, "llvm-C", true, "", 0);
178 auto Type =
179 DIB.createSubroutineType(DIB.getOrCreateTypeArray(std::nullopt));
180 auto SP = DIB.createFunction(
181 CU, "foo", "", File, 1, Type, 1, DINode::FlagZero,
182 DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized);
183 F->setSubprogram(SP);
184 auto Scope = DIB.createLexicalBlockFile(SP, File, 0);
185 DIB.finalize();
186 DL = DILocation::get(Ctx, 3, 7, Scope);
187 }
188
189 void TearDown() override {
190 BB = nullptr;
191 M.reset();
192 }
193
194 /// Create a function with a simple loop that calls printf using the logical
195 /// loop counter for use with tests that need a CanonicalLoopInfo object.
196 CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL,
197 OpenMPIRBuilder &OMPBuilder,
198 int UseIVBits,
199 CallInst **Call = nullptr,
200 BasicBlock **BodyCode = nullptr) {
201 OMPBuilder.initialize();
202 F->setName("func");
203
204 IRBuilder<> Builder(BB);
205 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
206 Value *TripCount = F->getArg(0);
207
208 Type *IVType = Type::getIntNTy(Builder.getContext(), UseIVBits);
209 Value *CastedTripCount =
210 Builder.CreateZExtOrTrunc(TripCount, IVType, "tripcount");
211
212 auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP,
213 llvm::Value *LC) {
214 Builder.restoreIP(CodeGenIP);
215 if (BodyCode)
216 *BodyCode = Builder.GetInsertBlock();
217
218 // Add something that consumes the induction variable to the body.
219 CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC});
220 if (Call)
221 *Call = CallInst;
222 };
223 CanonicalLoopInfo *Loop =
224 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount);
225
226 // Finalize the function.
227 Builder.restoreIP(Loop->getAfterIP());
228 Builder.CreateRetVoid();
229
230 return Loop;
231 }
232
233 LLVMContext Ctx;
234 std::unique_ptr<Module> M;
235 Function *F;
236 BasicBlock *BB;
237 DebugLoc DL;
238 };
239
240 class OpenMPIRBuilderTestWithParams
241 : public OpenMPIRBuilderTest,
242 public ::testing::WithParamInterface<omp::OMPScheduleType> {};
243
244 class OpenMPIRBuilderTestWithIVBits
245 : public OpenMPIRBuilderTest,
246 public ::testing::WithParamInterface<int> {};
247
248 // Returns the value stored in the given allocation. Returns null if the given
249 // value is not a result of an InstTy instruction, if no value is stored or if
250 // there is more than one store.
251 template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) {
252 Instruction *Inst = dyn_cast<InstTy>(AllocaValue);
253 if (!Inst)
254 return nullptr;
255 StoreInst *Store = nullptr;
256 for (Use &U : Inst->uses()) {
257 if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) {
258 EXPECT_EQ(Store, nullptr);
259 Store = CandidateStore;
260 }
261 }
262 if (!Store)
263 return nullptr;
264 return Store->getValueOperand();
265 }
266
267 // Returns the value stored in the aggregate argument of an outlined function,
268 // or nullptr if it is not found.
269 static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,
270 unsigned Idx) {
271 GetElementPtrInst *GEPAtIdx = nullptr;
272 // Find GEP instruction at that index.
273 for (User *Usr : Aggregate->users()) {
274 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Usr);
275 if (!GEP)
276 continue;
277
278 if (GEP->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx), Idx))
279 continue;
280
281 EXPECT_EQ(GEPAtIdx, nullptr);
282 GEPAtIdx = GEP;
283 }
284
285 EXPECT_NE(GEPAtIdx, nullptr);
286 EXPECT_EQ(GEPAtIdx->getNumUses(), 1U);
287
288 // Find the value stored to the aggregate.
289 StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin());
290 Value *StoredAggValue = StoreToAgg->getValueOperand();
291
292 Value *StoredValue = nullptr;
293
294 // Find the value stored to the value stored in the aggregate.
295 for (User *Usr : StoredAggValue->users()) {
296 StoreInst *Store = dyn_cast<StoreInst>(Usr);
297 if (!Store)
298 continue;
299
300 if (Store->getPointerOperand() != StoredAggValue)
301 continue;
302
303 EXPECT_EQ(StoredValue, nullptr);
304 StoredValue = Store->getValueOperand();
305 }
306
307 return StoredValue;
308 }
309
310 // Returns the aggregate that the value is originating from.
311 static Value *findAggregateFromValue(Value *V) {
312 // Expects a load instruction that loads from the aggregate.
313 LoadInst *Load = dyn_cast<LoadInst>(V);
314 EXPECT_NE(Load, nullptr);
315 // Find the GEP instruction used in the load instruction.
316 GetElementPtrInst *GEP =
317 dyn_cast<GetElementPtrInst>(Load->getPointerOperand());
318 EXPECT_NE(GEP, nullptr);
319 // Find the aggregate used in the GEP instruction.
320 Value *Aggregate = GEP->getPointerOperand();
321
322 return Aggregate;
323 }
324
325 TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
326 OpenMPIRBuilder OMPBuilder(*M);
327 OMPBuilder.initialize();
328
329 IRBuilder<> Builder(BB);
330
331 OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
332 EXPECT_TRUE(M->global_empty());
333 EXPECT_EQ(M->size(), 1U);
334 EXPECT_EQ(F->size(), 1U);
335 EXPECT_EQ(BB->size(), 0U);
336
337 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
338 OMPBuilder.createBarrier(Loc, OMPD_for);
339 EXPECT_FALSE(M->global_empty());
340 EXPECT_EQ(M->size(), 3U);
341 EXPECT_EQ(F->size(), 1U);
342 EXPECT_EQ(BB->size(), 2U);
343
344 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
345 EXPECT_NE(GTID, nullptr);
346 EXPECT_EQ(GTID->arg_size(), 1U);
347 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
348 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
349 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
350
351 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
352 EXPECT_NE(Barrier, nullptr);
353 EXPECT_EQ(Barrier->arg_size(), 2U);
354 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier");
355 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
356 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
357
358 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
359
360 Builder.CreateUnreachable();
361 EXPECT_FALSE(verifyModule(*M, &errs()));
362 }
363
364 TEST_F(OpenMPIRBuilderTest, CreateCancel) {
365 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
366 OpenMPIRBuilder OMPBuilder(*M);
367 OMPBuilder.initialize();
368
369 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
370 new UnreachableInst(Ctx, CBB);
371 auto FiniCB = [&](InsertPointTy IP) {
372 ASSERT_NE(IP.getBlock(), nullptr);
373 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
374 BranchInst::Create(CBB, IP.getBlock());
375 };
376 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
377
378 IRBuilder<> Builder(BB);
379
380 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
381 auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
382 Builder.restoreIP(NewIP);
383 EXPECT_FALSE(M->global_empty());
384 EXPECT_EQ(M->size(), 4U);
385 EXPECT_EQ(F->size(), 4U);
386 EXPECT_EQ(BB->size(), 4U);
387
388 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
389 EXPECT_NE(GTID, nullptr);
390 EXPECT_EQ(GTID->arg_size(), 1U);
391 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
392 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
393 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
394
395 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
396 EXPECT_NE(Cancel, nullptr);
397 EXPECT_EQ(Cancel->arg_size(), 3U);
398 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
399 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
400 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
401 EXPECT_EQ(Cancel->getNumUses(), 1U);
402 Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
403 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
404 EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
405 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
406 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
407 EXPECT_NE(GTID1, nullptr);
408 EXPECT_EQ(GTID1->arg_size(), 1U);
409 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
410 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
411 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
412 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
413 EXPECT_NE(Barrier, nullptr);
414 EXPECT_EQ(Barrier->arg_size(), 2U);
415 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
416 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
417 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
418 EXPECT_EQ(Barrier->getNumUses(), 0U);
419 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
420 1U);
421 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
422
423 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
424
425 OMPBuilder.popFinalizationCB();
426
427 Builder.CreateUnreachable();
428 EXPECT_FALSE(verifyModule(*M, &errs()));
429 }
430
431 TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
432 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
433 OpenMPIRBuilder OMPBuilder(*M);
434 OMPBuilder.initialize();
435
436 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
437 new UnreachableInst(Ctx, CBB);
438 auto FiniCB = [&](InsertPointTy IP) {
439 ASSERT_NE(IP.getBlock(), nullptr);
440 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
441 BranchInst::Create(CBB, IP.getBlock());
442 };
443 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
444
445 IRBuilder<> Builder(BB);
446
447 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
448 auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
449 Builder.restoreIP(NewIP);
450 EXPECT_FALSE(M->global_empty());
451 EXPECT_EQ(M->size(), 4U);
452 EXPECT_EQ(F->size(), 7U);
453 EXPECT_EQ(BB->size(), 1U);
454 ASSERT_TRUE(isa<BranchInst>(BB->getTerminator()));
455 ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U);
456 BB = BB->getTerminator()->getSuccessor(0);
457 EXPECT_EQ(BB->size(), 4U);
458
459 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
460 EXPECT_NE(GTID, nullptr);
461 EXPECT_EQ(GTID->arg_size(), 1U);
462 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
463 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
464 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
465
466 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
467 EXPECT_NE(Cancel, nullptr);
468 EXPECT_EQ(Cancel->arg_size(), 3U);
469 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
470 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
471 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
472 EXPECT_EQ(Cancel->getNumUses(), 1U);
473 Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
474 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
475 EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
476 EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),
477 NewIP.getBlock());
478 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
479 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
480 EXPECT_NE(GTID1, nullptr);
481 EXPECT_EQ(GTID1->arg_size(), 1U);
482 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
483 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
484 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
485 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
486 EXPECT_NE(Barrier, nullptr);
487 EXPECT_EQ(Barrier->arg_size(), 2U);
488 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
489 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
490 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
491 EXPECT_EQ(Barrier->getNumUses(), 0U);
492 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
493 1U);
494 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
495
496 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
497
498 OMPBuilder.popFinalizationCB();
499
500 Builder.CreateUnreachable();
501 EXPECT_FALSE(verifyModule(*M, &errs()));
502 }
503
504 TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
505 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
506 OpenMPIRBuilder OMPBuilder(*M);
507 OMPBuilder.initialize();
508
509 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
510 new UnreachableInst(Ctx, CBB);
511 auto FiniCB = [&](InsertPointTy IP) {
512 ASSERT_NE(IP.getBlock(), nullptr);
513 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
514 BranchInst::Create(CBB, IP.getBlock());
515 };
516 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
517
518 IRBuilder<> Builder(BB);
519
520 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
521 auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for);
522 Builder.restoreIP(NewIP);
523 EXPECT_FALSE(M->global_empty());
524 EXPECT_EQ(M->size(), 3U);
525 EXPECT_EQ(F->size(), 4U);
526 EXPECT_EQ(BB->size(), 4U);
527
528 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
529 EXPECT_NE(GTID, nullptr);
530 EXPECT_EQ(GTID->arg_size(), 1U);
531 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
532 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
533 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
534
535 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
536 EXPECT_NE(Barrier, nullptr);
537 EXPECT_EQ(Barrier->arg_size(), 2U);
538 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
539 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
540 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
541 EXPECT_EQ(Barrier->getNumUses(), 1U);
542 Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
543 EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
544 EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
545 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);
546 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
547 1U);
548 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0),
549 CBB);
550
551 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
552
553 OMPBuilder.popFinalizationCB();
554
555 Builder.CreateUnreachable();
556 EXPECT_FALSE(verifyModule(*M, &errs()));
557 }
558
559 TEST_F(OpenMPIRBuilderTest, DbgLoc) {
560 OpenMPIRBuilder OMPBuilder(*M);
561 OMPBuilder.initialize();
562 F->setName("func");
563
564 IRBuilder<> Builder(BB);
565
566 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
567 OMPBuilder.createBarrier(Loc, OMPD_for);
568 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
569 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
570 EXPECT_EQ(GTID->getDebugLoc(), DL);
571 EXPECT_EQ(Barrier->getDebugLoc(), DL);
572 EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0)));
573 if (!isa<GlobalVariable>(Barrier->getOperand(0)))
574 return;
575 GlobalVariable *Ident = cast<GlobalVariable>(Barrier->getOperand(0));
576 EXPECT_TRUE(Ident->hasInitializer());
577 if (!Ident->hasInitializer())
578 return;
579 Constant *Initializer = Ident->getInitializer();
580 EXPECT_TRUE(
581 isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()));
582 GlobalVariable *SrcStrGlob =
583 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
584 if (!SrcStrGlob)
585 return;
586 EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer()));
587 ConstantDataArray *SrcSrc =
588 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
589 if (!SrcSrc)
590 return;
591 EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;");
592 }
593
594 TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
595 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
596 OpenMPIRBuilder OMPBuilder(*M);
597 OMPBuilder.initialize();
598 F->setName("func");
599 IRBuilder<> Builder(BB);
600
601 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
602 Builder.CreateBr(EnterBB);
603 Builder.SetInsertPoint(EnterBB);
604 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
605
606 AllocaInst *PrivAI = nullptr;
607
608 unsigned NumBodiesGenerated = 0;
609 unsigned NumPrivatizedVars = 0;
610 unsigned NumFinalizationPoints = 0;
611
612 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
613 ++NumBodiesGenerated;
614
615 Builder.restoreIP(AllocaIP);
616 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
617 Builder.CreateStore(F->arg_begin(), PrivAI);
618
619 Builder.restoreIP(CodeGenIP);
620 Value *PrivLoad =
621 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
622 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
623 Instruction *ThenTerm, *ElseTerm;
624 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
625 &ThenTerm, &ElseTerm);
626 };
627
628 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
629 Value &Orig, Value &Inner,
630 Value *&ReplacementValue) -> InsertPointTy {
631 ++NumPrivatizedVars;
632
633 if (!isa<AllocaInst>(Orig)) {
634 EXPECT_EQ(&Orig, F->arg_begin());
635 ReplacementValue = &Inner;
636 return CodeGenIP;
637 }
638
639 // Since the original value is an allocation, it has a pointer type and
640 // therefore no additional wrapping should happen.
641 EXPECT_EQ(&Orig, &Inner);
642
643 // Trivial copy (=firstprivate).
644 Builder.restoreIP(AllocaIP);
645 Type *VTy = ReplacementValue->getType();
646 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
647 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
648 Builder.restoreIP(CodeGenIP);
649 Builder.CreateStore(V, ReplacementValue);
650 return CodeGenIP;
651 };
652
653 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
654
655 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
656 F->getEntryBlock().getFirstInsertionPt());
657 IRBuilder<>::InsertPoint AfterIP =
658 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
659 nullptr, nullptr, OMP_PROC_BIND_default, false);
660 EXPECT_EQ(NumBodiesGenerated, 1U);
661 EXPECT_EQ(NumPrivatizedVars, 1U);
662 EXPECT_EQ(NumFinalizationPoints, 1U);
663
664 Builder.restoreIP(AfterIP);
665 Builder.CreateRetVoid();
666
667 OMPBuilder.finalize();
668
669 EXPECT_NE(PrivAI, nullptr);
670 Function *OutlinedFn = PrivAI->getFunction();
671 EXPECT_NE(F, OutlinedFn);
672 EXPECT_FALSE(verifyModule(*M, &errs()));
673 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));
674 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse));
675 EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));
676 EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));
677
678 EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
679 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
680
681 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
682 EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
683 User *Usr = OutlinedFn->user_back();
684 ASSERT_TRUE(isa<CallInst>(Usr));
685 CallInst *ForkCI = dyn_cast<CallInst>(Usr);
686 ASSERT_NE(ForkCI, nullptr);
687
688 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
689 EXPECT_EQ(ForkCI->arg_size(), 4U);
690 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
691 EXPECT_EQ(ForkCI->getArgOperand(1),
692 ConstantInt::get(Type::getInt32Ty(Ctx), 1U));
693 EXPECT_EQ(ForkCI, Usr);
694 Value *StoredValue =
695 findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);
696 EXPECT_EQ(StoredValue, F->arg_begin());
697 }
698
699 TEST_F(OpenMPIRBuilderTest, ParallelNested) {
700 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
701 OpenMPIRBuilder OMPBuilder(*M);
702 OMPBuilder.initialize();
703 F->setName("func");
704 IRBuilder<> Builder(BB);
705
706 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
707 Builder.CreateBr(EnterBB);
708 Builder.SetInsertPoint(EnterBB);
709 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
710
711 unsigned NumInnerBodiesGenerated = 0;
712 unsigned NumOuterBodiesGenerated = 0;
713 unsigned NumFinalizationPoints = 0;
714
715 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
716 ++NumInnerBodiesGenerated;
717 };
718
719 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
720 Value &Orig, Value &Inner,
721 Value *&ReplacementValue) -> InsertPointTy {
722 // Trivial copy (=firstprivate).
723 Builder.restoreIP(AllocaIP);
724 Type *VTy = ReplacementValue->getType();
725 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
726 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
727 Builder.restoreIP(CodeGenIP);
728 Builder.CreateStore(V, ReplacementValue);
729 return CodeGenIP;
730 };
731
732 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
733
734 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
735 ++NumOuterBodiesGenerated;
736 Builder.restoreIP(CodeGenIP);
737 BasicBlock *CGBB = CodeGenIP.getBlock();
738 BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());
739 CGBB->getTerminator()->eraseFromParent();
740 ;
741
742 IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel(
743 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
744 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
745
746 Builder.restoreIP(AfterIP);
747 Builder.CreateBr(NewBB);
748 };
749
750 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
751 F->getEntryBlock().getFirstInsertionPt());
752 IRBuilder<>::InsertPoint AfterIP =
753 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
754 nullptr, nullptr, OMP_PROC_BIND_default, false);
755
756 EXPECT_EQ(NumInnerBodiesGenerated, 1U);
757 EXPECT_EQ(NumOuterBodiesGenerated, 1U);
758 EXPECT_EQ(NumFinalizationPoints, 2U);
759
760 Builder.restoreIP(AfterIP);
761 Builder.CreateRetVoid();
762
763 OMPBuilder.finalize();
764
765 EXPECT_EQ(M->size(), 5U);
766 for (Function &OutlinedFn : *M) {
767 if (F == &OutlinedFn || OutlinedFn.isDeclaration())
768 continue;
769 EXPECT_FALSE(verifyModule(*M, &errs()));
770 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
771 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
772 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
773 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
774
775 EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
776 EXPECT_EQ(OutlinedFn.arg_size(), 2U);
777
778 EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
779 User *Usr = OutlinedFn.user_back();
780 ASSERT_TRUE(isa<CallInst>(Usr));
781 CallInst *ForkCI = dyn_cast<CallInst>(Usr);
782 ASSERT_NE(ForkCI, nullptr);
783
784 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
785 EXPECT_EQ(ForkCI->arg_size(), 3U);
786 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
787 EXPECT_EQ(ForkCI->getArgOperand(1),
788 ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
789 EXPECT_EQ(ForkCI, Usr);
790 }
791 }
792
793 TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
794 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
795 OpenMPIRBuilder OMPBuilder(*M);
796 OMPBuilder.initialize();
797 F->setName("func");
798 IRBuilder<> Builder(BB);
799
800 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
801 Builder.CreateBr(EnterBB);
802 Builder.SetInsertPoint(EnterBB);
803 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
804
805 unsigned NumInnerBodiesGenerated = 0;
806 unsigned NumOuterBodiesGenerated = 0;
807 unsigned NumFinalizationPoints = 0;
808
809 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
810 ++NumInnerBodiesGenerated;
811 };
812
813 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
814 Value &Orig, Value &Inner,
815 Value *&ReplacementValue) -> InsertPointTy {
816 // Trivial copy (=firstprivate).
817 Builder.restoreIP(AllocaIP);
818 Type *VTy = ReplacementValue->getType();
819 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
820 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
821 Builder.restoreIP(CodeGenIP);
822 Builder.CreateStore(V, ReplacementValue);
823 return CodeGenIP;
824 };
825
826 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
827
828 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
829 ++NumOuterBodiesGenerated;
830 Builder.restoreIP(CodeGenIP);
831 BasicBlock *CGBB = CodeGenIP.getBlock();
832 BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint());
833 BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt());
834 CGBB->getTerminator()->eraseFromParent();
835 ;
836 NewBB1->getTerminator()->eraseFromParent();
837 ;
838
839 IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel(
840 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
841 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
842
843 Builder.restoreIP(AfterIP1);
844 Builder.CreateBr(NewBB1);
845
846 IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel(
847 InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
848 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
849
850 Builder.restoreIP(AfterIP2);
851 Builder.CreateBr(NewBB2);
852 };
853
854 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
855 F->getEntryBlock().getFirstInsertionPt());
856 IRBuilder<>::InsertPoint AfterIP =
857 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
858 nullptr, nullptr, OMP_PROC_BIND_default, false);
859
860 EXPECT_EQ(NumInnerBodiesGenerated, 2U);
861 EXPECT_EQ(NumOuterBodiesGenerated, 1U);
862 EXPECT_EQ(NumFinalizationPoints, 3U);
863
864 Builder.restoreIP(AfterIP);
865 Builder.CreateRetVoid();
866
867 OMPBuilder.finalize();
868
869 EXPECT_EQ(M->size(), 6U);
870 for (Function &OutlinedFn : *M) {
871 if (F == &OutlinedFn || OutlinedFn.isDeclaration())
872 continue;
873 EXPECT_FALSE(verifyModule(*M, &errs()));
874 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
875 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
876 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
877 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
878
879 EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
880 EXPECT_EQ(OutlinedFn.arg_size(), 2U);
881
882 unsigned NumAllocas = 0;
883 for (Instruction &I : instructions(OutlinedFn))
884 NumAllocas += isa<AllocaInst>(I);
885 EXPECT_EQ(NumAllocas, 1U);
886
887 EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
888 User *Usr = OutlinedFn.user_back();
889 ASSERT_TRUE(isa<CallInst>(Usr));
890 CallInst *ForkCI = dyn_cast<CallInst>(Usr);
891 ASSERT_NE(ForkCI, nullptr);
892
893 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
894 EXPECT_EQ(ForkCI->arg_size(), 3U);
895 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
896 EXPECT_EQ(ForkCI->getArgOperand(1),
897 ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
898 EXPECT_EQ(ForkCI, Usr);
899 }
900 }
901
902 TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
903 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
904 OpenMPIRBuilder OMPBuilder(*M);
905 OMPBuilder.initialize();
906 F->setName("func");
907 IRBuilder<> Builder(BB);
908
909 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
910 Builder.CreateBr(EnterBB);
911 Builder.SetInsertPoint(EnterBB);
912 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
913
914 AllocaInst *PrivAI = nullptr;
915
916 unsigned NumBodiesGenerated = 0;
917 unsigned NumPrivatizedVars = 0;
918 unsigned NumFinalizationPoints = 0;
919
920 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
921 ++NumBodiesGenerated;
922
923 Builder.restoreIP(AllocaIP);
924 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
925 Builder.CreateStore(F->arg_begin(), PrivAI);
926
927 Builder.restoreIP(CodeGenIP);
928 Value *PrivLoad =
929 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
930 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
931 Instruction *ThenTerm, *ElseTerm;
932 SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm,
933 &ElseTerm);
934 };
935
936 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
937 Value &Orig, Value &Inner,
938 Value *&ReplacementValue) -> InsertPointTy {
939 ++NumPrivatizedVars;
940
941 if (!isa<AllocaInst>(Orig)) {
942 EXPECT_EQ(&Orig, F->arg_begin());
943 ReplacementValue = &Inner;
944 return CodeGenIP;
945 }
946
947 // Since the original value is an allocation, it has a pointer type and
948 // therefore no additional wrapping should happen.
949 EXPECT_EQ(&Orig, &Inner);
950
951 // Trivial copy (=firstprivate).
952 Builder.restoreIP(AllocaIP);
953 Type *VTy = ReplacementValue->getType();
954 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
955 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
956 Builder.restoreIP(CodeGenIP);
957 Builder.CreateStore(V, ReplacementValue);
958 return CodeGenIP;
959 };
960
961 auto FiniCB = [&](InsertPointTy CodeGenIP) {
962 ++NumFinalizationPoints;
963 // No destructors.
964 };
965
966 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
967 F->getEntryBlock().getFirstInsertionPt());
968 IRBuilder<>::InsertPoint AfterIP =
969 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
970 Builder.CreateIsNotNull(F->arg_begin()),
971 nullptr, OMP_PROC_BIND_default, false);
972
973 EXPECT_EQ(NumBodiesGenerated, 1U);
974 EXPECT_EQ(NumPrivatizedVars, 1U);
975 EXPECT_EQ(NumFinalizationPoints, 1U);
976
977 Builder.restoreIP(AfterIP);
978 Builder.CreateRetVoid();
979 OMPBuilder.finalize();
980
981 EXPECT_NE(PrivAI, nullptr);
982 Function *OutlinedFn = PrivAI->getFunction();
983 EXPECT_NE(F, OutlinedFn);
984 EXPECT_FALSE(verifyModule(*M, &errs()));
985
986 EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
987 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
988
989 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
990 ASSERT_EQ(OutlinedFn->getNumUses(), 1U);
991
992 CallInst *ForkCI = nullptr;
993 for (User *Usr : OutlinedFn->users()) {
994 ASSERT_TRUE(isa<CallInst>(Usr));
995 ForkCI = cast<CallInst>(Usr);
996 }
997
998 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call_if");
999 EXPECT_EQ(ForkCI->arg_size(), 5U);
1000 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
1001 EXPECT_EQ(ForkCI->getArgOperand(1),
1002 ConstantInt::get(Type::getInt32Ty(Ctx), 1));
1003 EXPECT_EQ(ForkCI->getArgOperand(3)->getType(), Type::getInt32Ty(Ctx));
1004 }
1005
1006 TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
1007 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1008 OpenMPIRBuilder OMPBuilder(*M);
1009 OMPBuilder.initialize();
1010 F->setName("func");
1011 IRBuilder<> Builder(BB);
1012
1013 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
1014 Builder.CreateBr(EnterBB);
1015 Builder.SetInsertPoint(EnterBB);
1016 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1017
1018 unsigned NumBodiesGenerated = 0;
1019 unsigned NumPrivatizedVars = 0;
1020 unsigned NumFinalizationPoints = 0;
1021
1022 CallInst *CheckedBarrier = nullptr;
1023 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1024 ++NumBodiesGenerated;
1025
1026 Builder.restoreIP(CodeGenIP);
1027
1028 // Create three barriers, two cancel barriers but only one checked.
1029 Function *CBFn, *BFn;
1030
1031 Builder.restoreIP(
1032 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));
1033
1034 CBFn = M->getFunction("__kmpc_cancel_barrier");
1035 BFn = M->getFunction("__kmpc_barrier");
1036 ASSERT_NE(CBFn, nullptr);
1037 ASSERT_EQ(BFn, nullptr);
1038 ASSERT_EQ(CBFn->getNumUses(), 1U);
1039 ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1040 ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
1041 CheckedBarrier = cast<CallInst>(CBFn->user_back());
1042
1043 Builder.restoreIP(
1044 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));
1045 CBFn = M->getFunction("__kmpc_cancel_barrier");
1046 BFn = M->getFunction("__kmpc_barrier");
1047 ASSERT_NE(CBFn, nullptr);
1048 ASSERT_NE(BFn, nullptr);
1049 ASSERT_EQ(CBFn->getNumUses(), 1U);
1050 ASSERT_EQ(BFn->getNumUses(), 1U);
1051 ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
1052 ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
1053
1054 Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel,
1055 false, false));
1056 ASSERT_EQ(CBFn->getNumUses(), 2U);
1057 ASSERT_EQ(BFn->getNumUses(), 1U);
1058 ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
1059 ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1060 ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
1061 };
1062
1063 auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
1064 Value *&) -> InsertPointTy {
1065 ++NumPrivatizedVars;
1066 llvm_unreachable("No privatization callback call expected!");
1067 };
1068
1069 FunctionType *FakeDestructorTy =
1070 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
1071 /*isVarArg=*/false);
1072 auto *FakeDestructor = Function::Create(
1073 FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get());
1074
1075 auto FiniCB = [&](InsertPointTy IP) {
1076 ++NumFinalizationPoints;
1077 Builder.restoreIP(IP);
1078 Builder.CreateCall(FakeDestructor,
1079 {Builder.getInt32(NumFinalizationPoints)});
1080 };
1081
1082 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1083 F->getEntryBlock().getFirstInsertionPt());
1084 IRBuilder<>::InsertPoint AfterIP =
1085 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1086 Builder.CreateIsNotNull(F->arg_begin()),
1087 nullptr, OMP_PROC_BIND_default, true);
1088
1089 EXPECT_EQ(NumBodiesGenerated, 1U);
1090 EXPECT_EQ(NumPrivatizedVars, 0U);
1091 EXPECT_EQ(NumFinalizationPoints, 2U);
1092 EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
1093
1094 Builder.restoreIP(AfterIP);
1095 Builder.CreateRetVoid();
1096 OMPBuilder.finalize();
1097
1098 EXPECT_FALSE(verifyModule(*M, &errs()));
1099
1100 BasicBlock *ExitBB = nullptr;
1101 for (const User *Usr : FakeDestructor->users()) {
1102 const CallInst *CI = dyn_cast<CallInst>(Usr);
1103 ASSERT_EQ(CI->getCalledFunction(), FakeDestructor);
1104 ASSERT_TRUE(isa<BranchInst>(CI->getNextNode()));
1105 ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U);
1106 if (ExitBB)
1107 ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB);
1108 else
1109 ExitBB = CI->getNextNode()->getSuccessor(0);
1110 ASSERT_EQ(ExitBB->size(), 1U);
1111 if (!isa<ReturnInst>(ExitBB->front())) {
1112 ASSERT_TRUE(isa<BranchInst>(ExitBB->front()));
1113 ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U);
1114 ASSERT_TRUE(isa<ReturnInst>(
1115 cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front()));
1116 }
1117 }
1118 }
1119
1120 TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
1121 OpenMPIRBuilder OMPBuilder(*M);
1122 OMPBuilder.initialize();
1123 F->setName("func");
1124 IRBuilder<> Builder(BB);
1125 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1126 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1127
1128 Type *I32Ty = Type::getInt32Ty(M->getContext());
1129 Type *PtrTy = PointerType::get(M->getContext(), 0);
1130 Type *StructTy = StructType::get(I32Ty, PtrTy);
1131 Type *VoidTy = Type::getVoidTy(M->getContext());
1132 FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty);
1133 FunctionCallee TakeI32Func =
1134 M->getOrInsertFunction("take_i32", VoidTy, I32Ty);
1135 FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", PtrTy);
1136 FunctionCallee TakeI32PtrFunc =
1137 M->getOrInsertFunction("take_i32ptr", VoidTy, PtrTy);
1138 FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy);
1139 FunctionCallee TakeStructFunc =
1140 M->getOrInsertFunction("take_struct", VoidTy, StructTy);
1141 FunctionCallee RetStructPtrFunc =
1142 M->getOrInsertFunction("ret_structptr", PtrTy);
1143 FunctionCallee TakeStructPtrFunc =
1144 M->getOrInsertFunction("take_structPtr", VoidTy, PtrTy);
1145 Value *I32Val = Builder.CreateCall(RetI32Func);
1146 Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc);
1147 Value *StructVal = Builder.CreateCall(RetStructFunc);
1148 Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);
1149
1150 Instruction *Internal;
1151 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1152 IRBuilder<>::InsertPointGuard Guard(Builder);
1153 Builder.restoreIP(CodeGenIP);
1154 Internal = Builder.CreateCall(TakeI32Func, I32Val);
1155 Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);
1156 Builder.CreateCall(TakeStructFunc, StructVal);
1157 Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);
1158 };
1159 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
1160 Value &Inner, Value *&ReplacementValue) {
1161 ReplacementValue = &Inner;
1162 return CodeGenIP;
1163 };
1164 auto FiniCB = [](InsertPointTy) {};
1165
1166 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1167 F->getEntryBlock().getFirstInsertionPt());
1168 IRBuilder<>::InsertPoint AfterIP =
1169 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1170 nullptr, nullptr, OMP_PROC_BIND_default, false);
1171 Builder.restoreIP(AfterIP);
1172 Builder.CreateRetVoid();
1173
1174 OMPBuilder.finalize();
1175
1176 EXPECT_FALSE(verifyModule(*M, &errs()));
1177 Function *OutlinedFn = Internal->getFunction();
1178
1179 Type *Arg2Type = OutlinedFn->getArg(2)->getType();
1180 EXPECT_TRUE(Arg2Type->isPointerTy());
1181 }
1182
1183 TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
1184 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1185 OpenMPIRBuilder OMPBuilder(*M);
1186 OMPBuilder.initialize();
1187 IRBuilder<> Builder(BB);
1188 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1189 Value *TripCount = F->getArg(0);
1190
1191 unsigned NumBodiesGenerated = 0;
1192 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
1193 NumBodiesGenerated += 1;
1194
1195 Builder.restoreIP(CodeGenIP);
1196
1197 Value *Cmp = Builder.CreateICmpEQ(LC, TripCount);
1198 Instruction *ThenTerm, *ElseTerm;
1199 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
1200 &ThenTerm, &ElseTerm);
1201 };
1202
1203 CanonicalLoopInfo *Loop =
1204 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount);
1205
1206 Builder.restoreIP(Loop->getAfterIP());
1207 ReturnInst *RetInst = Builder.CreateRetVoid();
1208 OMPBuilder.finalize();
1209
1210 Loop->assertOK();
1211 EXPECT_FALSE(verifyModule(*M, &errs()));
1212
1213 EXPECT_EQ(NumBodiesGenerated, 1U);
1214
1215 // Verify control flow structure (in addition to Loop->assertOK()).
1216 EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock());
1217 EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock());
1218
1219 Instruction *IndVar = Loop->getIndVar();
1220 EXPECT_TRUE(isa<PHINode>(IndVar));
1221 EXPECT_EQ(IndVar->getType(), TripCount->getType());
1222 EXPECT_EQ(IndVar->getParent(), Loop->getHeader());
1223
1224 EXPECT_EQ(Loop->getTripCount(), TripCount);
1225
1226 BasicBlock *Body = Loop->getBody();
1227 Instruction *CmpInst = &Body->front();
1228 EXPECT_TRUE(isa<ICmpInst>(CmpInst));
1229 EXPECT_EQ(CmpInst->getOperand(0), IndVar);
1230
1231 BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor();
1232 EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) {
1233 return SuccBB->getSingleSuccessor() == LatchPred;
1234 }));
1235
1236 EXPECT_EQ(&Loop->getAfter()->front(), RetInst);
1237 }
1238
1239 TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
1240 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1241 OpenMPIRBuilder OMPBuilder(*M);
1242 OMPBuilder.initialize();
1243 IRBuilder<> Builder(BB);
1244
1245 // Check the trip count is computed correctly. We generate the canonical loop
1246 // but rely on the IRBuilder's constant folder to compute the final result
1247 // since all inputs are constant. To verify overflow situations, limit the
1248 // trip count / loop counter widths to 16 bits.
1249 auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1250 bool IsSigned, bool InclusiveStop) -> int64_t {
1251 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1252 Type *LCTy = Type::getInt16Ty(Ctx);
1253 Value *StartVal = ConstantInt::get(LCTy, Start);
1254 Value *StopVal = ConstantInt::get(LCTy, Stop);
1255 Value *StepVal = ConstantInt::get(LCTy, Step);
1256 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1257 CanonicalLoopInfo *Loop =
1258 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1259 StepVal, IsSigned, InclusiveStop);
1260 Loop->assertOK();
1261 Builder.restoreIP(Loop->getAfterIP());
1262 Value *TripCount = Loop->getTripCount();
1263 return cast<ConstantInt>(TripCount)->getValue().getZExtValue();
1264 };
1265
1266 EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
1267 EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
1268 EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
1269 EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
1270 EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
1271 EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
1272 EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
1273 EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
1274 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
1275 EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
1276 EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
1277 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
1278 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
1279
1280 EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
1281 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
1282 EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
1283 EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
1284 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
1285 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
1286
1287 EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
1288 EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
1289 EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
1290 EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
1291 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
1292 EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
1293 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
1294
1295 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
1296 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
1297 EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
1298 EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
1299 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
1300 EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
1301 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
1302 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
1303 EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
1304 EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
1305 EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
1306
1307 // Finalize the function and verify it.
1308 Builder.CreateRetVoid();
1309 OMPBuilder.finalize();
1310 EXPECT_FALSE(verifyModule(*M, &errs()));
1311 }
1312
1313 TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {
1314 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1315 OpenMPIRBuilder OMPBuilder(*M);
1316 OMPBuilder.initialize();
1317 F->setName("func");
1318
1319 IRBuilder<> Builder(BB);
1320
1321 Type *LCTy = F->getArg(0)->getType();
1322 Constant *One = ConstantInt::get(LCTy, 1);
1323 Constant *Two = ConstantInt::get(LCTy, 2);
1324 Value *OuterTripCount =
1325 Builder.CreateAdd(F->getArg(0), Two, "tripcount.outer");
1326 Value *InnerTripCount =
1327 Builder.CreateAdd(F->getArg(0), One, "tripcount.inner");
1328
1329 // Fix an insertion point for ComputeIP.
1330 BasicBlock *LoopNextEnter =
1331 BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1332 Builder.GetInsertBlock()->getNextNode());
1333 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1334 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1335
1336 Builder.SetInsertPoint(LoopNextEnter);
1337 OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL);
1338
1339 CanonicalLoopInfo *InnerLoop = nullptr;
1340 CallInst *InbetweenLead = nullptr;
1341 CallInst *InbetweenTrail = nullptr;
1342 CallInst *Call = nullptr;
1343 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) {
1344 Builder.restoreIP(OuterCodeGenIP);
1345 InbetweenLead =
1346 createPrintfCall(Builder, "In-between lead i=%d\\n", {OuterLC});
1347
1348 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1349 Value *InnerLC) {
1350 Builder.restoreIP(InnerCodeGenIP);
1351 Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC});
1352 };
1353 InnerLoop = OMPBuilder.createCanonicalLoop(
1354 Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner");
1355
1356 Builder.restoreIP(InnerLoop->getAfterIP());
1357 InbetweenTrail =
1358 createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC});
1359 };
1360 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1361 OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer");
1362
1363 // Finish the function.
1364 Builder.restoreIP(OuterLoop->getAfterIP());
1365 Builder.CreateRetVoid();
1366
1367 CanonicalLoopInfo *Collapsed =
1368 OMPBuilder.collapseLoops(DL, {OuterLoop, InnerLoop}, ComputeIP);
1369
1370 OMPBuilder.finalize();
1371 EXPECT_FALSE(verifyModule(*M, &errs()));
1372
1373 // Verify control flow and BB order.
1374 BasicBlock *RefOrder[] = {
1375 Collapsed->getPreheader(), Collapsed->getHeader(),
1376 Collapsed->getCond(), Collapsed->getBody(),
1377 InbetweenLead->getParent(), Call->getParent(),
1378 InbetweenTrail->getParent(), Collapsed->getLatch(),
1379 Collapsed->getExit(), Collapsed->getAfter(),
1380 };
1381 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1382 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1383
1384 // Verify the total trip count.
1385 auto *TripCount = cast<MulOperator>(Collapsed->getTripCount());
1386 EXPECT_EQ(TripCount->getOperand(0), OuterTripCount);
1387 EXPECT_EQ(TripCount->getOperand(1), InnerTripCount);
1388
1389 // Verify the changed indvar.
1390 auto *OuterIV = cast<BinaryOperator>(Call->getOperand(1));
1391 EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv);
1392 EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody());
1393 EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount);
1394 EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar());
1395
1396 auto *InnerIV = cast<BinaryOperator>(Call->getOperand(2));
1397 EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem);
1398 EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody());
1399 EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar());
1400 EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount);
1401
1402 EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV);
1403 EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV);
1404 }
1405
1406 TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
1407 OpenMPIRBuilder OMPBuilder(*M);
1408 CallInst *Call;
1409 BasicBlock *BodyCode;
1410 CanonicalLoopInfo *Loop =
1411 buildSingleLoopFunction(DL, OMPBuilder, 32, &Call, &BodyCode);
1412
1413 Instruction *OrigIndVar = Loop->getIndVar();
1414 EXPECT_EQ(Call->getOperand(1), OrigIndVar);
1415
1416 // Tile the loop.
1417 Constant *TileSize = ConstantInt::get(Loop->getIndVarType(), APInt(32, 7));
1418 std::vector<CanonicalLoopInfo *> GenLoops =
1419 OMPBuilder.tileLoops(DL, {Loop}, {TileSize});
1420
1421 OMPBuilder.finalize();
1422 EXPECT_FALSE(verifyModule(*M, &errs()));
1423
1424 EXPECT_EQ(GenLoops.size(), 2u);
1425 CanonicalLoopInfo *Floor = GenLoops[0];
1426 CanonicalLoopInfo *Tile = GenLoops[1];
1427
1428 BasicBlock *RefOrder[] = {
1429 Floor->getPreheader(), Floor->getHeader(), Floor->getCond(),
1430 Floor->getBody(), Tile->getPreheader(), Tile->getHeader(),
1431 Tile->getCond(), Tile->getBody(), BodyCode,
1432 Tile->getLatch(), Tile->getExit(), Tile->getAfter(),
1433 Floor->getLatch(), Floor->getExit(), Floor->getAfter(),
1434 };
1435 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1436 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1437
1438 // Check the induction variable.
1439 EXPECT_EQ(Call->getParent(), BodyCode);
1440 auto *Shift = cast<AddOperator>(Call->getOperand(1));
1441 EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody());
1442 EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar());
1443 auto *Scale = cast<MulOperator>(Shift->getOperand(0));
1444 EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody());
1445 EXPECT_EQ(Scale->getOperand(0), TileSize);
1446 EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar());
1447 }
1448
1449 TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
1450 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1451 OpenMPIRBuilder OMPBuilder(*M);
1452 OMPBuilder.initialize();
1453 F->setName("func");
1454
1455 IRBuilder<> Builder(BB);
1456 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1457 Value *TripCount = F->getArg(0);
1458 Type *LCTy = TripCount->getType();
1459
1460 BasicBlock *BodyCode = nullptr;
1461 CanonicalLoopInfo *InnerLoop = nullptr;
1462 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1463 llvm::Value *OuterLC) {
1464 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1465 llvm::Value *InnerLC) {
1466 Builder.restoreIP(InnerCodeGenIP);
1467 BodyCode = Builder.GetInsertBlock();
1468
1469 // Add something that consumes the induction variables to the body.
1470 createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1471 };
1472 InnerLoop = OMPBuilder.createCanonicalLoop(
1473 OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner");
1474 };
1475 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1476 Loc, OuterLoopBodyGenCB, TripCount, "outer");
1477
1478 // Finalize the function.
1479 Builder.restoreIP(OuterLoop->getAfterIP());
1480 Builder.CreateRetVoid();
1481
1482 // Tile to loop nest.
1483 Constant *OuterTileSize = ConstantInt::get(LCTy, APInt(32, 11));
1484 Constant *InnerTileSize = ConstantInt::get(LCTy, APInt(32, 7));
1485 std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops(
1486 DL, {OuterLoop, InnerLoop}, {OuterTileSize, InnerTileSize});
1487
1488 OMPBuilder.finalize();
1489 EXPECT_FALSE(verifyModule(*M, &errs()));
1490
1491 EXPECT_EQ(GenLoops.size(), 4u);
1492 CanonicalLoopInfo *Floor1 = GenLoops[0];
1493 CanonicalLoopInfo *Floor2 = GenLoops[1];
1494 CanonicalLoopInfo *Tile1 = GenLoops[2];
1495 CanonicalLoopInfo *Tile2 = GenLoops[3];
1496
1497 BasicBlock *RefOrder[] = {
1498 Floor1->getPreheader(),
1499 Floor1->getHeader(),
1500 Floor1->getCond(),
1501 Floor1->getBody(),
1502 Floor2->getPreheader(),
1503 Floor2->getHeader(),
1504 Floor2->getCond(),
1505 Floor2->getBody(),
1506 Tile1->getPreheader(),
1507 Tile1->getHeader(),
1508 Tile1->getCond(),
1509 Tile1->getBody(),
1510 Tile2->getPreheader(),
1511 Tile2->getHeader(),
1512 Tile2->getCond(),
1513 Tile2->getBody(),
1514 BodyCode,
1515 Tile2->getLatch(),
1516 Tile2->getExit(),
1517 Tile2->getAfter(),
1518 Tile1->getLatch(),
1519 Tile1->getExit(),
1520 Tile1->getAfter(),
1521 Floor2->getLatch(),
1522 Floor2->getExit(),
1523 Floor2->getAfter(),
1524 Floor1->getLatch(),
1525 Floor1->getExit(),
1526 Floor1->getAfter(),
1527 };
1528 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1529 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1530 }
1531
1532 TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
1533 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1534 OpenMPIRBuilder OMPBuilder(*M);
1535 OMPBuilder.initialize();
1536 F->setName("func");
1537
1538 IRBuilder<> Builder(BB);
1539 Value *TripCount = F->getArg(0);
1540 Type *LCTy = TripCount->getType();
1541
1542 Value *OuterStartVal = ConstantInt::get(LCTy, 2);
1543 Value *OuterStopVal = TripCount;
1544 Value *OuterStep = ConstantInt::get(LCTy, 5);
1545 Value *InnerStartVal = ConstantInt::get(LCTy, 13);
1546 Value *InnerStopVal = TripCount;
1547 Value *InnerStep = ConstantInt::get(LCTy, 3);
1548
1549 // Fix an insertion point for ComputeIP.
1550 BasicBlock *LoopNextEnter =
1551 BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1552 Builder.GetInsertBlock()->getNextNode());
1553 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1554 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1555
1556 InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()};
1557 OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL});
1558
1559 BasicBlock *BodyCode = nullptr;
1560 CanonicalLoopInfo *InnerLoop = nullptr;
1561 CallInst *Call = nullptr;
1562 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1563 llvm::Value *OuterLC) {
1564 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1565 llvm::Value *InnerLC) {
1566 Builder.restoreIP(InnerCodeGenIP);
1567 BodyCode = Builder.GetInsertBlock();
1568
1569 // Add something that consumes the induction variable to the body.
1570 Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1571 };
1572 InnerLoop = OMPBuilder.createCanonicalLoop(
1573 OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal,
1574 InnerStep, false, false, ComputeIP, "inner");
1575 };
1576 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1577 Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false,
1578 false, ComputeIP, "outer");
1579
1580 // Finalize the function
1581 Builder.restoreIP(OuterLoop->getAfterIP());
1582 Builder.CreateRetVoid();
1583
1584 // Tile the loop nest.
1585 Constant *TileSize0 = ConstantInt::get(LCTy, APInt(32, 11));
1586 Constant *TileSize1 = ConstantInt::get(LCTy, APInt(32, 7));
1587 std::vector<CanonicalLoopInfo *> GenLoops =
1588 OMPBuilder.tileLoops(DL, {OuterLoop, InnerLoop}, {TileSize0, TileSize1});
1589
1590 OMPBuilder.finalize();
1591 EXPECT_FALSE(verifyModule(*M, &errs()));
1592
1593 EXPECT_EQ(GenLoops.size(), 4u);
1594 CanonicalLoopInfo *Floor0 = GenLoops[0];
1595 CanonicalLoopInfo *Floor1 = GenLoops[1];
1596 CanonicalLoopInfo *Tile0 = GenLoops[2];
1597 CanonicalLoopInfo *Tile1 = GenLoops[3];
1598
1599 BasicBlock *RefOrder[] = {
1600 Floor0->getPreheader(),
1601 Floor0->getHeader(),
1602 Floor0->getCond(),
1603 Floor0->getBody(),
1604 Floor1->getPreheader(),
1605 Floor1->getHeader(),
1606 Floor1->getCond(),
1607 Floor1->getBody(),
1608 Tile0->getPreheader(),
1609 Tile0->getHeader(),
1610 Tile0->getCond(),
1611 Tile0->getBody(),
1612 Tile1->getPreheader(),
1613 Tile1->getHeader(),
1614 Tile1->getCond(),
1615 Tile1->getBody(),
1616 BodyCode,
1617 Tile1->getLatch(),
1618 Tile1->getExit(),
1619 Tile1->getAfter(),
1620 Tile0->getLatch(),
1621 Tile0->getExit(),
1622 Tile0->getAfter(),
1623 Floor1->getLatch(),
1624 Floor1->getExit(),
1625 Floor1->getAfter(),
1626 Floor0->getLatch(),
1627 Floor0->getExit(),
1628 Floor0->getAfter(),
1629 };
1630 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1631 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1632
1633 EXPECT_EQ(Call->getParent(), BodyCode);
1634
1635 auto *RangeShift0 = cast<AddOperator>(Call->getOperand(1));
1636 EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal);
1637 auto *RangeScale0 = cast<MulOperator>(RangeShift0->getOperand(0));
1638 EXPECT_EQ(RangeScale0->getOperand(1), OuterStep);
1639 auto *TileShift0 = cast<AddOperator>(RangeScale0->getOperand(0));
1640 EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody());
1641 EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar());
1642 auto *TileScale0 = cast<MulOperator>(TileShift0->getOperand(0));
1643 EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody());
1644 EXPECT_EQ(TileScale0->getOperand(0), TileSize0);
1645 EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar());
1646
1647 auto *RangeShift1 = cast<AddOperator>(Call->getOperand(2));
1648 EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode);
1649 EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal);
1650 auto *RangeScale1 = cast<MulOperator>(RangeShift1->getOperand(0));
1651 EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode);
1652 EXPECT_EQ(RangeScale1->getOperand(1), InnerStep);
1653 auto *TileShift1 = cast<AddOperator>(RangeScale1->getOperand(0));
1654 EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody());
1655 EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar());
1656 auto *TileScale1 = cast<MulOperator>(TileShift1->getOperand(0));
1657 EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody());
1658 EXPECT_EQ(TileScale1->getOperand(0), TileSize1);
1659 EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar());
1660 }
1661
1662 TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {
1663 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1664 OpenMPIRBuilder OMPBuilder(*M);
1665 OMPBuilder.initialize();
1666 IRBuilder<> Builder(BB);
1667
1668 // Create a loop, tile it, and extract its trip count. All input values are
1669 // constant and IRBuilder evaluates all-constant arithmetic inplace, such that
1670 // the floor trip count itself will be a ConstantInt. Unfortunately we cannot
1671 // do the same for the tile loop.
1672 auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1673 bool IsSigned, bool InclusiveStop,
1674 int64_t TileSize) -> uint64_t {
1675 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
1676 Type *LCTy = Type::getInt16Ty(Ctx);
1677 Value *StartVal = ConstantInt::get(LCTy, Start);
1678 Value *StopVal = ConstantInt::get(LCTy, Stop);
1679 Value *StepVal = ConstantInt::get(LCTy, Step);
1680
1681 // Generate a loop.
1682 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1683 CanonicalLoopInfo *Loop =
1684 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1685 StepVal, IsSigned, InclusiveStop);
1686 InsertPointTy AfterIP = Loop->getAfterIP();
1687
1688 // Tile the loop.
1689 Value *TileSizeVal = ConstantInt::get(LCTy, TileSize);
1690 std::vector<CanonicalLoopInfo *> GenLoops =
1691 OMPBuilder.tileLoops(Loc.DL, {Loop}, {TileSizeVal});
1692
1693 // Set the insertion pointer to after loop, where the next loop will be
1694 // emitted.
1695 Builder.restoreIP(AfterIP);
1696
1697 // Extract the trip count.
1698 CanonicalLoopInfo *FloorLoop = GenLoops[0];
1699 Value *FloorTripCount = FloorLoop->getTripCount();
1700 return cast<ConstantInt>(FloorTripCount)->getValue().getZExtValue();
1701 };
1702
1703 // Empty iteration domain.
1704 EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);
1705 EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);
1706 EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);
1707 EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);
1708 EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);
1709
1710 // Only complete tiles.
1711 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1712 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1713 EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);
1714 EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);
1715 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);
1716 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);
1717
1718 // Only a partial tile.
1719 EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);
1720 EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);
1721 EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);
1722 EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);
1723 EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);
1724
1725 // Complete and partial tiles.
1726 EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);
1727 EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);
1728 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);
1729 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);
1730 EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);
1731
1732 // Close to 16-bit integer range.
1733 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu);
1734 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1);
1735 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1);
1736 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1);
1737 EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1);
1738 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);
1739 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);
1740
1741 // Finalize the function.
1742 Builder.CreateRetVoid();
1743 OMPBuilder.finalize();
1744
1745 EXPECT_FALSE(verifyModule(*M, &errs()));
1746 }
1747
1748 TEST_F(OpenMPIRBuilderTest, ApplySimd) {
1749 OpenMPIRBuilder OMPBuilder(*M);
1750 MapVector<Value *, Value *> AlignedVars;
1751 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1752
1753 // Simd-ize the loop.
1754 OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1755 OrderKind::OMP_ORDER_unknown,
1756 /* Simdlen */ nullptr,
1757 /* Safelen */ nullptr);
1758
1759 OMPBuilder.finalize();
1760 EXPECT_FALSE(verifyModule(*M, &errs()));
1761
1762 PassBuilder PB;
1763 FunctionAnalysisManager FAM;
1764 PB.registerFunctionAnalyses(FAM);
1765 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1766
1767 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1768 EXPECT_EQ(TopLvl.size(), 1u);
1769
1770 Loop *L = TopLvl.front();
1771 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1772 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1773
1774 // Check for llvm.access.group metadata attached to the printf
1775 // function in the loop body.
1776 BasicBlock *LoopBody = CLI->getBody();
1777 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1778 return I.getMetadata("llvm.access.group") != nullptr;
1779 }));
1780 }
1781
1782 TEST_F(OpenMPIRBuilderTest, ApplySimdCustomAligned) {
1783 OpenMPIRBuilder OMPBuilder(*M);
1784 IRBuilder<> Builder(BB);
1785 const int AlignmentValue = 32;
1786 AllocaInst *Alloc1 =
1787 Builder.CreateAlloca(Builder.getInt8PtrTy(), Builder.getInt64(1));
1788 LoadInst *Load1 = Builder.CreateLoad(Alloc1->getAllocatedType(), Alloc1);
1789 MapVector<Value *, Value *> AlignedVars;
1790 AlignedVars.insert({Load1, Builder.getInt64(AlignmentValue)});
1791
1792 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1793
1794 // Simd-ize the loop.
1795 OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1796 OrderKind::OMP_ORDER_unknown,
1797 /* Simdlen */ nullptr,
1798 /* Safelen */ nullptr);
1799
1800 OMPBuilder.finalize();
1801 EXPECT_FALSE(verifyModule(*M, &errs()));
1802
1803 PassBuilder PB;
1804 FunctionAnalysisManager FAM;
1805 PB.registerFunctionAnalyses(FAM);
1806 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1807
1808 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1809 EXPECT_EQ(TopLvl.size(), 1u);
1810
1811 Loop *L = TopLvl.front();
1812 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1813 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1814
1815 // Check for llvm.access.group metadata attached to the printf
1816 // function in the loop body.
1817 BasicBlock *LoopBody = CLI->getBody();
1818 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1819 return I.getMetadata("llvm.access.group") != nullptr;
1820 }));
1821
1822 // Check if number of assumption instructions is equal to number of aligned
1823 // variables
1824 BasicBlock *LoopPreheader = CLI->getPreheader();
1825 size_t NumAssummptionCallsInPreheader = count_if(
1826 *LoopPreheader, [](Instruction &I) { return isa<AssumeInst>(I); });
1827 EXPECT_EQ(NumAssummptionCallsInPreheader, AlignedVars.size());
1828
1829 // Check if variables are correctly aligned
1830 for (Instruction &Instr : *LoopPreheader) {
1831 if (!isa<AssumeInst>(Instr))
1832 continue;
1833 AssumeInst *AssumeInstruction = cast<AssumeInst>(&Instr);
1834 if (AssumeInstruction->getNumTotalBundleOperands()) {
1835 auto Bundle = AssumeInstruction->getOperandBundleAt(0);
1836 if (Bundle.getTagName() == "align") {
1837 EXPECT_TRUE(isa<ConstantInt>(Bundle.Inputs[1]));
1838 auto ConstIntVal = dyn_cast<ConstantInt>(Bundle.Inputs[1]);
1839 EXPECT_EQ(ConstIntVal->getSExtValue(), AlignmentValue);
1840 }
1841 }
1842 }
1843 }
1844 TEST_F(OpenMPIRBuilderTest, ApplySimdlen) {
1845 OpenMPIRBuilder OMPBuilder(*M);
1846 MapVector<Value *, Value *> AlignedVars;
1847 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1848
1849 // Simd-ize the loop.
1850 OMPBuilder.applySimd(CLI, AlignedVars,
1851 /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,
1852 ConstantInt::get(Type::getInt32Ty(Ctx), 3),
1853 /* Safelen */ nullptr);
1854
1855 OMPBuilder.finalize();
1856 EXPECT_FALSE(verifyModule(*M, &errs()));
1857
1858 PassBuilder PB;
1859 FunctionAnalysisManager FAM;
1860 PB.registerFunctionAnalyses(FAM);
1861 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1862
1863 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1864 EXPECT_EQ(TopLvl.size(), 1u);
1865
1866 Loop *L = TopLvl.front();
1867 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1868 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1869 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1870
1871 // Check for llvm.access.group metadata attached to the printf
1872 // function in the loop body.
1873 BasicBlock *LoopBody = CLI->getBody();
1874 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1875 return I.getMetadata("llvm.access.group") != nullptr;
1876 }));
1877 }
1878
1879 TEST_F(OpenMPIRBuilderTest, ApplySafelenOrderConcurrent) {
1880 OpenMPIRBuilder OMPBuilder(*M);
1881 MapVector<Value *, Value *> AlignedVars;
1882
1883 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1884
1885 // Simd-ize the loop.
1886 OMPBuilder.applySimd(
1887 CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_concurrent,
1888 /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1889
1890 OMPBuilder.finalize();
1891 EXPECT_FALSE(verifyModule(*M, &errs()));
1892
1893 PassBuilder PB;
1894 FunctionAnalysisManager FAM;
1895 PB.registerFunctionAnalyses(FAM);
1896 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1897
1898 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1899 EXPECT_EQ(TopLvl.size(), 1u);
1900
1901 Loop *L = TopLvl.front();
1902 // Parallel metadata shoudl be attached because of presence of
1903 // the order(concurrent) OpenMP clause
1904 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1905 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1906 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1907
1908 // Check for llvm.access.group metadata attached to the printf
1909 // function in the loop body.
1910 BasicBlock *LoopBody = CLI->getBody();
1911 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1912 return I.getMetadata("llvm.access.group") != nullptr;
1913 }));
1914 }
1915
1916 TEST_F(OpenMPIRBuilderTest, ApplySafelen) {
1917 OpenMPIRBuilder OMPBuilder(*M);
1918 MapVector<Value *, Value *> AlignedVars;
1919
1920 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1921
1922 OMPBuilder.applySimd(
1923 CLI, AlignedVars, /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown,
1924 /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1925
1926 OMPBuilder.finalize();
1927 EXPECT_FALSE(verifyModule(*M, &errs()));
1928
1929 PassBuilder PB;
1930 FunctionAnalysisManager FAM;
1931 PB.registerFunctionAnalyses(FAM);
1932 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1933
1934 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1935 EXPECT_EQ(TopLvl.size(), 1u);
1936
1937 Loop *L = TopLvl.front();
1938 EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1939 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1940 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1941
1942 // Check for llvm.access.group metadata attached to the printf
1943 // function in the loop body.
1944 BasicBlock *LoopBody = CLI->getBody();
1945 EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {
1946 return I.getMetadata("llvm.access.group") != nullptr;
1947 }));
1948 }
1949
1950 TEST_F(OpenMPIRBuilderTest, ApplySimdlenSafelen) {
1951 OpenMPIRBuilder OMPBuilder(*M);
1952 MapVector<Value *, Value *> AlignedVars;
1953
1954 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1955
1956 OMPBuilder.applySimd(CLI, AlignedVars, /* IfCond */ nullptr,
1957 OrderKind::OMP_ORDER_unknown,
1958 ConstantInt::get(Type::getInt32Ty(Ctx), 2),
1959 ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1960
1961 OMPBuilder.finalize();
1962 EXPECT_FALSE(verifyModule(*M, &errs()));
1963
1964 PassBuilder PB;
1965 FunctionAnalysisManager FAM;
1966 PB.registerFunctionAnalyses(FAM);
1967 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1968
1969 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1970 EXPECT_EQ(TopLvl.size(), 1u);
1971
1972 Loop *L = TopLvl.front();
1973 EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1974 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1975 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 2);
1976
1977 // Check for llvm.access.group metadata attached to the printf
1978 // function in the loop body.
1979 BasicBlock *LoopBody = CLI->getBody();
1980 EXPECT_FALSE(any_of(*LoopBody, [](Instruction &I) {
1981 return I.getMetadata("llvm.access.group") != nullptr;
1982 }));
1983 }
1984
1985 TEST_F(OpenMPIRBuilderTest, ApplySimdLoopIf) {
1986 OpenMPIRBuilder OMPBuilder(*M);
1987 IRBuilder<> Builder(BB);
1988 MapVector<Value *, Value *> AlignedVars;
1989 AllocaInst *Alloc1 = Builder.CreateAlloca(Builder.getInt32Ty());
1990 AllocaInst *Alloc2 = Builder.CreateAlloca(Builder.getInt32Ty());
1991
1992 // Generation of if condition
1993 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), Alloc1);
1994 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 1U), Alloc2);
1995 LoadInst *Load1 = Builder.CreateLoad(Alloc1->getAllocatedType(), Alloc1);
1996 LoadInst *Load2 = Builder.CreateLoad(Alloc2->getAllocatedType(), Alloc2);
1997
1998 Value *IfCmp = Builder.CreateICmpNE(Load1, Load2);
1999
2000 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2001
2002 // Simd-ize the loop with if condition
2003 OMPBuilder.applySimd(CLI, AlignedVars, IfCmp, OrderKind::OMP_ORDER_unknown,
2004 ConstantInt::get(Type::getInt32Ty(Ctx), 3),
2005 /* Safelen */ nullptr);
2006
2007 OMPBuilder.finalize();
2008 EXPECT_FALSE(verifyModule(*M, &errs()));
2009
2010 PassBuilder PB;
2011 FunctionAnalysisManager FAM;
2012 PB.registerFunctionAnalyses(FAM);
2013 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2014
2015 // Check if there are two loops (one with enabled vectorization)
2016 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2017 EXPECT_EQ(TopLvl.size(), 2u);
2018
2019 Loop *L = TopLvl[0];
2020 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2021 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2022 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
2023
2024 // The second loop should have disabled vectorization
2025 L = TopLvl[1];
2026 EXPECT_FALSE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
2027 EXPECT_FALSE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
2028 // Check for llvm.access.group metadata attached to the printf
2029 // function in the loop body.
2030 BasicBlock *LoopBody = CLI->getBody();
2031 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
2032 return I.getMetadata("llvm.access.group") != nullptr;
2033 }));
2034 }
2035
2036 TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) {
2037 OpenMPIRBuilder OMPBuilder(*M);
2038
2039 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2040
2041 // Unroll the loop.
2042 OMPBuilder.unrollLoopFull(DL, CLI);
2043
2044 OMPBuilder.finalize();
2045 EXPECT_FALSE(verifyModule(*M, &errs()));
2046
2047 PassBuilder PB;
2048 FunctionAnalysisManager FAM;
2049 PB.registerFunctionAnalyses(FAM);
2050 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2051
2052 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2053 EXPECT_EQ(TopLvl.size(), 1u);
2054
2055 Loop *L = TopLvl.front();
2056 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
2057 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full"));
2058 }
2059
2060 TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) {
2061 OpenMPIRBuilder OMPBuilder(*M);
2062 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2063
2064 // Unroll the loop.
2065 CanonicalLoopInfo *UnrolledLoop = nullptr;
2066 OMPBuilder.unrollLoopPartial(DL, CLI, 5, &UnrolledLoop);
2067 ASSERT_NE(UnrolledLoop, nullptr);
2068
2069 OMPBuilder.finalize();
2070 EXPECT_FALSE(verifyModule(*M, &errs()));
2071 UnrolledLoop->assertOK();
2072
2073 PassBuilder PB;
2074 FunctionAnalysisManager FAM;
2075 PB.registerFunctionAnalyses(FAM);
2076 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2077
2078 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2079 EXPECT_EQ(TopLvl.size(), 1u);
2080 Loop *Outer = TopLvl.front();
2081 EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader());
2082 EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch());
2083 EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond());
2084 EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit());
2085
2086 EXPECT_EQ(Outer->getSubLoops().size(), 1u);
2087 Loop *Inner = Outer->getSubLoops().front();
2088
2089 EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable"));
2090 EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5);
2091 }
2092
2093 TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) {
2094 OpenMPIRBuilder OMPBuilder(*M);
2095
2096 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
2097
2098 // Unroll the loop.
2099 OMPBuilder.unrollLoopHeuristic(DL, CLI);
2100
2101 OMPBuilder.finalize();
2102 EXPECT_FALSE(verifyModule(*M, &errs()));
2103
2104 PassBuilder PB;
2105 FunctionAnalysisManager FAM;
2106 PB.registerFunctionAnalyses(FAM);
2107 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
2108
2109 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
2110 EXPECT_EQ(TopLvl.size(), 1u);
2111
2112 Loop *L = TopLvl.front();
2113 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
2114 }
2115
2116 TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
2117 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2118 OpenMPIRBuilder OMPBuilder(*M);
2119 OMPBuilder.initialize();
2120 IRBuilder<> Builder(BB);
2121 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2122
2123 Type *LCTy = Type::getInt32Ty(Ctx);
2124 Value *StartVal = ConstantInt::get(LCTy, 10);
2125 Value *StopVal = ConstantInt::get(LCTy, 52);
2126 Value *StepVal = ConstantInt::get(LCTy, 2);
2127 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2128
2129 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2130 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2131 /*IsSigned=*/false, /*InclusiveStop=*/false);
2132 BasicBlock *Preheader = CLI->getPreheader();
2133 BasicBlock *Body = CLI->getBody();
2134 Value *IV = CLI->getIndVar();
2135 BasicBlock *ExitBlock = CLI->getExit();
2136
2137 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2138 InsertPointTy AllocaIP = Builder.saveIP();
2139
2140 OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2141 OMP_SCHEDULE_Static);
2142
2143 BasicBlock *Cond = Body->getSinglePredecessor();
2144 Instruction *Cmp = &*Cond->begin();
2145 Value *TripCount = Cmp->getOperand(1);
2146
2147 auto AllocaIter = BB->begin();
2148 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2149 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
2150 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2151 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2152 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
2153 EXPECT_NE(PLastIter, nullptr);
2154 EXPECT_NE(PLowerBound, nullptr);
2155 EXPECT_NE(PUpperBound, nullptr);
2156 EXPECT_NE(PStride, nullptr);
2157
2158 auto PreheaderIter = Preheader->begin();
2159 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7);
2160 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2161 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2162 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2163 ASSERT_NE(LowerBoundStore, nullptr);
2164 ASSERT_NE(UpperBoundStore, nullptr);
2165 ASSERT_NE(StrideStore, nullptr);
2166
2167 auto *OrigLowerBound =
2168 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
2169 auto *OrigUpperBound =
2170 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
2171 auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand());
2172 ASSERT_NE(OrigLowerBound, nullptr);
2173 ASSERT_NE(OrigUpperBound, nullptr);
2174 ASSERT_NE(OrigStride, nullptr);
2175 EXPECT_EQ(OrigLowerBound->getValue(), 0);
2176 EXPECT_EQ(OrigUpperBound->getValue(), 20);
2177 EXPECT_EQ(OrigStride->getValue(), 1);
2178
2179 // Check that the loop IV is updated to account for the lower bound returned
2180 // by the OpenMP runtime call.
2181 BinaryOperator *Add = dyn_cast<BinaryOperator>(&Body->front());
2182 EXPECT_EQ(Add->getOperand(0), IV);
2183 auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1));
2184 ASSERT_NE(LoadedLowerBound, nullptr);
2185 EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);
2186
2187 // Check that the trip count is updated to account for the lower and upper
2188 // bounds return by the OpenMP runtime call.
2189 auto *AddOne = dyn_cast<Instruction>(TripCount);
2190 ASSERT_NE(AddOne, nullptr);
2191 ASSERT_TRUE(AddOne->isBinaryOp());
2192 auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1));
2193 ASSERT_NE(One, nullptr);
2194 EXPECT_EQ(One->getValue(), 1);
2195 auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0));
2196 ASSERT_NE(Difference, nullptr);
2197 ASSERT_TRUE(Difference->isBinaryOp());
2198 EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);
2199 auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0));
2200 ASSERT_NE(LoadedUpperBound, nullptr);
2201 EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);
2202
2203 // The original loop iterator should only be used in the condition, in the
2204 // increment and in the statement that adds the lower bound to it.
2205 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2206
2207 // The exit block should contain the "fini" call and the barrier call,
2208 // plus the call to obtain the thread ID.
2209 size_t NumCallsInExitBlock =
2210 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2211 EXPECT_EQ(NumCallsInExitBlock, 3u);
2212 }
2213
2214 TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {
2215 unsigned IVBits = GetParam();
2216
2217 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2218 OpenMPIRBuilder OMPBuilder(*M);
2219
2220 BasicBlock *Body;
2221 CallInst *Call;
2222 CanonicalLoopInfo *CLI =
2223 buildSingleLoopFunction(DL, OMPBuilder, IVBits, &Call, &Body);
2224
2225 Instruction *OrigIndVar = CLI->getIndVar();
2226 EXPECT_EQ(Call->getOperand(1), OrigIndVar);
2227
2228 Type *LCTy = Type::getInt32Ty(Ctx);
2229 Value *ChunkSize = ConstantInt::get(LCTy, 5);
2230 InsertPointTy AllocaIP{&F->getEntryBlock(),
2231 F->getEntryBlock().getFirstInsertionPt()};
2232 OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2233 OMP_SCHEDULE_Static, ChunkSize);
2234
2235 OMPBuilder.finalize();
2236 EXPECT_FALSE(verifyModule(*M, &errs()));
2237
2238 BasicBlock *Entry = &F->getEntryBlock();
2239 BasicBlock *Preheader = Entry->getSingleSuccessor();
2240
2241 BasicBlock *DispatchPreheader = Preheader->getSingleSuccessor();
2242 BasicBlock *DispatchHeader = DispatchPreheader->getSingleSuccessor();
2243 BasicBlock *DispatchCond = DispatchHeader->getSingleSuccessor();
2244 BasicBlock *DispatchBody = succ_begin(DispatchCond)[0];
2245 BasicBlock *DispatchExit = succ_begin(DispatchCond)[1];
2246 BasicBlock *DispatchAfter = DispatchExit->getSingleSuccessor();
2247 BasicBlock *Return = DispatchAfter->getSingleSuccessor();
2248
2249 BasicBlock *ChunkPreheader = DispatchBody->getSingleSuccessor();
2250 BasicBlock *ChunkHeader = ChunkPreheader->getSingleSuccessor();
2251 BasicBlock *ChunkCond = ChunkHeader->getSingleSuccessor();
2252 BasicBlock *ChunkBody = succ_begin(ChunkCond)[0];
2253 BasicBlock *ChunkExit = succ_begin(ChunkCond)[1];
2254 BasicBlock *ChunkInc = ChunkBody->getSingleSuccessor();
2255 BasicBlock *ChunkAfter = ChunkExit->getSingleSuccessor();
2256
2257 BasicBlock *DispatchInc = ChunkAfter;
2258
2259 EXPECT_EQ(ChunkBody, Body);
2260 EXPECT_EQ(ChunkInc->getSingleSuccessor(), ChunkHeader);
2261 EXPECT_EQ(DispatchInc->getSingleSuccessor(), DispatchHeader);
2262
2263 EXPECT_TRUE(isa<ReturnInst>(Return->front()));
2264
2265 Value *NewIV = Call->getOperand(1);
2266 EXPECT_EQ(NewIV->getType()->getScalarSizeInBits(), IVBits);
2267
2268 CallInst *InitCall = findSingleCall(
2269 F,
2270 (IVBits > 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u
2271 : omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u,
2272 OMPBuilder);
2273 EXPECT_EQ(InitCall->getParent(), Preheader);
2274 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(2))->getSExtValue(), 33);
2275 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(7))->getSExtValue(), 1);
2276 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(8))->getSExtValue(), 5);
2277
2278 CallInst *FiniCall = findSingleCall(
2279 F, omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini, OMPBuilder);
2280 EXPECT_EQ(FiniCall->getParent(), DispatchExit);
2281
2282 CallInst *BarrierCall = findSingleCall(
2283 F, omp::RuntimeFunction::OMPRTL___kmpc_barrier, OMPBuilder);
2284 EXPECT_EQ(BarrierCall->getParent(), DispatchExit);
2285 }
2286
2287 INSTANTIATE_TEST_SUITE_P(IVBits, OpenMPIRBuilderTestWithIVBits,
2288 ::testing::Values(8, 16, 32, 64));
2289
2290 TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
2291 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2292 OpenMPIRBuilder OMPBuilder(*M);
2293 OMPBuilder.initialize();
2294 IRBuilder<> Builder(BB);
2295 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2296
2297 omp::OMPScheduleType SchedType = GetParam();
2298 uint32_t ChunkSize = 1;
2299 switch (SchedType & ~OMPScheduleType::ModifierMask) {
2300 case omp::OMPScheduleType::BaseDynamicChunked:
2301 case omp::OMPScheduleType::BaseGuidedChunked:
2302 ChunkSize = 7;
2303 break;
2304 case omp::OMPScheduleType::BaseAuto:
2305 case omp::OMPScheduleType::BaseRuntime:
2306 ChunkSize = 1;
2307 break;
2308 default:
2309 assert(0 && "unknown type for this test");
2310 break;
2311 }
2312
2313 Type *LCTy = Type::getInt32Ty(Ctx);
2314 Value *StartVal = ConstantInt::get(LCTy, 10);
2315 Value *StopVal = ConstantInt::get(LCTy, 52);
2316 Value *StepVal = ConstantInt::get(LCTy, 2);
2317 Value *ChunkVal =
2318 (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize);
2319 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2320
2321 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2322 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2323 /*IsSigned=*/false, /*InclusiveStop=*/false);
2324
2325 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2326 InsertPointTy AllocaIP = Builder.saveIP();
2327
2328 // Collect all the info from CLI, as it isn't usable after the call to
2329 // createDynamicWorkshareLoop.
2330 InsertPointTy AfterIP = CLI->getAfterIP();
2331 BasicBlock *Preheader = CLI->getPreheader();
2332 BasicBlock *ExitBlock = CLI->getExit();
2333 BasicBlock *LatchBlock = CLI->getLatch();
2334 Value *IV = CLI->getIndVar();
2335
2336 InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2337 DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),
2338 ChunkVal, /*Simd=*/false,
2339 (SchedType & omp::OMPScheduleType::ModifierMonotonic) ==
2340 omp::OMPScheduleType::ModifierMonotonic,
2341 (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==
2342 omp::OMPScheduleType::ModifierNonmonotonic,
2343 /*Ordered=*/false);
2344
2345 // The returned value should be the "after" point.
2346 ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());
2347 ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());
2348
2349 auto AllocaIter = BB->begin();
2350 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2351 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
2352 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2353 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2354 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
2355 EXPECT_NE(PLastIter, nullptr);
2356 EXPECT_NE(PLowerBound, nullptr);
2357 EXPECT_NE(PUpperBound, nullptr);
2358 EXPECT_NE(PStride, nullptr);
2359
2360 auto PreheaderIter = Preheader->begin();
2361 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6);
2362 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2363 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2364 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2365 ASSERT_NE(LowerBoundStore, nullptr);
2366 ASSERT_NE(UpperBoundStore, nullptr);
2367 ASSERT_NE(StrideStore, nullptr);
2368
2369 CallInst *ThreadIdCall = dyn_cast<CallInst>(&*(PreheaderIter++));
2370 ASSERT_NE(ThreadIdCall, nullptr);
2371 EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(),
2372 "__kmpc_global_thread_num");
2373
2374 CallInst *InitCall = dyn_cast<CallInst>(&*PreheaderIter);
2375
2376 ASSERT_NE(InitCall, nullptr);
2377 EXPECT_EQ(InitCall->getCalledFunction()->getName(),
2378 "__kmpc_dispatch_init_4u");
2379 EXPECT_EQ(InitCall->arg_size(), 7U);
2380 EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize));
2381 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2382 if ((SchedType & OMPScheduleType::MonotonicityMask) ==
2383 OMPScheduleType::None) {
2384 // Implementation is allowed to add default nonmonotonicity flag
2385 EXPECT_EQ(
2386 static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()) |
2387 OMPScheduleType::ModifierNonmonotonic,
2388 SchedType | OMPScheduleType::ModifierNonmonotonic);
2389 } else {
2390 EXPECT_EQ(static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()),
2391 SchedType);
2392 }
2393
2394 ConstantInt *OrigLowerBound =
2395 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
2396 ConstantInt *OrigUpperBound =
2397 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
2398 ConstantInt *OrigStride =
2399 dyn_cast<ConstantInt>(StrideStore->getValueOperand());
2400 ASSERT_NE(OrigLowerBound, nullptr);
2401 ASSERT_NE(OrigUpperBound, nullptr);
2402 ASSERT_NE(OrigStride, nullptr);
2403 EXPECT_EQ(OrigLowerBound->getValue(), 1);
2404 EXPECT_EQ(OrigUpperBound->getValue(), 21);
2405 EXPECT_EQ(OrigStride->getValue(), 1);
2406
2407 CallInst *FiniCall = dyn_cast<CallInst>(
2408 &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2409 EXPECT_EQ(FiniCall, nullptr);
2410
2411 // The original loop iterator should only be used in the condition, in the
2412 // increment and in the statement that adds the lower bound to it.
2413 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2414
2415 // The exit block should contain the barrier call, plus the call to obtain
2416 // the thread ID.
2417 size_t NumCallsInExitBlock =
2418 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2419 EXPECT_EQ(NumCallsInExitBlock, 2u);
2420
2421 // Add a termination to our block and check that it is internally consistent.
2422 Builder.restoreIP(EndIP);
2423 Builder.CreateRetVoid();
2424 OMPBuilder.finalize();
2425 EXPECT_FALSE(verifyModule(*M, &errs()));
2426 }
2427
2428 INSTANTIATE_TEST_SUITE_P(
2429 OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams,
2430 ::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked,
2431 omp::OMPScheduleType::UnorderedGuidedChunked,
2432 omp::OMPScheduleType::UnorderedAuto,
2433 omp::OMPScheduleType::UnorderedRuntime,
2434 omp::OMPScheduleType::UnorderedDynamicChunked |
2435 omp::OMPScheduleType::ModifierMonotonic,
2436 omp::OMPScheduleType::UnorderedDynamicChunked |
2437 omp::OMPScheduleType::ModifierNonmonotonic,
2438 omp::OMPScheduleType::UnorderedGuidedChunked |
2439 omp::OMPScheduleType::ModifierMonotonic,
2440 omp::OMPScheduleType::UnorderedGuidedChunked |
2441 omp::OMPScheduleType::ModifierNonmonotonic,
2442 omp::OMPScheduleType::UnorderedAuto |
2443 omp::OMPScheduleType::ModifierMonotonic,
2444 omp::OMPScheduleType::UnorderedRuntime |
2445 omp::OMPScheduleType::ModifierMonotonic));
2446
2447 TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
2448 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2449 OpenMPIRBuilder OMPBuilder(*M);
2450 OMPBuilder.initialize();
2451 IRBuilder<> Builder(BB);
2452 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2453
2454 uint32_t ChunkSize = 1;
2455 Type *LCTy = Type::getInt32Ty(Ctx);
2456 Value *StartVal = ConstantInt::get(LCTy, 10);
2457 Value *StopVal = ConstantInt::get(LCTy, 52);
2458 Value *StepVal = ConstantInt::get(LCTy, 2);
2459 Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize);
2460 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2461
2462 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2463 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2464 /*IsSigned=*/false, /*InclusiveStop=*/false);
2465
2466 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2467 InsertPointTy AllocaIP = Builder.saveIP();
2468
2469 // Collect all the info from CLI, as it isn't usable after the call to
2470 // createDynamicWorkshareLoop.
2471 BasicBlock *Preheader = CLI->getPreheader();
2472 BasicBlock *ExitBlock = CLI->getExit();
2473 BasicBlock *LatchBlock = CLI->getLatch();
2474 Value *IV = CLI->getIndVar();
2475
2476 InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2477 DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal,
2478 /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
2479 /*HasNonmonotonicModifier=*/false,
2480 /*HasOrderedClause=*/true);
2481
2482 // Add a termination to our block and check that it is internally consistent.
2483 Builder.restoreIP(EndIP);
2484 Builder.CreateRetVoid();
2485 OMPBuilder.finalize();
2486 EXPECT_FALSE(verifyModule(*M, &errs()));
2487
2488 CallInst *InitCall = nullptr;
2489 for (Instruction &EI : *Preheader) {
2490 Instruction *Cur = &EI;
2491 if (isa<CallInst>(Cur)) {
2492 InitCall = cast<CallInst>(Cur);
2493 if (InitCall->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")
2494 break;
2495 InitCall = nullptr;
2496 }
2497 }
2498 EXPECT_NE(InitCall, nullptr);
2499 EXPECT_EQ(InitCall->arg_size(), 7U);
2500 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2501 EXPECT_EQ(SchedVal->getValue(),
2502 static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked));
2503
2504 CallInst *FiniCall = dyn_cast<CallInst>(
2505 &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2506 ASSERT_NE(FiniCall, nullptr);
2507 EXPECT_EQ(FiniCall->getCalledFunction()->getName(),
2508 "__kmpc_dispatch_fini_4u");
2509 EXPECT_EQ(FiniCall->arg_size(), 2U);
2510 EXPECT_EQ(InitCall->getArgOperand(0), FiniCall->getArgOperand(0));
2511 EXPECT_EQ(InitCall->getArgOperand(1), FiniCall->getArgOperand(1));
2512
2513 // The original loop iterator should only be used in the condition, in the
2514 // increment and in the statement that adds the lower bound to it.
2515 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2516
2517 // The exit block should contain the barrier call, plus the call to obtain
2518 // the thread ID.
2519 size_t NumCallsInExitBlock =
2520 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2521 EXPECT_EQ(NumCallsInExitBlock, 2u);
2522 }
2523
2524 TEST_F(OpenMPIRBuilderTest, MasterDirective) {
2525 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2526 OpenMPIRBuilder OMPBuilder(*M);
2527 OMPBuilder.initialize();
2528 F->setName("func");
2529 IRBuilder<> Builder(BB);
2530
2531 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2532
2533 AllocaInst *PrivAI = nullptr;
2534
2535 BasicBlock *EntryBB = nullptr;
2536 BasicBlock *ThenBB = nullptr;
2537
2538 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2539 if (AllocaIP.isSet())
2540 Builder.restoreIP(AllocaIP);
2541 else
2542 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2543 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2544 Builder.CreateStore(F->arg_begin(), PrivAI);
2545
2546 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2547 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2548 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2549
2550 Builder.restoreIP(CodeGenIP);
2551
2552 // collect some info for checks later
2553 ThenBB = Builder.GetInsertBlock();
2554 EntryBB = ThenBB->getUniquePredecessor();
2555
2556 // simple instructions for body
2557 Value *PrivLoad =
2558 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2559 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2560 };
2561
2562 auto FiniCB = [&](InsertPointTy IP) {
2563 BasicBlock *IPBB = IP.getBlock();
2564 EXPECT_NE(IPBB->end(), IP.getPoint());
2565 };
2566
2567 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
2568 Value *EntryBBTI = EntryBB->getTerminator();
2569 EXPECT_NE(EntryBBTI, nullptr);
2570 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2571 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2572 EXPECT_TRUE(EntryBr->isConditional());
2573 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2574 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2575 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2576
2577 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2578 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2579
2580 CallInst *MasterEntryCI = cast<CallInst>(CondInst->getOperand(0));
2581 EXPECT_EQ(MasterEntryCI->arg_size(), 2U);
2582 EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master");
2583 EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0)));
2584
2585 CallInst *MasterEndCI = nullptr;
2586 for (auto &FI : *ThenBB) {
2587 Instruction *cur = &FI;
2588 if (isa<CallInst>(cur)) {
2589 MasterEndCI = cast<CallInst>(cur);
2590 if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master")
2591 break;
2592 MasterEndCI = nullptr;
2593 }
2594 }
2595 EXPECT_NE(MasterEndCI, nullptr);
2596 EXPECT_EQ(MasterEndCI->arg_size(), 2U);
2597 EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0)));
2598 EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1));
2599 }
2600
2601 TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
2602 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2603 OpenMPIRBuilder OMPBuilder(*M);
2604 OMPBuilder.initialize();
2605 F->setName("func");
2606 IRBuilder<> Builder(BB);
2607
2608 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2609
2610 AllocaInst *PrivAI = nullptr;
2611
2612 BasicBlock *EntryBB = nullptr;
2613 BasicBlock *ThenBB = nullptr;
2614
2615 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2616 if (AllocaIP.isSet())
2617 Builder.restoreIP(AllocaIP);
2618 else
2619 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2620 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2621 Builder.CreateStore(F->arg_begin(), PrivAI);
2622
2623 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2624 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2625 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2626
2627 Builder.restoreIP(CodeGenIP);
2628
2629 // collect some info for checks later
2630 ThenBB = Builder.GetInsertBlock();
2631 EntryBB = ThenBB->getUniquePredecessor();
2632
2633 // simple instructions for body
2634 Value *PrivLoad =
2635 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2636 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2637 };
2638
2639 auto FiniCB = [&](InsertPointTy IP) {
2640 BasicBlock *IPBB = IP.getBlock();
2641 EXPECT_NE(IPBB->end(), IP.getPoint());
2642 };
2643
2644 Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
2645 Builder.restoreIP(
2646 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter));
2647 Value *EntryBBTI = EntryBB->getTerminator();
2648 EXPECT_NE(EntryBBTI, nullptr);
2649 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2650 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2651 EXPECT_TRUE(EntryBr->isConditional());
2652 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2653 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2654 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2655
2656 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2657 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2658
2659 CallInst *MaskedEntryCI = cast<CallInst>(CondInst->getOperand(0));
2660 EXPECT_EQ(MaskedEntryCI->arg_size(), 3U);
2661 EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked");
2662 EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0)));
2663
2664 CallInst *MaskedEndCI = nullptr;
2665 for (auto &FI : *ThenBB) {
2666 Instruction *cur = &FI;
2667 if (isa<CallInst>(cur)) {
2668 MaskedEndCI = cast<CallInst>(cur);
2669 if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked")
2670 break;
2671 MaskedEndCI = nullptr;
2672 }
2673 }
2674 EXPECT_NE(MaskedEndCI, nullptr);
2675 EXPECT_EQ(MaskedEndCI->arg_size(), 2U);
2676 EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0)));
2677 EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1));
2678 }
2679
2680 TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
2681 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2682 OpenMPIRBuilder OMPBuilder(*M);
2683 OMPBuilder.initialize();
2684 F->setName("func");
2685 IRBuilder<> Builder(BB);
2686
2687 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2688
2689 AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2690
2691 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2692 // actual start for bodyCB
2693 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2694 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2695 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2696
2697 // body begin
2698 Builder.restoreIP(CodeGenIP);
2699 Builder.CreateStore(F->arg_begin(), PrivAI);
2700 Value *PrivLoad =
2701 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2702 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2703 };
2704
2705 auto FiniCB = [&](InsertPointTy IP) {
2706 BasicBlock *IPBB = IP.getBlock();
2707 EXPECT_NE(IPBB->end(), IP.getPoint());
2708 };
2709 BasicBlock *EntryBB = Builder.GetInsertBlock();
2710
2711 Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
2712 "testCRT", nullptr));
2713
2714 CallInst *CriticalEntryCI = nullptr;
2715 for (auto &EI : *EntryBB) {
2716 Instruction *cur = &EI;
2717 if (isa<CallInst>(cur)) {
2718 CriticalEntryCI = cast<CallInst>(cur);
2719 if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical")
2720 break;
2721 CriticalEntryCI = nullptr;
2722 }
2723 }
2724 EXPECT_NE(CriticalEntryCI, nullptr);
2725 EXPECT_EQ(CriticalEntryCI->arg_size(), 3U);
2726 EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical");
2727 EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0)));
2728
2729 CallInst *CriticalEndCI = nullptr;
2730 for (auto &FI : *EntryBB) {
2731 Instruction *cur = &FI;
2732 if (isa<CallInst>(cur)) {
2733 CriticalEndCI = cast<CallInst>(cur);
2734 if (CriticalEndCI->getCalledFunction()->getName() ==
2735 "__kmpc_end_critical")
2736 break;
2737 CriticalEndCI = nullptr;
2738 }
2739 }
2740 EXPECT_NE(CriticalEndCI, nullptr);
2741 EXPECT_EQ(CriticalEndCI->arg_size(), 3U);
2742 EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0)));
2743 EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1));
2744 PointerType *CriticalNamePtrTy =
2745 PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx), 8));
2746 EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2));
2747 GlobalVariable *GV =
2748 dyn_cast<GlobalVariable>(CriticalEndCI->getArgOperand(2));
2749 ASSERT_NE(GV, nullptr);
2750 EXPECT_EQ(GV->getType(), CriticalNamePtrTy);
2751 const DataLayout &DL = M->getDataLayout();
2752 const llvm::Align TypeAlign = DL.getABITypeAlign(CriticalNamePtrTy);
2753 const llvm::Align PtrAlign = DL.getPointerABIAlignment(GV->getAddressSpace());
2754 if (const llvm::MaybeAlign Alignment = GV->getAlign())
2755 EXPECT_EQ(*Alignment, std::max(TypeAlign, PtrAlign));
2756 }
2757
2758 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) {
2759 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2760 OpenMPIRBuilder OMPBuilder(*M);
2761 OMPBuilder.initialize();
2762 F->setName("func");
2763 IRBuilder<> Builder(BB);
2764 LLVMContext &Ctx = M->getContext();
2765
2766 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2767
2768 InsertPointTy AllocaIP(&F->getEntryBlock(),
2769 F->getEntryBlock().getFirstInsertionPt());
2770
2771 unsigned NumLoops = 2;
2772 SmallVector<Value *, 2> StoreValues;
2773 Type *LCTy = Type::getInt64Ty(Ctx);
2774 StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2775 StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2776
2777 // Test for "#omp ordered depend(source)"
2778 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2779 StoreValues, ".cnt.addr",
2780 /*IsDependSource=*/true));
2781
2782 Builder.CreateRetVoid();
2783 OMPBuilder.finalize();
2784 EXPECT_FALSE(verifyModule(*M, &errs()));
2785
2786 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2787 ASSERT_NE(AllocInst, nullptr);
2788 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2789 EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2790 EXPECT_TRUE(
2791 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2792
2793 Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2794 for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2795 GetElementPtrInst *DependAddrGEPIter =
2796 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2797 ASSERT_NE(DependAddrGEPIter, nullptr);
2798 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2799 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2800 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2801 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2802 ASSERT_NE(FirstIdx, nullptr);
2803 ASSERT_NE(SecondIdx, nullptr);
2804 EXPECT_EQ(FirstIdx->getValue(), 0);
2805 EXPECT_EQ(SecondIdx->getValue(), Iter);
2806 StoreInst *StoreValue =
2807 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2808 ASSERT_NE(StoreValue, nullptr);
2809 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2810 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2811 EXPECT_EQ(StoreValue->getAlign(), Align(8));
2812 IterInst = dyn_cast<Instruction>(StoreValue);
2813 }
2814
2815 GetElementPtrInst *DependBaseAddrGEP =
2816 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2817 ASSERT_NE(DependBaseAddrGEP, nullptr);
2818 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2819 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2820 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2821 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2822 ASSERT_NE(FirstIdx, nullptr);
2823 ASSERT_NE(SecondIdx, nullptr);
2824 EXPECT_EQ(FirstIdx->getValue(), 0);
2825 EXPECT_EQ(SecondIdx->getValue(), 0);
2826
2827 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2828 ASSERT_NE(GTID, nullptr);
2829 EXPECT_EQ(GTID->arg_size(), 1U);
2830 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2831 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2832 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2833
2834 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2835 ASSERT_NE(Depend, nullptr);
2836 EXPECT_EQ(Depend->arg_size(), 3U);
2837 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post");
2838 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2839 EXPECT_EQ(Depend->getArgOperand(1), GTID);
2840 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2841 }
2842
2843 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) {
2844 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2845 OpenMPIRBuilder OMPBuilder(*M);
2846 OMPBuilder.initialize();
2847 F->setName("func");
2848 IRBuilder<> Builder(BB);
2849 LLVMContext &Ctx = M->getContext();
2850
2851 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2852
2853 InsertPointTy AllocaIP(&F->getEntryBlock(),
2854 F->getEntryBlock().getFirstInsertionPt());
2855
2856 unsigned NumLoops = 2;
2857 SmallVector<Value *, 2> StoreValues;
2858 Type *LCTy = Type::getInt64Ty(Ctx);
2859 StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2860 StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2861
2862 // Test for "#omp ordered depend(sink: vec)"
2863 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2864 StoreValues, ".cnt.addr",
2865 /*IsDependSource=*/false));
2866
2867 Builder.CreateRetVoid();
2868 OMPBuilder.finalize();
2869 EXPECT_FALSE(verifyModule(*M, &errs()));
2870
2871 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2872 ASSERT_NE(AllocInst, nullptr);
2873 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2874 EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2875 EXPECT_TRUE(
2876 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2877
2878 Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2879 for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2880 GetElementPtrInst *DependAddrGEPIter =
2881 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2882 ASSERT_NE(DependAddrGEPIter, nullptr);
2883 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2884 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2885 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2886 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2887 ASSERT_NE(FirstIdx, nullptr);
2888 ASSERT_NE(SecondIdx, nullptr);
2889 EXPECT_EQ(FirstIdx->getValue(), 0);
2890 EXPECT_EQ(SecondIdx->getValue(), Iter);
2891 StoreInst *StoreValue =
2892 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2893 ASSERT_NE(StoreValue, nullptr);
2894 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2895 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2896 EXPECT_EQ(StoreValue->getAlign(), Align(8));
2897 IterInst = dyn_cast<Instruction>(StoreValue);
2898 }
2899
2900 GetElementPtrInst *DependBaseAddrGEP =
2901 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2902 ASSERT_NE(DependBaseAddrGEP, nullptr);
2903 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2904 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2905 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2906 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2907 ASSERT_NE(FirstIdx, nullptr);
2908 ASSERT_NE(SecondIdx, nullptr);
2909 EXPECT_EQ(FirstIdx->getValue(), 0);
2910 EXPECT_EQ(SecondIdx->getValue(), 0);
2911
2912 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2913 ASSERT_NE(GTID, nullptr);
2914 EXPECT_EQ(GTID->arg_size(), 1U);
2915 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2916 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2917 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2918
2919 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2920 ASSERT_NE(Depend, nullptr);
2921 EXPECT_EQ(Depend->arg_size(), 3U);
2922 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait");
2923 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2924 EXPECT_EQ(Depend->getArgOperand(1), GTID);
2925 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2926 }
2927
2928 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
2929 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2930 OpenMPIRBuilder OMPBuilder(*M);
2931 OMPBuilder.initialize();
2932 F->setName("func");
2933 IRBuilder<> Builder(BB);
2934
2935 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2936
2937 AllocaInst *PrivAI =
2938 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
2939
2940 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2941 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2942 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2943 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2944
2945 Builder.restoreIP(CodeGenIP);
2946 Builder.CreateStore(F->arg_begin(), PrivAI);
2947 Value *PrivLoad =
2948 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2949 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2950 };
2951
2952 auto FiniCB = [&](InsertPointTy IP) {
2953 BasicBlock *IPBB = IP.getBlock();
2954 EXPECT_NE(IPBB->end(), IP.getPoint());
2955 };
2956
2957 // Test for "#omp ordered [threads]"
2958 BasicBlock *EntryBB = Builder.GetInsertBlock();
2959 Builder.restoreIP(
2960 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true));
2961
2962 Builder.CreateRetVoid();
2963 OMPBuilder.finalize();
2964 EXPECT_FALSE(verifyModule(*M, &errs()));
2965
2966 EXPECT_NE(EntryBB->getTerminator(), nullptr);
2967
2968 CallInst *OrderedEntryCI = nullptr;
2969 for (auto &EI : *EntryBB) {
2970 Instruction *Cur = &EI;
2971 if (isa<CallInst>(Cur)) {
2972 OrderedEntryCI = cast<CallInst>(Cur);
2973 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
2974 break;
2975 OrderedEntryCI = nullptr;
2976 }
2977 }
2978 EXPECT_NE(OrderedEntryCI, nullptr);
2979 EXPECT_EQ(OrderedEntryCI->arg_size(), 2U);
2980 EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered");
2981 EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0)));
2982
2983 CallInst *OrderedEndCI = nullptr;
2984 for (auto &FI : *EntryBB) {
2985 Instruction *Cur = &FI;
2986 if (isa<CallInst>(Cur)) {
2987 OrderedEndCI = cast<CallInst>(Cur);
2988 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
2989 break;
2990 OrderedEndCI = nullptr;
2991 }
2992 }
2993 EXPECT_NE(OrderedEndCI, nullptr);
2994 EXPECT_EQ(OrderedEndCI->arg_size(), 2U);
2995 EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0)));
2996 EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1));
2997 }
2998
2999 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
3000 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3001 OpenMPIRBuilder OMPBuilder(*M);
3002 OMPBuilder.initialize();
3003 F->setName("func");
3004 IRBuilder<> Builder(BB);
3005
3006 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3007
3008 AllocaInst *PrivAI =
3009 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
3010
3011 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3012 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3013 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3014 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3015
3016 Builder.restoreIP(CodeGenIP);
3017 Builder.CreateStore(F->arg_begin(), PrivAI);
3018 Value *PrivLoad =
3019 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3020 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3021 };
3022
3023 auto FiniCB = [&](InsertPointTy IP) {
3024 BasicBlock *IPBB = IP.getBlock();
3025 EXPECT_NE(IPBB->end(), IP.getPoint());
3026 };
3027
3028 // Test for "#omp ordered simd"
3029 BasicBlock *EntryBB = Builder.GetInsertBlock();
3030 Builder.restoreIP(
3031 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false));
3032
3033 Builder.CreateRetVoid();
3034 OMPBuilder.finalize();
3035 EXPECT_FALSE(verifyModule(*M, &errs()));
3036
3037 EXPECT_NE(EntryBB->getTerminator(), nullptr);
3038
3039 CallInst *OrderedEntryCI = nullptr;
3040 for (auto &EI : *EntryBB) {
3041 Instruction *Cur = &EI;
3042 if (isa<CallInst>(Cur)) {
3043 OrderedEntryCI = cast<CallInst>(Cur);
3044 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
3045 break;
3046 OrderedEntryCI = nullptr;
3047 }
3048 }
3049 EXPECT_EQ(OrderedEntryCI, nullptr);
3050
3051 CallInst *OrderedEndCI = nullptr;
3052 for (auto &FI : *EntryBB) {
3053 Instruction *Cur = &FI;
3054 if (isa<CallInst>(Cur)) {
3055 OrderedEndCI = cast<CallInst>(Cur);
3056 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
3057 break;
3058 OrderedEndCI = nullptr;
3059 }
3060 }
3061 EXPECT_EQ(OrderedEndCI, nullptr);
3062 }
3063
3064 TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {
3065 OpenMPIRBuilder OMPBuilder(*M);
3066 OMPBuilder.initialize();
3067 F->setName("func");
3068 IRBuilder<> Builder(BB);
3069
3070 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3071
3072 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3073 AllocaInst *MasterAddress = Builder.CreateAlloca(Builder.getPtrTy());
3074 AllocaInst *PrivAddress = Builder.CreateAlloca(Builder.getPtrTy());
3075
3076 BasicBlock *EntryBB = BB;
3077
3078 OMPBuilder.createCopyinClauseBlocks(Builder.saveIP(), MasterAddress,
3079 PrivAddress, Int32, /*BranchtoEnd*/ true);
3080
3081 BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator());
3082
3083 EXPECT_NE(EntryBr, nullptr);
3084 EXPECT_TRUE(EntryBr->isConditional());
3085
3086 BasicBlock *NotMasterBB = EntryBr->getSuccessor(0);
3087 BasicBlock *CopyinEnd = EntryBr->getSuccessor(1);
3088 CmpInst *CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition());
3089
3090 EXPECT_NE(CMP, nullptr);
3091 EXPECT_NE(NotMasterBB, nullptr);
3092 EXPECT_NE(CopyinEnd, nullptr);
3093
3094 BranchInst *NotMasterBr =
3095 dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator());
3096 EXPECT_NE(NotMasterBr, nullptr);
3097 EXPECT_FALSE(NotMasterBr->isConditional());
3098 EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0));
3099 }
3100
3101 TEST_F(OpenMPIRBuilderTest, SingleDirective) {
3102 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3103 OpenMPIRBuilder OMPBuilder(*M);
3104 OMPBuilder.initialize();
3105 F->setName("func");
3106 IRBuilder<> Builder(BB);
3107
3108 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3109
3110 AllocaInst *PrivAI = nullptr;
3111
3112 BasicBlock *EntryBB = nullptr;
3113 BasicBlock *ThenBB = nullptr;
3114
3115 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3116 if (AllocaIP.isSet())
3117 Builder.restoreIP(AllocaIP);
3118 else
3119 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3120 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
3121 Builder.CreateStore(F->arg_begin(), PrivAI);
3122
3123 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3124 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3125 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3126
3127 Builder.restoreIP(CodeGenIP);
3128
3129 // collect some info for checks later
3130 ThenBB = Builder.GetInsertBlock();
3131 EntryBB = ThenBB->getUniquePredecessor();
3132
3133 // simple instructions for body
3134 Value *PrivLoad =
3135 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3136 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3137 };
3138
3139 auto FiniCB = [&](InsertPointTy IP) {
3140 BasicBlock *IPBB = IP.getBlock();
3141 EXPECT_NE(IPBB->end(), IP.getPoint());
3142 };
3143
3144 Builder.restoreIP(OMPBuilder.createSingle(
3145 Builder, BodyGenCB, FiniCB, /*IsNowait*/ false, /*DidIt*/ nullptr));
3146 Value *EntryBBTI = EntryBB->getTerminator();
3147 EXPECT_NE(EntryBBTI, nullptr);
3148 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3149 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
3150 EXPECT_TRUE(EntryBr->isConditional());
3151 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3152 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3153 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3154
3155 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
3156 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3157
3158 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
3159 EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3160 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3161 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3162
3163 CallInst *SingleEndCI = nullptr;
3164 for (auto &FI : *ThenBB) {
3165 Instruction *cur = &FI;
3166 if (isa<CallInst>(cur)) {
3167 SingleEndCI = cast<CallInst>(cur);
3168 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3169 break;
3170 SingleEndCI = nullptr;
3171 }
3172 }
3173 EXPECT_NE(SingleEndCI, nullptr);
3174 EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3175 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3176 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3177
3178 bool FoundBarrier = false;
3179 for (auto &FI : *ExitBB) {
3180 Instruction *cur = &FI;
3181 if (auto CI = dyn_cast<CallInst>(cur)) {
3182 if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3183 FoundBarrier = true;
3184 break;
3185 }
3186 }
3187 }
3188 EXPECT_TRUE(FoundBarrier);
3189 }
3190
3191 TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
3192 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3193 OpenMPIRBuilder OMPBuilder(*M);
3194 OMPBuilder.initialize();
3195 F->setName("func");
3196 IRBuilder<> Builder(BB);
3197
3198 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3199
3200 AllocaInst *PrivAI = nullptr;
3201
3202 BasicBlock *EntryBB = nullptr;
3203 BasicBlock *ThenBB = nullptr;
3204
3205 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
3206 if (AllocaIP.isSet())
3207 Builder.restoreIP(AllocaIP);
3208 else
3209 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
3210 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
3211 Builder.CreateStore(F->arg_begin(), PrivAI);
3212
3213 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3214 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3215 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3216
3217 Builder.restoreIP(CodeGenIP);
3218
3219 // collect some info for checks later
3220 ThenBB = Builder.GetInsertBlock();
3221 EntryBB = ThenBB->getUniquePredecessor();
3222
3223 // simple instructions for body
3224 Value *PrivLoad =
3225 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3226 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3227 };
3228
3229 auto FiniCB = [&](InsertPointTy IP) {
3230 BasicBlock *IPBB = IP.getBlock();
3231 EXPECT_NE(IPBB->end(), IP.getPoint());
3232 };
3233
3234 Builder.restoreIP(OMPBuilder.createSingle(
3235 Builder, BodyGenCB, FiniCB, /*IsNowait*/ true, /*DidIt*/ nullptr));
3236 Value *EntryBBTI = EntryBB->getTerminator();
3237 EXPECT_NE(EntryBBTI, nullptr);
3238 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3239 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
3240 EXPECT_TRUE(EntryBr->isConditional());
3241 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3242 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3243 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3244
3245 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
3246 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3247
3248 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
3249 EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3250 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3251 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3252
3253 CallInst *SingleEndCI = nullptr;
3254 for (auto &FI : *ThenBB) {
3255 Instruction *cur = &FI;
3256 if (isa<CallInst>(cur)) {
3257 SingleEndCI = cast<CallInst>(cur);
3258 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3259 break;
3260 SingleEndCI = nullptr;
3261 }
3262 }
3263 EXPECT_NE(SingleEndCI, nullptr);
3264 EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3265 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3266 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3267
3268 CallInst *ExitBarrier = nullptr;
3269 for (auto &FI : *ExitBB) {
3270 Instruction *cur = &FI;
3271 if (auto CI = dyn_cast<CallInst>(cur)) {
3272 if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3273 ExitBarrier = CI;
3274 break;
3275 }
3276 }
3277 }
3278 EXPECT_EQ(ExitBarrier, nullptr);
3279 }
3280
3281 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) {
3282 OpenMPIRBuilder OMPBuilder(*M);
3283 OMPBuilder.initialize();
3284 F->setName("func");
3285 IRBuilder<> Builder(BB);
3286
3287 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3288
3289 Type *Float32 = Type::getFloatTy(M->getContext());
3290 AllocaInst *XVal = Builder.CreateAlloca(Float32);
3291 XVal->setName("AtomicVar");
3292 AllocaInst *VVal = Builder.CreateAlloca(Float32);
3293 VVal->setName("AtomicRead");
3294 AtomicOrdering AO = AtomicOrdering::Monotonic;
3295 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3296 OpenMPIRBuilder::AtomicOpValue V = {VVal, Float32, false, false};
3297
3298 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3299
3300 IntegerType *IntCastTy =
3301 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3302
3303 LoadInst *AtomicLoad = cast<LoadInst>(VVal->getNextNode());
3304 EXPECT_TRUE(AtomicLoad->isAtomic());
3305 EXPECT_EQ(AtomicLoad->getPointerOperand(), XVal);
3306
3307 BitCastInst *CastToFlt = cast<BitCastInst>(AtomicLoad->getNextNode());
3308 EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy);
3309 EXPECT_EQ(CastToFlt->getDestTy(), Float32);
3310 EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad);
3311
3312 StoreInst *StoreofAtomic = cast<StoreInst>(CastToFlt->getNextNode());
3313 EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt);
3314 EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal);
3315
3316 Builder.CreateRetVoid();
3317 OMPBuilder.finalize();
3318 EXPECT_FALSE(verifyModule(*M, &errs()));
3319 }
3320
3321 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) {
3322 OpenMPIRBuilder OMPBuilder(*M);
3323 OMPBuilder.initialize();
3324 F->setName("func");
3325 IRBuilder<> Builder(BB);
3326
3327 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3328
3329 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3330 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3331 XVal->setName("AtomicVar");
3332 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3333 VVal->setName("AtomicRead");
3334 AtomicOrdering AO = AtomicOrdering::Monotonic;
3335 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3336 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3337
3338 BasicBlock *EntryBB = BB;
3339
3340 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3341 LoadInst *AtomicLoad = nullptr;
3342 StoreInst *StoreofAtomic = nullptr;
3343
3344 for (Instruction &Cur : *EntryBB) {
3345 if (isa<LoadInst>(Cur)) {
3346 AtomicLoad = cast<LoadInst>(&Cur);
3347 if (AtomicLoad->getPointerOperand() == XVal)
3348 continue;
3349 AtomicLoad = nullptr;
3350 } else if (isa<StoreInst>(Cur)) {
3351 StoreofAtomic = cast<StoreInst>(&Cur);
3352 if (StoreofAtomic->getPointerOperand() == VVal)
3353 continue;
3354 StoreofAtomic = nullptr;
3355 }
3356 }
3357
3358 EXPECT_NE(AtomicLoad, nullptr);
3359 EXPECT_TRUE(AtomicLoad->isAtomic());
3360
3361 EXPECT_NE(StoreofAtomic, nullptr);
3362 EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad);
3363
3364 Builder.CreateRetVoid();
3365 OMPBuilder.finalize();
3366
3367 EXPECT_FALSE(verifyModule(*M, &errs()));
3368 }
3369
3370 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) {
3371 OpenMPIRBuilder OMPBuilder(*M);
3372 OMPBuilder.initialize();
3373 F->setName("func");
3374 IRBuilder<> Builder(BB);
3375
3376 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3377
3378 LLVMContext &Ctx = M->getContext();
3379 Type *Float32 = Type::getFloatTy(Ctx);
3380 AllocaInst *XVal = Builder.CreateAlloca(Float32);
3381 XVal->setName("AtomicVar");
3382 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3383 AtomicOrdering AO = AtomicOrdering::Monotonic;
3384 Constant *ValToWrite = ConstantFP::get(Float32, 1.0);
3385
3386 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3387
3388 IntegerType *IntCastTy =
3389 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3390
3391 Value *ExprCast = Builder.CreateBitCast(ValToWrite, IntCastTy);
3392
3393 StoreInst *StoreofAtomic = cast<StoreInst>(XVal->getNextNode());
3394 EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast);
3395 EXPECT_EQ(StoreofAtomic->getPointerOperand(), XVal);
3396 EXPECT_TRUE(StoreofAtomic->isAtomic());
3397
3398 Builder.CreateRetVoid();
3399 OMPBuilder.finalize();
3400 EXPECT_FALSE(verifyModule(*M, &errs()));
3401 }
3402
3403 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) {
3404 OpenMPIRBuilder OMPBuilder(*M);
3405 OMPBuilder.initialize();
3406 F->setName("func");
3407 IRBuilder<> Builder(BB);
3408
3409 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3410
3411 LLVMContext &Ctx = M->getContext();
3412 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3413 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3414 XVal->setName("AtomicVar");
3415 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3416 AtomicOrdering AO = AtomicOrdering::Monotonic;
3417 ConstantInt *ValToWrite = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3418
3419 BasicBlock *EntryBB = BB;
3420
3421 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3422
3423 StoreInst *StoreofAtomic = nullptr;
3424
3425 for (Instruction &Cur : *EntryBB) {
3426 if (isa<StoreInst>(Cur)) {
3427 StoreofAtomic = cast<StoreInst>(&Cur);
3428 if (StoreofAtomic->getPointerOperand() == XVal)
3429 continue;
3430 StoreofAtomic = nullptr;
3431 }
3432 }
3433
3434 EXPECT_NE(StoreofAtomic, nullptr);
3435 EXPECT_TRUE(StoreofAtomic->isAtomic());
3436 EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite);
3437
3438 Builder.CreateRetVoid();
3439 OMPBuilder.finalize();
3440 EXPECT_FALSE(verifyModule(*M, &errs()));
3441 }
3442
3443 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
3444 OpenMPIRBuilder OMPBuilder(*M);
3445 OMPBuilder.initialize();
3446 F->setName("func");
3447 IRBuilder<> Builder(BB);
3448
3449 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3450
3451 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3452 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3453 XVal->setName("AtomicVar");
3454 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3455 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3456 AtomicOrdering AO = AtomicOrdering::Monotonic;
3457 ConstantInt *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3458 Value *Expr = nullptr;
3459 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub;
3460 bool IsXLHSInRHSPart = false;
3461
3462 BasicBlock *EntryBB = BB;
3463 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3464 EntryBB->getFirstInsertionPt());
3465 Value *Sub = nullptr;
3466
3467 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3468 Sub = IRB.CreateSub(ConstVal, Atomic);
3469 return Sub;
3470 };
3471 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3472 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3473 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3474 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3475 EXPECT_NE(ContTI, nullptr);
3476 BasicBlock *EndBB = ContTI->getSuccessor(0);
3477 EXPECT_TRUE(ContTI->isConditional());
3478 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3479 EXPECT_NE(EndBB, nullptr);
3480
3481 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3482 EXPECT_NE(Phi, nullptr);
3483 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3484 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3485 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3486
3487 EXPECT_EQ(Sub->getNumUses(), 1U);
3488 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3489 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3490
3491 ExtractValueInst *ExVI1 =
3492 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3493 EXPECT_NE(ExVI1, nullptr);
3494 AtomicCmpXchgInst *CmpExchg =
3495 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3496 EXPECT_NE(CmpExchg, nullptr);
3497 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3498 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3499 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3500
3501 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3502 EXPECT_NE(Ld, nullptr);
3503 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3504
3505 Builder.CreateRetVoid();
3506 OMPBuilder.finalize();
3507 EXPECT_FALSE(verifyModule(*M, &errs()));
3508 }
3509
3510 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
3511 OpenMPIRBuilder OMPBuilder(*M);
3512 OMPBuilder.initialize();
3513 F->setName("func");
3514 IRBuilder<> Builder(BB);
3515
3516 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3517
3518 Type *FloatTy = Type::getFloatTy(M->getContext());
3519 AllocaInst *XVal = Builder.CreateAlloca(FloatTy);
3520 XVal->setName("AtomicVar");
3521 Builder.CreateStore(ConstantFP::get(Type::getFloatTy(Ctx), 0.0), XVal);
3522 OpenMPIRBuilder::AtomicOpValue X = {XVal, FloatTy, false, false};
3523 AtomicOrdering AO = AtomicOrdering::Monotonic;
3524 Constant *ConstVal = ConstantFP::get(Type::getFloatTy(Ctx), 1.0);
3525 Value *Expr = nullptr;
3526 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::FSub;
3527 bool IsXLHSInRHSPart = false;
3528
3529 BasicBlock *EntryBB = BB;
3530 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3531 EntryBB->getFirstInsertionPt());
3532 Value *Sub = nullptr;
3533
3534 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3535 Sub = IRB.CreateFSub(ConstVal, Atomic);
3536 return Sub;
3537 };
3538 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3539 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3540 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3541 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3542 EXPECT_NE(ContTI, nullptr);
3543 BasicBlock *EndBB = ContTI->getSuccessor(0);
3544 EXPECT_TRUE(ContTI->isConditional());
3545 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3546 EXPECT_NE(EndBB, nullptr);
3547
3548 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3549 EXPECT_NE(Phi, nullptr);
3550 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3551 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3552 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3553
3554 EXPECT_EQ(Sub->getNumUses(), 1U);
3555 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3556 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3557
3558 ExtractValueInst *ExVI1 =
3559 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3560 EXPECT_NE(ExVI1, nullptr);
3561 AtomicCmpXchgInst *CmpExchg =
3562 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3563 EXPECT_NE(CmpExchg, nullptr);
3564 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3565 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3566 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3567
3568 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3569 EXPECT_NE(Ld, nullptr);
3570 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3571 Builder.CreateRetVoid();
3572 OMPBuilder.finalize();
3573 EXPECT_FALSE(verifyModule(*M, &errs()));
3574 }
3575
3576 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
3577 OpenMPIRBuilder OMPBuilder(*M);
3578 OMPBuilder.initialize();
3579 F->setName("func");
3580 IRBuilder<> Builder(BB);
3581
3582 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3583
3584 Type *IntTy = Type::getInt32Ty(M->getContext());
3585 AllocaInst *XVal = Builder.CreateAlloca(IntTy);
3586 XVal->setName("AtomicVar");
3587 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);
3588 OpenMPIRBuilder::AtomicOpValue X = {XVal, IntTy, false, false};
3589 AtomicOrdering AO = AtomicOrdering::Monotonic;
3590 Constant *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3591 Value *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3592 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::UMax;
3593 bool IsXLHSInRHSPart = false;
3594
3595 BasicBlock *EntryBB = BB;
3596 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3597 EntryBB->getFirstInsertionPt());
3598 Value *Sub = nullptr;
3599
3600 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3601 Sub = IRB.CreateSub(ConstVal, Atomic);
3602 return Sub;
3603 };
3604 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3605 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3606 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3607 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3608 EXPECT_NE(ContTI, nullptr);
3609 BasicBlock *EndBB = ContTI->getSuccessor(0);
3610 EXPECT_TRUE(ContTI->isConditional());
3611 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3612 EXPECT_NE(EndBB, nullptr);
3613
3614 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3615 EXPECT_NE(Phi, nullptr);
3616 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3617 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3618 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3619
3620 EXPECT_EQ(Sub->getNumUses(), 1U);
3621 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3622 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3623
3624 ExtractValueInst *ExVI1 =
3625 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3626 EXPECT_NE(ExVI1, nullptr);
3627 AtomicCmpXchgInst *CmpExchg =
3628 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3629 EXPECT_NE(CmpExchg, nullptr);
3630 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3631 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3632 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3633
3634 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3635 EXPECT_NE(Ld, nullptr);
3636 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3637
3638 Builder.CreateRetVoid();
3639 OMPBuilder.finalize();
3640 EXPECT_FALSE(verifyModule(*M, &errs()));
3641 }
3642
3643 TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {
3644 OpenMPIRBuilder OMPBuilder(*M);
3645 OMPBuilder.initialize();
3646 F->setName("func");
3647 IRBuilder<> Builder(BB);
3648
3649 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3650
3651 LLVMContext &Ctx = M->getContext();
3652 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3653 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3654 XVal->setName("AtomicVar");
3655 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3656 VVal->setName("AtomicCapTar");
3657 StoreInst *Init =
3658 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3659
3660 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3661 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3662 AtomicOrdering AO = AtomicOrdering::Monotonic;
3663 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3664 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add;
3665 bool IsXLHSInRHSPart = true;
3666 bool IsPostfixUpdate = true;
3667 bool UpdateExpr = true;
3668
3669 BasicBlock *EntryBB = BB;
3670 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3671 EntryBB->getFirstInsertionPt());
3672
3673 // integer update - not used
3674 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };
3675
3676 Builder.restoreIP(OMPBuilder.createAtomicCapture(
3677 Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr,
3678 IsPostfixUpdate, IsXLHSInRHSPart));
3679 EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3680 AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3681 EXPECT_NE(ARWM, nullptr);
3682 EXPECT_EQ(ARWM->getPointerOperand(), XVal);
3683 EXPECT_EQ(ARWM->getOperation(), RMWOp);
3684 StoreInst *St = dyn_cast<StoreInst>(ARWM->user_back());
3685 EXPECT_NE(St, nullptr);
3686 EXPECT_EQ(St->getPointerOperand(), VVal);
3687
3688 Builder.CreateRetVoid();
3689 OMPBuilder.finalize();
3690 EXPECT_FALSE(verifyModule(*M, &errs()));
3691 }
3692
3693 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompare) {
3694 OpenMPIRBuilder OMPBuilder(*M);
3695 OMPBuilder.initialize();
3696 F->setName("func");
3697 IRBuilder<> Builder(BB);
3698
3699 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3700
3701 LLVMContext &Ctx = M->getContext();
3702 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3703 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3704 XVal->setName("x");
3705 StoreInst *Init =
3706 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3707
3708 OpenMPIRBuilder::AtomicOpValue XSigned = {XVal, Int32, true, false};
3709 OpenMPIRBuilder::AtomicOpValue XUnsigned = {XVal, Int32, false, false};
3710 // V and R are not used in atomic compare
3711 OpenMPIRBuilder::AtomicOpValue V = {nullptr, nullptr, false, false};
3712 OpenMPIRBuilder::AtomicOpValue R = {nullptr, nullptr, false, false};
3713 AtomicOrdering AO = AtomicOrdering::Monotonic;
3714 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3715 ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3716 OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3717 OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3718
3719 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3720 Builder, XSigned, V, R, Expr, nullptr, AO, OpMax, true, false, false));
3721 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3722 Builder, XUnsigned, V, R, Expr, nullptr, AO, OpMax, false, false, false));
3723 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3724 Builder, XSigned, V, R, Expr, D, AO, OpEQ, true, false, false));
3725
3726 BasicBlock *EntryBB = BB;
3727 EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3728 EXPECT_EQ(EntryBB->size(), 5U);
3729
3730 AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3731 EXPECT_NE(ARWM1, nullptr);
3732 EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3733 EXPECT_EQ(ARWM1->getValOperand(), Expr);
3734 EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3735
3736 AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(ARWM1->getNextNode());
3737 EXPECT_NE(ARWM2, nullptr);
3738 EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3739 EXPECT_EQ(ARWM2->getValOperand(), Expr);
3740 EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::UMax);
3741
3742 AtomicCmpXchgInst *AXCHG = dyn_cast<AtomicCmpXchgInst>(ARWM2->getNextNode());
3743 EXPECT_NE(AXCHG, nullptr);
3744 EXPECT_EQ(AXCHG->getPointerOperand(), XVal);
3745 EXPECT_EQ(AXCHG->getCompareOperand(), Expr);
3746 EXPECT_EQ(AXCHG->getNewValOperand(), D);
3747
3748 Builder.CreateRetVoid();
3749 OMPBuilder.finalize();
3750 EXPECT_FALSE(verifyModule(*M, &errs()));
3751 }
3752
3753 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) {
3754 OpenMPIRBuilder OMPBuilder(*M);
3755 OMPBuilder.initialize();
3756 F->setName("func");
3757 IRBuilder<> Builder(BB);
3758
3759 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3760
3761 LLVMContext &Ctx = M->getContext();
3762 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3763 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3764 XVal->setName("x");
3765 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3766 VVal->setName("v");
3767 AllocaInst *RVal = Builder.CreateAlloca(Int32);
3768 RVal->setName("r");
3769
3770 StoreInst *Init =
3771 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3772
3773 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, true, false};
3774 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3775 OpenMPIRBuilder::AtomicOpValue NoV = {nullptr, nullptr, false, false};
3776 OpenMPIRBuilder::AtomicOpValue R = {RVal, Int32, false, false};
3777 OpenMPIRBuilder::AtomicOpValue NoR = {nullptr, nullptr, false, false};
3778
3779 AtomicOrdering AO = AtomicOrdering::Monotonic;
3780 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3781 ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3782 OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3783 OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3784
3785 // { cond-update-stmt v = x; }
3786 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3787 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3788 /* IsPostfixUpdate */ false,
3789 /* IsFailOnly */ false));
3790 // { v = x; cond-update-stmt }
3791 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3792 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3793 /* IsPostfixUpdate */ true,
3794 /* IsFailOnly */ false));
3795 // if(x == e) { x = d; } else { v = x; }
3796 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3797 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3798 /* IsPostfixUpdate */ false,
3799 /* IsFailOnly */ true));
3800 // { r = x == e; if(r) { x = d; } }
3801 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3802 Builder, X, NoV, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3803 /* IsPostfixUpdate */ false,
3804 /* IsFailOnly */ false));
3805 // { r = x == e; if(r) { x = d; } else { v = x; } }
3806 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3807 Builder, X, V, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3808 /* IsPostfixUpdate */ false,
3809 /* IsFailOnly */ true));
3810
3811 // { v = x; cond-update-stmt }
3812 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3813 Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ true,
3814 /* IsPostfixUpdate */ true,
3815 /* IsFailOnly */ false));
3816 // { cond-update-stmt v = x; }
3817 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3818 Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ false,
3819 /* IsPostfixUpdate */ false,
3820 /* IsFailOnly */ false));
3821
3822 BasicBlock *EntryBB = BB;
3823 EXPECT_EQ(EntryBB->getParent()->size(), 5U);
3824 BasicBlock *Cont1 = dyn_cast<BasicBlock>(EntryBB->getNextNode());
3825 EXPECT_NE(Cont1, nullptr);
3826 BasicBlock *Exit1 = dyn_cast<BasicBlock>(Cont1->getNextNode());
3827 EXPECT_NE(Exit1, nullptr);
3828 BasicBlock *Cont2 = dyn_cast<BasicBlock>(Exit1->getNextNode());
3829 EXPECT_NE(Cont2, nullptr);
3830 BasicBlock *Exit2 = dyn_cast<BasicBlock>(Cont2->getNextNode());
3831 EXPECT_NE(Exit2, nullptr);
3832
3833 AtomicCmpXchgInst *CmpXchg1 =
3834 dyn_cast<AtomicCmpXchgInst>(Init->getNextNode());
3835 EXPECT_NE(CmpXchg1, nullptr);
3836 EXPECT_EQ(CmpXchg1->getPointerOperand(), XVal);
3837 EXPECT_EQ(CmpXchg1->getCompareOperand(), Expr);
3838 EXPECT_EQ(CmpXchg1->getNewValOperand(), D);
3839 ExtractValueInst *ExtVal1 =
3840 dyn_cast<ExtractValueInst>(CmpXchg1->getNextNode());
3841 EXPECT_NE(ExtVal1, nullptr);
3842 EXPECT_EQ(ExtVal1->getAggregateOperand(), CmpXchg1);
3843 EXPECT_EQ(ExtVal1->getIndices(), ArrayRef<unsigned int>(0U));
3844 ExtractValueInst *ExtVal2 =
3845 dyn_cast<ExtractValueInst>(ExtVal1->getNextNode());
3846 EXPECT_NE(ExtVal2, nullptr);
3847 EXPECT_EQ(ExtVal2->getAggregateOperand(), CmpXchg1);
3848 EXPECT_EQ(ExtVal2->getIndices(), ArrayRef<unsigned int>(1U));
3849 SelectInst *Sel1 = dyn_cast<SelectInst>(ExtVal2->getNextNode());
3850 EXPECT_NE(Sel1, nullptr);
3851 EXPECT_EQ(Sel1->getCondition(), ExtVal2);
3852 EXPECT_EQ(Sel1->getTrueValue(), Expr);
3853 EXPECT_EQ(Sel1->getFalseValue(), ExtVal1);
3854 StoreInst *Store1 = dyn_cast<StoreInst>(Sel1->getNextNode());
3855 EXPECT_NE(Store1, nullptr);
3856 EXPECT_EQ(Store1->getPointerOperand(), VVal);
3857 EXPECT_EQ(Store1->getValueOperand(), Sel1);
3858
3859 AtomicCmpXchgInst *CmpXchg2 =
3860 dyn_cast<AtomicCmpXchgInst>(Store1->getNextNode());
3861 EXPECT_NE(CmpXchg2, nullptr);
3862 EXPECT_EQ(CmpXchg2->getPointerOperand(), XVal);
3863 EXPECT_EQ(CmpXchg2->getCompareOperand(), Expr);
3864 EXPECT_EQ(CmpXchg2->getNewValOperand(), D);
3865 ExtractValueInst *ExtVal3 =
3866 dyn_cast<ExtractValueInst>(CmpXchg2->getNextNode());
3867 EXPECT_NE(ExtVal3, nullptr);
3868 EXPECT_EQ(ExtVal3->getAggregateOperand(), CmpXchg2);
3869 EXPECT_EQ(ExtVal3->getIndices(), ArrayRef<unsigned int>(0U));
3870 StoreInst *Store2 = dyn_cast<StoreInst>(ExtVal3->getNextNode());
3871 EXPECT_NE(Store2, nullptr);
3872 EXPECT_EQ(Store2->getPointerOperand(), VVal);
3873 EXPECT_EQ(Store2->getValueOperand(), ExtVal3);
3874
3875 AtomicCmpXchgInst *CmpXchg3 =
3876 dyn_cast<AtomicCmpXchgInst>(Store2->getNextNode());
3877 EXPECT_NE(CmpXchg3, nullptr);
3878 EXPECT_EQ(CmpXchg3->getPointerOperand(), XVal);
3879 EXPECT_EQ(CmpXchg3->getCompareOperand(), Expr);
3880 EXPECT_EQ(CmpXchg3->getNewValOperand(), D);
3881 ExtractValueInst *ExtVal4 =
3882 dyn_cast<ExtractValueInst>(CmpXchg3->getNextNode());
3883 EXPECT_NE(ExtVal4, nullptr);
3884 EXPECT_EQ(ExtVal4->getAggregateOperand(), CmpXchg3);
3885 EXPECT_EQ(ExtVal4->getIndices(), ArrayRef<unsigned int>(0U));
3886 ExtractValueInst *ExtVal5 =
3887 dyn_cast<ExtractValueInst>(ExtVal4->getNextNode());
3888 EXPECT_NE(ExtVal5, nullptr);
3889 EXPECT_EQ(ExtVal5->getAggregateOperand(), CmpXchg3);
3890 EXPECT_EQ(ExtVal5->getIndices(), ArrayRef<unsigned int>(1U));
3891 BranchInst *Br1 = dyn_cast<BranchInst>(ExtVal5->getNextNode());
3892 EXPECT_NE(Br1, nullptr);
3893 EXPECT_EQ(Br1->isConditional(), true);
3894 EXPECT_EQ(Br1->getCondition(), ExtVal5);
3895 EXPECT_EQ(Br1->getSuccessor(0), Exit1);
3896 EXPECT_EQ(Br1->getSuccessor(1), Cont1);
3897
3898 StoreInst *Store3 = dyn_cast<StoreInst>(&Cont1->front());
3899 EXPECT_NE(Store3, nullptr);
3900 EXPECT_EQ(Store3->getPointerOperand(), VVal);
3901 EXPECT_EQ(Store3->getValueOperand(), ExtVal4);
3902 BranchInst *Br2 = dyn_cast<BranchInst>(Store3->getNextNode());
3903 EXPECT_NE(Br2, nullptr);
3904 EXPECT_EQ(Br2->isUnconditional(), true);
3905 EXPECT_EQ(Br2->getSuccessor(0), Exit1);
3906
3907 AtomicCmpXchgInst *CmpXchg4 = dyn_cast<AtomicCmpXchgInst>(&Exit1->front());
3908 EXPECT_NE(CmpXchg4, nullptr);
3909 EXPECT_EQ(CmpXchg4->getPointerOperand(), XVal);
3910 EXPECT_EQ(CmpXchg4->getCompareOperand(), Expr);
3911 EXPECT_EQ(CmpXchg4->getNewValOperand(), D);
3912 ExtractValueInst *ExtVal6 =
3913 dyn_cast<ExtractValueInst>(CmpXchg4->getNextNode());
3914 EXPECT_NE(ExtVal6, nullptr);
3915 EXPECT_EQ(ExtVal6->getAggregateOperand(), CmpXchg4);
3916 EXPECT_EQ(ExtVal6->getIndices(), ArrayRef<unsigned int>(1U));
3917 ZExtInst *ZExt1 = dyn_cast<ZExtInst>(ExtVal6->getNextNode());
3918 EXPECT_NE(ZExt1, nullptr);
3919 EXPECT_EQ(ZExt1->getDestTy(), Int32);
3920 StoreInst *Store4 = dyn_cast<StoreInst>(ZExt1->getNextNode());
3921 EXPECT_NE(Store4, nullptr);
3922 EXPECT_EQ(Store4->getPointerOperand(), RVal);
3923 EXPECT_EQ(Store4->getValueOperand(), ZExt1);
3924
3925 AtomicCmpXchgInst *CmpXchg5 =
3926 dyn_cast<AtomicCmpXchgInst>(Store4->getNextNode());
3927 EXPECT_NE(CmpXchg5, nullptr);
3928 EXPECT_EQ(CmpXchg5->getPointerOperand(), XVal);
3929 EXPECT_EQ(CmpXchg5->getCompareOperand(), Expr);
3930 EXPECT_EQ(CmpXchg5->getNewValOperand(), D);
3931 ExtractValueInst *ExtVal7 =
3932 dyn_cast<ExtractValueInst>(CmpXchg5->getNextNode());
3933 EXPECT_NE(ExtVal7, nullptr);
3934 EXPECT_EQ(ExtVal7->getAggregateOperand(), CmpXchg5);
3935 EXPECT_EQ(ExtVal7->getIndices(), ArrayRef<unsigned int>(0U));
3936 ExtractValueInst *ExtVal8 =
3937 dyn_cast<ExtractValueInst>(ExtVal7->getNextNode());
3938 EXPECT_NE(ExtVal8, nullptr);
3939 EXPECT_EQ(ExtVal8->getAggregateOperand(), CmpXchg5);
3940 EXPECT_EQ(ExtVal8->getIndices(), ArrayRef<unsigned int>(1U));
3941 BranchInst *Br3 = dyn_cast<BranchInst>(ExtVal8->getNextNode());
3942 EXPECT_NE(Br3, nullptr);
3943 EXPECT_EQ(Br3->isConditional(), true);
3944 EXPECT_EQ(Br3->getCondition(), ExtVal8);
3945 EXPECT_EQ(Br3->getSuccessor(0), Exit2);
3946 EXPECT_EQ(Br3->getSuccessor(1), Cont2);
3947
3948 StoreInst *Store5 = dyn_cast<StoreInst>(&Cont2->front());
3949 EXPECT_NE(Store5, nullptr);
3950 EXPECT_EQ(Store5->getPointerOperand(), VVal);
3951 EXPECT_EQ(Store5->getValueOperand(), ExtVal7);
3952 BranchInst *Br4 = dyn_cast<BranchInst>(Store5->getNextNode());
3953 EXPECT_NE(Br4, nullptr);
3954 EXPECT_EQ(Br4->isUnconditional(), true);
3955 EXPECT_EQ(Br4->getSuccessor(0), Exit2);
3956
3957 ExtractValueInst *ExtVal9 = dyn_cast<ExtractValueInst>(&Exit2->front());
3958 EXPECT_NE(ExtVal9, nullptr);
3959 EXPECT_EQ(ExtVal9->getAggregateOperand(), CmpXchg5);
3960 EXPECT_EQ(ExtVal9->getIndices(), ArrayRef<unsigned int>(1U));
3961 ZExtInst *ZExt2 = dyn_cast<ZExtInst>(ExtVal9->getNextNode());
3962 EXPECT_NE(ZExt2, nullptr);
3963 EXPECT_EQ(ZExt2->getDestTy(), Int32);
3964 StoreInst *Store6 = dyn_cast<StoreInst>(ZExt2->getNextNode());
3965 EXPECT_NE(Store6, nullptr);
3966 EXPECT_EQ(Store6->getPointerOperand(), RVal);
3967 EXPECT_EQ(Store6->getValueOperand(), ZExt2);
3968
3969 AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Store6->getNextNode());
3970 EXPECT_NE(ARWM1, nullptr);
3971 EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3972 EXPECT_EQ(ARWM1->getValOperand(), Expr);
3973 EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3974 StoreInst *Store7 = dyn_cast<StoreInst>(ARWM1->getNextNode());
3975 EXPECT_NE(Store7, nullptr);
3976 EXPECT_EQ(Store7->getPointerOperand(), VVal);
3977 EXPECT_EQ(Store7->getValueOperand(), ARWM1);
3978
3979 AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Store7->getNextNode());
3980 EXPECT_NE(ARWM2, nullptr);
3981 EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3982 EXPECT_EQ(ARWM2->getValOperand(), Expr);
3983 EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::Max);
3984 CmpInst *Cmp1 = dyn_cast<CmpInst>(ARWM2->getNextNode());
3985 EXPECT_NE(Cmp1, nullptr);
3986 EXPECT_EQ(Cmp1->getPredicate(), CmpInst::ICMP_SGT);
3987 EXPECT_EQ(Cmp1->getOperand(0), ARWM2);
3988 EXPECT_EQ(Cmp1->getOperand(1), Expr);
3989 SelectInst *Sel2 = dyn_cast<SelectInst>(Cmp1->getNextNode());
3990 EXPECT_NE(Sel2, nullptr);
3991 EXPECT_EQ(Sel2->getCondition(), Cmp1);
3992 EXPECT_EQ(Sel2->getTrueValue(), Expr);
3993 EXPECT_EQ(Sel2->getFalseValue(), ARWM2);
3994 StoreInst *Store8 = dyn_cast<StoreInst>(Sel2->getNextNode());
3995 EXPECT_NE(Store8, nullptr);
3996 EXPECT_EQ(Store8->getPointerOperand(), VVal);
3997 EXPECT_EQ(Store8->getValueOperand(), Sel2);
3998
3999 Builder.CreateRetVoid();
4000 OMPBuilder.finalize();
4001 EXPECT_FALSE(verifyModule(*M, &errs()));
4002 }
4003
4004 TEST_F(OpenMPIRBuilderTest, CreateTeams) {
4005 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4006 OpenMPIRBuilder OMPBuilder(*M);
4007 OMPBuilder.initialize();
4008 F->setName("func");
4009 IRBuilder<> Builder(BB);
4010
4011 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
4012 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
4013 Value *Val128 = Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "load");
4014
4015 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4016 Builder.restoreIP(AllocaIP);
4017 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
4018 "bodygen.alloca128");
4019
4020 Builder.restoreIP(CodeGenIP);
4021 // Loading and storing captured pointer and values
4022 Builder.CreateStore(Val128, Local128);
4023 Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
4024 "bodygen.load32");
4025
4026 LoadInst *PrivLoad128 = Builder.CreateLoad(
4027 Local128->getAllocatedType(), Local128, "bodygen.local.load128");
4028 Value *Cmp = Builder.CreateICmpNE(
4029 Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));
4030 Instruction *ThenTerm, *ElseTerm;
4031 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
4032 &ThenTerm, &ElseTerm);
4033 };
4034
4035 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4036 Builder.restoreIP(OMPBuilder.createTeams(
4037 Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
4038 /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
4039
4040 OMPBuilder.finalize();
4041 Builder.CreateRetVoid();
4042
4043 EXPECT_FALSE(verifyModule(*M, &errs()));
4044
4045 CallInst *TeamsForkCall = dyn_cast<CallInst>(
4046 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams)
4047 ->user_back());
4048
4049 // Verify the Ident argument
4050 GlobalVariable *Ident = cast<GlobalVariable>(TeamsForkCall->getArgOperand(0));
4051 ASSERT_NE(Ident, nullptr);
4052 EXPECT_TRUE(Ident->hasInitializer());
4053 Constant *Initializer = Ident->getInitializer();
4054 GlobalVariable *SrcStrGlob =
4055 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
4056 ASSERT_NE(SrcStrGlob, nullptr);
4057 ConstantDataArray *SrcSrc =
4058 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
4059 ASSERT_NE(SrcSrc, nullptr);
4060
4061 // Verify the outlined function signature.
4062 Function *OutlinedFn =
4063 dyn_cast<Function>(TeamsForkCall->getArgOperand(2)->stripPointerCasts());
4064 ASSERT_NE(OutlinedFn, nullptr);
4065 EXPECT_FALSE(OutlinedFn->isDeclaration());
4066 EXPECT_TRUE(OutlinedFn->arg_size() >= 3);
4067 EXPECT_EQ(OutlinedFn->getArg(0)->getType(), Builder.getPtrTy()); // global_tid
4068 EXPECT_EQ(OutlinedFn->getArg(1)->getType(), Builder.getPtrTy()); // bound_tid
4069 EXPECT_EQ(OutlinedFn->getArg(2)->getType(),
4070 Builder.getPtrTy()); // captured args
4071
4072 // Check for TruncInst and ICmpInst in the outlined function.
4073 EXPECT_TRUE(any_of(instructions(OutlinedFn),
4074 [](Instruction &inst) { return isa<TruncInst>(&inst); }));
4075 EXPECT_TRUE(any_of(instructions(OutlinedFn),
4076 [](Instruction &inst) { return isa<ICmpInst>(&inst); }));
4077 }
4078
4079 TEST_F(OpenMPIRBuilderTest, CreateTeamsWithThreadLimit) {
4080 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4081 OpenMPIRBuilder OMPBuilder(*M);
4082 OMPBuilder.initialize();
4083 F->setName("func");
4084 IRBuilder<> &Builder = OMPBuilder.Builder;
4085 Builder.SetInsertPoint(BB);
4086
4087 Function *FakeFunction =
4088 Function::Create(FunctionType::get(Builder.getVoidTy(), false),
4089 GlobalValue::ExternalLinkage, "fakeFunction", M.get());
4090
4091 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4092 Builder.restoreIP(CodeGenIP);
4093 Builder.CreateCall(FakeFunction, {});
4094 };
4095
4096 // `F` has an argument - an integer, so we use that as the thread limit.
4097 Builder.restoreIP(OMPBuilder.createTeams(/*=*/Builder, BodyGenCB,
4098 /*NumTeamsLower=*/nullptr,
4099 /*NumTeamsUpper=*/nullptr,
4100 /*ThreadLimit=*/F->arg_begin(),
4101 /*IfExpr=*/nullptr));
4102
4103 Builder.CreateRetVoid();
4104 OMPBuilder.finalize();
4105
4106 ASSERT_FALSE(verifyModule(*M));
4107
4108 CallInst *PushNumTeamsCallInst =
4109 findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4110 ASSERT_NE(PushNumTeamsCallInst, nullptr);
4111
4112 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), Builder.getInt32(0));
4113 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), Builder.getInt32(0));
4114 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), &*F->arg_begin());
4115
4116 // Verifying that the next instruction to execute is kmpc_fork_teams
4117 BranchInst *BrInst =
4118 dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
4119 ASSERT_NE(BrInst, nullptr);
4120 ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4121 Instruction *NextInstruction =
4122 BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4123 CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
4124 ASSERT_NE(ForkTeamsCI, nullptr);
4125 EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4126 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4127 }
4128
4129 TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsUpper) {
4130 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4131 OpenMPIRBuilder OMPBuilder(*M);
4132 OMPBuilder.initialize();
4133 F->setName("func");
4134 IRBuilder<> &Builder = OMPBuilder.Builder;
4135 Builder.SetInsertPoint(BB);
4136
4137 Function *FakeFunction =
4138 Function::Create(FunctionType::get(Builder.getVoidTy(), false),
4139 GlobalValue::ExternalLinkage, "fakeFunction", M.get());
4140
4141 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4142 Builder.restoreIP(CodeGenIP);
4143 Builder.CreateCall(FakeFunction, {});
4144 };
4145
4146 // `F` already has an integer argument, so we use that as upper bound to
4147 // `num_teams`
4148 Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB,
4149 /*NumTeamsLower=*/nullptr,
4150 /*NumTeamsUpper=*/F->arg_begin(),
4151 /*ThreadLimit=*/nullptr,
4152 /*IfExpr=*/nullptr));
4153
4154 Builder.CreateRetVoid();
4155 OMPBuilder.finalize();
4156
4157 ASSERT_FALSE(verifyModule(*M));
4158
4159 CallInst *PushNumTeamsCallInst =
4160 findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4161 ASSERT_NE(PushNumTeamsCallInst, nullptr);
4162
4163 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), &*F->arg_begin());
4164 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), &*F->arg_begin());
4165 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));
4166
4167 // Verifying that the next instruction to execute is kmpc_fork_teams
4168 BranchInst *BrInst =
4169 dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
4170 ASSERT_NE(BrInst, nullptr);
4171 ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4172 Instruction *NextInstruction =
4173 BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4174 CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
4175 ASSERT_NE(ForkTeamsCI, nullptr);
4176 EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4177 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4178 }
4179
4180 TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsBoth) {
4181 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4182 OpenMPIRBuilder OMPBuilder(*M);
4183 OMPBuilder.initialize();
4184 F->setName("func");
4185 IRBuilder<> &Builder = OMPBuilder.Builder;
4186 Builder.SetInsertPoint(BB);
4187
4188 Function *FakeFunction =
4189 Function::Create(FunctionType::get(Builder.getVoidTy(), false),
4190 GlobalValue::ExternalLinkage, "fakeFunction", M.get());
4191
4192 Value *NumTeamsLower =
4193 Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5), "numTeamsLower");
4194 Value *NumTeamsUpper =
4195 Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");
4196
4197 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4198 Builder.restoreIP(CodeGenIP);
4199 Builder.CreateCall(FakeFunction, {});
4200 };
4201
4202 // `F` already has an integer argument, so we use that as upper bound to
4203 // `num_teams`
4204 Builder.restoreIP(
4205 OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper,
4206 /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr));
4207
4208 Builder.CreateRetVoid();
4209 OMPBuilder.finalize();
4210
4211 ASSERT_FALSE(verifyModule(*M));
4212
4213 CallInst *PushNumTeamsCallInst =
4214 findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4215 ASSERT_NE(PushNumTeamsCallInst, nullptr);
4216
4217 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);
4218 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);
4219 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), Builder.getInt32(0));
4220
4221 // Verifying that the next instruction to execute is kmpc_fork_teams
4222 BranchInst *BrInst =
4223 dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
4224 ASSERT_NE(BrInst, nullptr);
4225 ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4226 Instruction *NextInstruction =
4227 BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4228 CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
4229 ASSERT_NE(ForkTeamsCI, nullptr);
4230 EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4231 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4232 }
4233
4234 TEST_F(OpenMPIRBuilderTest, CreateTeamsWithNumTeamsAndThreadLimit) {
4235 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4236 OpenMPIRBuilder OMPBuilder(*M);
4237 OMPBuilder.initialize();
4238 F->setName("func");
4239 IRBuilder<> &Builder = OMPBuilder.Builder;
4240 Builder.SetInsertPoint(BB);
4241
4242 BasicBlock *CodegenBB = splitBB(Builder, true);
4243 Builder.SetInsertPoint(CodegenBB);
4244
4245 // Generate values for `num_teams` and `thread_limit` using the first argument
4246 // of the testing function.
4247 Value *NumTeamsLower =
4248 Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5), "numTeamsLower");
4249 Value *NumTeamsUpper =
4250 Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10), "numTeamsUpper");
4251 Value *ThreadLimit =
4252 Builder.CreateAdd(F->arg_begin(), Builder.getInt32(20), "threadLimit");
4253
4254 Function *FakeFunction =
4255 Function::Create(FunctionType::get(Builder.getVoidTy(), false),
4256 GlobalValue::ExternalLinkage, "fakeFunction", M.get());
4257
4258 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4259 Builder.restoreIP(CodeGenIP);
4260 Builder.CreateCall(FakeFunction, {});
4261 };
4262
4263 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4264 Builder.restoreIP(OMPBuilder.createTeams(
4265 Builder, BodyGenCB, NumTeamsLower, NumTeamsUpper, ThreadLimit, nullptr));
4266
4267 Builder.CreateRetVoid();
4268 OMPBuilder.finalize();
4269
4270 ASSERT_FALSE(verifyModule(*M));
4271
4272 CallInst *PushNumTeamsCallInst =
4273 findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4274 ASSERT_NE(PushNumTeamsCallInst, nullptr);
4275
4276 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(2), NumTeamsLower);
4277 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(3), NumTeamsUpper);
4278 EXPECT_EQ(PushNumTeamsCallInst->getArgOperand(4), ThreadLimit);
4279
4280 // Verifying that the next instruction to execute is kmpc_fork_teams
4281 BranchInst *BrInst =
4282 dyn_cast<BranchInst>(PushNumTeamsCallInst->getNextNonDebugInstruction());
4283 ASSERT_NE(BrInst, nullptr);
4284 ASSERT_EQ(BrInst->getNumSuccessors(), 1U);
4285 Instruction *NextInstruction =
4286 BrInst->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4287 CallInst *ForkTeamsCI = dyn_cast_if_present<CallInst>(NextInstruction);
4288 ASSERT_NE(ForkTeamsCI, nullptr);
4289 EXPECT_EQ(ForkTeamsCI->getCalledFunction(),
4290 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams));
4291 }
4292
4293 TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfCondition) {
4294 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4295 OpenMPIRBuilder OMPBuilder(*M);
4296 OMPBuilder.initialize();
4297 F->setName("func");
4298 IRBuilder<> &Builder = OMPBuilder.Builder;
4299 Builder.SetInsertPoint(BB);
4300
4301 Value *IfExpr = Builder.CreateLoad(Builder.getInt1Ty(),
4302 Builder.CreateAlloca(Builder.getInt1Ty()));
4303
4304 Function *FakeFunction =
4305 Function::Create(FunctionType::get(Builder.getVoidTy(), false),
4306 GlobalValue::ExternalLinkage, "fakeFunction", M.get());
4307
4308 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4309 Builder.restoreIP(CodeGenIP);
4310 Builder.CreateCall(FakeFunction, {});
4311 };
4312
4313 // `F` already has an integer argument, so we use that as upper bound to
4314 // `num_teams`
4315 Builder.restoreIP(OMPBuilder.createTeams(
4316 Builder, BodyGenCB, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
4317 /*ThreadLimit=*/nullptr, IfExpr));
4318
4319 Builder.CreateRetVoid();
4320 OMPBuilder.finalize();
4321
4322 ASSERT_FALSE(verifyModule(*M));
4323
4324 CallInst *PushNumTeamsCallInst =
4325 findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4326 ASSERT_NE(PushNumTeamsCallInst, nullptr);
4327 Value *NumTeamsLower = PushNumTeamsCallInst->getArgOperand(2);
4328 Value *NumTeamsUpper = PushNumTeamsCallInst->getArgOperand(3);
4329 Value *ThreadLimit = PushNumTeamsCallInst->getArgOperand(4);
4330
4331 // Check the lower_bound
4332 ASSERT_NE(NumTeamsLower, nullptr);
4333 SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(NumTeamsLower);
4334 ASSERT_NE(NumTeamsLowerSelectInst, nullptr);
4335 EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExpr);
4336 EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), Builder.getInt32(0));
4337 EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));
4338
4339 // Check the upper_bound
4340 ASSERT_NE(NumTeamsUpper, nullptr);
4341 SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(NumTeamsUpper);
4342 ASSERT_NE(NumTeamsUpperSelectInst, nullptr);
4343 EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExpr);
4344 EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), Builder.getInt32(0));
4345 EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));
4346
4347 // Check thread_limit
4348 EXPECT_EQ(ThreadLimit, Builder.getInt32(0));
4349 }
4350
4351 TEST_F(OpenMPIRBuilderTest, CreateTeamsWithIfConditionAndNumTeams) {
4352 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4353 OpenMPIRBuilder OMPBuilder(*M);
4354 OMPBuilder.initialize();
4355 F->setName("func");
4356 IRBuilder<> &Builder = OMPBuilder.Builder;
4357 Builder.SetInsertPoint(BB);
4358
4359 Value *IfExpr = Builder.CreateLoad(
4360 Builder.getInt32Ty(), Builder.CreateAlloca(Builder.getInt32Ty()));
4361 Value *NumTeamsLower = Builder.CreateAdd(F->arg_begin(), Builder.getInt32(5));
4362 Value *NumTeamsUpper =
4363 Builder.CreateAdd(F->arg_begin(), Builder.getInt32(10));
4364 Value *ThreadLimit = Builder.CreateAdd(F->arg_begin(), Builder.getInt32(20));
4365
4366 Function *FakeFunction =
4367 Function::Create(FunctionType::get(Builder.getVoidTy(), false),
4368 GlobalValue::ExternalLinkage, "fakeFunction", M.get());
4369
4370 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4371 Builder.restoreIP(CodeGenIP);
4372 Builder.CreateCall(FakeFunction, {});
4373 };
4374
4375 // `F` already has an integer argument, so we use that as upper bound to
4376 // `num_teams`
4377 Builder.restoreIP(OMPBuilder.createTeams(Builder, BodyGenCB, NumTeamsLower,
4378 NumTeamsUpper, ThreadLimit, IfExpr));
4379
4380 Builder.CreateRetVoid();
4381 OMPBuilder.finalize();
4382
4383 ASSERT_FALSE(verifyModule(*M));
4384
4385 CallInst *PushNumTeamsCallInst =
4386 findSingleCall(F, OMPRTL___kmpc_push_num_teams_51, OMPBuilder);
4387 ASSERT_NE(PushNumTeamsCallInst, nullptr);
4388 Value *NumTeamsLowerArg = PushNumTeamsCallInst->getArgOperand(2);
4389 Value *NumTeamsUpperArg = PushNumTeamsCallInst->getArgOperand(3);
4390 Value *ThreadLimitArg = PushNumTeamsCallInst->getArgOperand(4);
4391
4392 // Get the boolean conversion of if expression
4393 ASSERT_EQ(IfExpr->getNumUses(), 1U);
4394 User *IfExprInst = IfExpr->user_back();
4395 ICmpInst *IfExprCmpInst = dyn_cast<ICmpInst>(IfExprInst);
4396 ASSERT_NE(IfExprCmpInst, nullptr);
4397 EXPECT_EQ(IfExprCmpInst->getPredicate(), ICmpInst::Predicate::ICMP_NE);
4398 EXPECT_EQ(IfExprCmpInst->getOperand(0), IfExpr);
4399 EXPECT_EQ(IfExprCmpInst->getOperand(1), Builder.getInt32(0));
4400
4401 // Check the lower_bound
4402 ASSERT_NE(NumTeamsLowerArg, nullptr);
4403 SelectInst *NumTeamsLowerSelectInst = dyn_cast<SelectInst>(NumTeamsLowerArg);
4404 ASSERT_NE(NumTeamsLowerSelectInst, nullptr);
4405 EXPECT_EQ(NumTeamsLowerSelectInst->getCondition(), IfExprCmpInst);
4406 EXPECT_EQ(NumTeamsLowerSelectInst->getTrueValue(), NumTeamsLower);
4407 EXPECT_EQ(NumTeamsLowerSelectInst->getFalseValue(), Builder.getInt32(1));
4408
4409 // Check the upper_bound
4410 ASSERT_NE(NumTeamsUpperArg, nullptr);
4411 SelectInst *NumTeamsUpperSelectInst = dyn_cast<SelectInst>(NumTeamsUpperArg);
4412 ASSERT_NE(NumTeamsUpperSelectInst, nullptr);
4413 EXPECT_EQ(NumTeamsUpperSelectInst->getCondition(), IfExprCmpInst);
4414 EXPECT_EQ(NumTeamsUpperSelectInst->getTrueValue(), NumTeamsUpper);
4415 EXPECT_EQ(NumTeamsUpperSelectInst->getFalseValue(), Builder.getInt32(1));
4416
4417 // Check thread_limit
4418 EXPECT_EQ(ThreadLimitArg, ThreadLimit);
4419 }
4420
4421 /// Returns the single instruction of InstTy type in BB that uses the value V.
4422 /// If there is more than one such instruction, returns null.
4423 template <typename InstTy>
4424 static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) {
4425 InstTy *Result = nullptr;
4426 for (User *U : V->users()) {
4427 auto *Inst = dyn_cast<InstTy>(U);
4428 if (!Inst || Inst->getParent() != BB)
4429 continue;
4430 if (Result) {
4431 if (auto *SI = dyn_cast<StoreInst>(Inst)) {
4432 if (V == SI->getValueOperand())
4433 continue;
4434 } else {
4435 return nullptr;
4436 }
4437 }
4438 Result = Inst;
4439 }
4440 return Result;
4441 }
4442
4443 /// Returns true if BB contains a simple binary reduction that loads a value
4444 /// from Accum, performs some binary operation with it, and stores it back to
4445 /// Accum.
4446 static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB,
4447 Instruction::BinaryOps *OpCode = nullptr) {
4448 StoreInst *Store = findSingleUserInBlock<StoreInst>(Accum, BB);
4449 if (!Store)
4450 return false;
4451 auto *Stored = dyn_cast<BinaryOperator>(Store->getOperand(0));
4452 if (!Stored)
4453 return false;
4454 if (OpCode && *OpCode != Stored->getOpcode())
4455 return false;
4456 auto *Load = dyn_cast<LoadInst>(Stored->getOperand(0));
4457 return Load && Load->getOperand(0) == Accum;
4458 }
4459
4460 /// Returns true if BB contains a binary reduction that reduces V using a binary
4461 /// operator into an accumulator that is a function argument.
4462 static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) {
4463 auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB);
4464 if (!ReductionOp)
4465 return false;
4466
4467 auto *GlobalLoad = dyn_cast<LoadInst>(ReductionOp->getOperand(0));
4468 if (!GlobalLoad)
4469 return false;
4470
4471 auto *Store = findSingleUserInBlock<StoreInst>(ReductionOp, BB);
4472 if (!Store)
4473 return false;
4474
4475 return Store->getPointerOperand() == GlobalLoad->getPointerOperand() &&
4476 isa<Argument>(findAggregateFromValue(GlobalLoad->getPointerOperand()));
4477 }
4478
4479 /// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and
4480 /// [0, 1], respectively, and assigns results of these instructions to Zero and
4481 /// One. Returns true on success, false on failure or if such instructions are
4482 /// not unique among the users of Ptr.
4483 static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) {
4484 Zero = nullptr;
4485 One = nullptr;
4486 for (User *U : Ptr->users()) {
4487 if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
4488 if (GEP->getNumIndices() != 2)
4489 continue;
4490 auto *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
4491 auto *SecondIdx = dyn_cast<ConstantInt>(GEP->getOperand(2));
4492 EXPECT_NE(FirstIdx, nullptr);
4493 EXPECT_NE(SecondIdx, nullptr);
4494
4495 EXPECT_TRUE(FirstIdx->isZero());
4496 if (SecondIdx->isZero()) {
4497 if (Zero)
4498 return false;
4499 Zero = GEP;
4500 } else if (SecondIdx->isOne()) {
4501 if (One)
4502 return false;
4503 One = GEP;
4504 } else {
4505 return false;
4506 }
4507 }
4508 }
4509 return Zero != nullptr && One != nullptr;
4510 }
4511
4512 static OpenMPIRBuilder::InsertPointTy
4513 sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
4514 Value *&Result) {
4515 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4516 Result = Builder.CreateFAdd(LHS, RHS, "red.add");
4517 return Builder.saveIP();
4518 }
4519
4520 static OpenMPIRBuilder::InsertPointTy
4521 sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
4522 Value *RHS) {
4523 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4524 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
4525 Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, LHS, Partial, std::nullopt,
4526 AtomicOrdering::Monotonic);
4527 return Builder.saveIP();
4528 }
4529
4530 static OpenMPIRBuilder::InsertPointTy
4531 xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
4532 Value *&Result) {
4533 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4534 Result = Builder.CreateXor(LHS, RHS, "red.xor");
4535 return Builder.saveIP();
4536 }
4537
4538 static OpenMPIRBuilder::InsertPointTy
4539 xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
4540 Value *RHS) {
4541 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
4542 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
4543 Builder.CreateAtomicRMW(AtomicRMWInst::Xor, LHS, Partial, std::nullopt,
4544 AtomicOrdering::Monotonic);
4545 return Builder.saveIP();
4546 }
4547
4548 TEST_F(OpenMPIRBuilderTest, CreateReductions) {
4549 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4550 OpenMPIRBuilder OMPBuilder(*M);
4551 OMPBuilder.initialize();
4552 F->setName("func");
4553 IRBuilder<> Builder(BB);
4554
4555 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
4556 Builder.CreateBr(EnterBB);
4557 Builder.SetInsertPoint(EnterBB);
4558 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4559
4560 // Create variables to be reduced.
4561 InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4562 F->getEntryBlock().getFirstInsertionPt());
4563 Type *SumType = Builder.getFloatTy();
4564 Type *XorType = Builder.getInt32Ty();
4565 Value *SumReduced;
4566 Value *XorReduced;
4567 {
4568 IRBuilderBase::InsertPointGuard Guard(Builder);
4569 Builder.restoreIP(OuterAllocaIP);
4570 SumReduced = Builder.CreateAlloca(SumType);
4571 XorReduced = Builder.CreateAlloca(XorType);
4572 }
4573
4574 // Store initial values of reductions into global variables.
4575 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
4576 Builder.CreateStore(Builder.getInt32(1), XorReduced);
4577
4578 // The loop body computes two reductions:
4579 // sum of (float) thread-id;
4580 // xor of thread-id;
4581 // and store the result in global variables.
4582 InsertPointTy BodyIP, BodyAllocaIP;
4583 auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {
4584 IRBuilderBase::InsertPointGuard Guard(Builder);
4585 Builder.restoreIP(CodeGenIP);
4586
4587 uint32_t StrSize;
4588 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4589 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4590 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4591 Value *SumLocal =
4592 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
4593 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
4594 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
4595 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
4596 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
4597 Builder.CreateStore(Sum, SumReduced);
4598 Builder.CreateStore(Xor, XorReduced);
4599
4600 BodyIP = Builder.saveIP();
4601 BodyAllocaIP = InnerAllocaIP;
4602 };
4603
4604 // Privatization for reduction creates local copies of reduction variables and
4605 // initializes them to reduction-neutral values.
4606 Value *SumPrivatized;
4607 Value *XorPrivatized;
4608 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4609 Value &Original, Value &Inner, Value *&ReplVal) {
4610 IRBuilderBase::InsertPointGuard Guard(Builder);
4611 Builder.restoreIP(InnerAllocaIP);
4612 if (&Original == SumReduced) {
4613 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
4614 ReplVal = SumPrivatized;
4615 } else if (&Original == XorReduced) {
4616 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
4617 ReplVal = XorPrivatized;
4618 } else {
4619 ReplVal = &Inner;
4620 return CodeGenIP;
4621 }
4622
4623 Builder.restoreIP(CodeGenIP);
4624 if (&Original == SumReduced)
4625 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4626 SumPrivatized);
4627 else if (&Original == XorReduced)
4628 Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4629
4630 return Builder.saveIP();
4631 };
4632
4633 // Do nothing in finalization.
4634 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4635
4636 InsertPointTy AfterIP =
4637 OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
4638 /* IfCondition */ nullptr,
4639 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4640 /* IsCancellable */ false);
4641 Builder.restoreIP(AfterIP);
4642
4643 OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
4644 {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
4645 {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
4646
4647 OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
4648
4649 Builder.restoreIP(AfterIP);
4650 Builder.CreateRetVoid();
4651
4652 OMPBuilder.finalize(F);
4653
4654 // The IR must be valid.
4655 EXPECT_FALSE(verifyModule(*M));
4656
4657 // Outlining must have happened.
4658 SmallVector<CallInst *> ForkCalls;
4659 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4660 ForkCalls);
4661 ASSERT_EQ(ForkCalls.size(), 1u);
4662 Value *CalleeVal = ForkCalls[0]->getOperand(2);
4663 Function *Outlined = dyn_cast<Function>(CalleeVal);
4664 EXPECT_NE(Outlined, nullptr);
4665
4666 // Check that the lock variable was created with the expected name.
4667 GlobalVariable *LockVar =
4668 M->getGlobalVariable(".gomp_critical_user_.reduction.var");
4669 EXPECT_NE(LockVar, nullptr);
4670
4671 // Find the allocation of a local array that will be used to call the runtime
4672 // reduciton function.
4673 BasicBlock &AllocBlock = Outlined->getEntryBlock();
4674 Value *LocalArray = nullptr;
4675 for (Instruction &I : AllocBlock) {
4676 if (AllocaInst *Alloc = dyn_cast<AllocaInst>(&I)) {
4677 if (!Alloc->getAllocatedType()->isArrayTy() ||
4678 !Alloc->getAllocatedType()->getArrayElementType()->isPointerTy())
4679 continue;
4680 LocalArray = Alloc;
4681 break;
4682 }
4683 }
4684 ASSERT_NE(LocalArray, nullptr);
4685
4686 // Find the call to the runtime reduction function.
4687 BasicBlock *BB = AllocBlock.getUniqueSuccessor();
4688 Value *LocalArrayPtr = nullptr;
4689 Value *ReductionFnVal = nullptr;
4690 Value *SwitchArg = nullptr;
4691 for (Instruction &I : *BB) {
4692 if (CallInst *Call = dyn_cast<CallInst>(&I)) {
4693 if (Call->getCalledFunction() !=
4694 OMPBuilder.getOrCreateRuntimeFunctionPtr(
4695 RuntimeFunction::OMPRTL___kmpc_reduce))
4696 continue;
4697 LocalArrayPtr = Call->getOperand(4);
4698 ReductionFnVal = Call->getOperand(5);
4699 SwitchArg = Call;
4700 break;
4701 }
4702 }
4703
4704 // Check that the local array is passed to the function.
4705 ASSERT_NE(LocalArrayPtr, nullptr);
4706 EXPECT_EQ(LocalArrayPtr, LocalArray);
4707
4708 // Find the GEP instructions preceding stores to the local array.
4709 Value *FirstArrayElemPtr = nullptr;
4710 Value *SecondArrayElemPtr = nullptr;
4711 EXPECT_EQ(LocalArray->getNumUses(), 3u);
4712 ASSERT_TRUE(
4713 findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));
4714
4715 // Check that the values stored into the local array are privatized reduction
4716 // variables.
4717 auto *FirstPrivatized = dyn_cast_or_null<AllocaInst>(
4718 findStoredValue<GetElementPtrInst>(FirstArrayElemPtr));
4719 auto *SecondPrivatized = dyn_cast_or_null<AllocaInst>(
4720 findStoredValue<GetElementPtrInst>(SecondArrayElemPtr));
4721 ASSERT_NE(FirstPrivatized, nullptr);
4722 ASSERT_NE(SecondPrivatized, nullptr);
4723 ASSERT_TRUE(isa<Instruction>(FirstArrayElemPtr));
4724 EXPECT_TRUE(isSimpleBinaryReduction(
4725 FirstPrivatized, cast<Instruction>(FirstArrayElemPtr)->getParent()));
4726 EXPECT_TRUE(isSimpleBinaryReduction(
4727 SecondPrivatized, cast<Instruction>(FirstArrayElemPtr)->getParent()));
4728
4729 // Check that the result of the runtime reduction call is used for further
4730 // dispatch.
4731 ASSERT_EQ(SwitchArg->getNumUses(), 1u);
4732 SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin());
4733 ASSERT_NE(Switch, nullptr);
4734 EXPECT_EQ(Switch->getNumSuccessors(), 3u);
4735 BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor();
4736 BasicBlock *AtomicBB = std::next(Switch->case_begin())->getCaseSuccessor();
4737
4738 // Non-atomic block contains reductions to the global reduction variable,
4739 // which is passed into the outlined function as an argument.
4740 Value *FirstLoad =
4741 findSingleUserInBlock<LoadInst>(FirstPrivatized, NonAtomicBB);
4742 Value *SecondLoad =
4743 findSingleUserInBlock<LoadInst>(SecondPrivatized, NonAtomicBB);
4744 EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB));
4745 EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB));
4746
4747 // Atomic block also constains reductions to the global reduction variable.
4748 FirstLoad = findSingleUserInBlock<LoadInst>(FirstPrivatized, AtomicBB);
4749 SecondLoad = findSingleUserInBlock<LoadInst>(SecondPrivatized, AtomicBB);
4750 auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(FirstLoad, AtomicBB);
4751 auto *SecondAtomic =
4752 findSingleUserInBlock<AtomicRMWInst>(SecondLoad, AtomicBB);
4753 ASSERT_NE(FirstAtomic, nullptr);
4754 Value *AtomicStorePointer = FirstAtomic->getPointerOperand();
4755 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4756 ASSERT_NE(SecondAtomic, nullptr);
4757 AtomicStorePointer = SecondAtomic->getPointerOperand();
4758 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4759
4760 // Check that the separate reduction function also performs (non-atomic)
4761 // reductions after extracting reduction variables from its arguments.
4762 Function *ReductionFn = cast<Function>(ReductionFnVal);
4763 BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock();
4764 Value *FirstLHSPtr;
4765 Value *SecondLHSPtr;
4766 ASSERT_TRUE(
4767 findGEPZeroOne(ReductionFn->getArg(0), FirstLHSPtr, SecondLHSPtr));
4768 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4769 ASSERT_NE(Opaque, nullptr);
4770 EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB));
4771 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4772 ASSERT_NE(Opaque, nullptr);
4773 EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB));
4774
4775 Value *FirstRHS;
4776 Value *SecondRHS;
4777 EXPECT_TRUE(findGEPZeroOne(ReductionFn->getArg(1), FirstRHS, SecondRHS));
4778 }
4779
4780 TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
4781 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4782 OpenMPIRBuilder OMPBuilder(*M);
4783 OMPBuilder.initialize();
4784 F->setName("func");
4785 IRBuilder<> Builder(BB);
4786
4787 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
4788 Builder.CreateBr(EnterBB);
4789 Builder.SetInsertPoint(EnterBB);
4790 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4791
4792 // Create variables to be reduced.
4793 InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4794 F->getEntryBlock().getFirstInsertionPt());
4795 Type *SumType = Builder.getFloatTy();
4796 Type *XorType = Builder.getInt32Ty();
4797 Value *SumReduced;
4798 Value *XorReduced;
4799 {
4800 IRBuilderBase::InsertPointGuard Guard(Builder);
4801 Builder.restoreIP(OuterAllocaIP);
4802 SumReduced = Builder.CreateAlloca(SumType);
4803 XorReduced = Builder.CreateAlloca(XorType);
4804 }
4805
4806 // Store initial values of reductions into global variables.
4807 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
4808 Builder.CreateStore(Builder.getInt32(1), XorReduced);
4809
4810 InsertPointTy FirstBodyIP, FirstBodyAllocaIP;
4811 auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4812 InsertPointTy CodeGenIP) {
4813 IRBuilderBase::InsertPointGuard Guard(Builder);
4814 Builder.restoreIP(CodeGenIP);
4815
4816 uint32_t StrSize;
4817 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4818 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4819 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4820 Value *SumLocal =
4821 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
4822 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
4823 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
4824 Builder.CreateStore(Sum, SumReduced);
4825
4826 FirstBodyIP = Builder.saveIP();
4827 FirstBodyAllocaIP = InnerAllocaIP;
4828 };
4829
4830 InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
4831 auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4832 InsertPointTy CodeGenIP) {
4833 IRBuilderBase::InsertPointGuard Guard(Builder);
4834 Builder.restoreIP(CodeGenIP);
4835
4836 uint32_t StrSize;
4837 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4838 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4839 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4840 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
4841 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
4842 Builder.CreateStore(Xor, XorReduced);
4843
4844 SecondBodyIP = Builder.saveIP();
4845 SecondBodyAllocaIP = InnerAllocaIP;
4846 };
4847
4848 // Privatization for reduction creates local copies of reduction variables and
4849 // initializes them to reduction-neutral values. The same privatization
4850 // callback is used for both loops, with dispatch based on the value being
4851 // privatized.
4852 Value *SumPrivatized;
4853 Value *XorPrivatized;
4854 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4855 Value &Original, Value &Inner, Value *&ReplVal) {
4856 IRBuilderBase::InsertPointGuard Guard(Builder);
4857 Builder.restoreIP(InnerAllocaIP);
4858 if (&Original == SumReduced) {
4859 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
4860 ReplVal = SumPrivatized;
4861 } else if (&Original == XorReduced) {
4862 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
4863 ReplVal = XorPrivatized;
4864 } else {
4865 ReplVal = &Inner;
4866 return CodeGenIP;
4867 }
4868
4869 Builder.restoreIP(CodeGenIP);
4870 if (&Original == SumReduced)
4871 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4872 SumPrivatized);
4873 else if (&Original == XorReduced)
4874 Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4875
4876 return Builder.saveIP();
4877 };
4878
4879 // Do nothing in finalization.
4880 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4881
4882 Builder.restoreIP(
4883 OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
4884 FiniCB, /* IfCondition */ nullptr,
4885 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4886 /* IsCancellable */ false));
4887 InsertPointTy AfterIP = OMPBuilder.createParallel(
4888 {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB,
4889 /* IfCondition */ nullptr,
4890 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4891 /* IsCancellable */ false);
4892
4893 OMPBuilder.createReductions(
4894 FirstBodyIP, FirstBodyAllocaIP,
4895 {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
4896 OMPBuilder.createReductions(
4897 SecondBodyIP, SecondBodyAllocaIP,
4898 {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
4899
4900 Builder.restoreIP(AfterIP);
4901 Builder.CreateRetVoid();
4902
4903 OMPBuilder.finalize(F);
4904
4905 // The IR must be valid.
4906 EXPECT_FALSE(verifyModule(*M));
4907
4908 // Two different outlined functions must have been created.
4909 SmallVector<CallInst *> ForkCalls;
4910 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4911 ForkCalls);
4912 ASSERT_EQ(ForkCalls.size(), 2u);
4913 Value *CalleeVal = ForkCalls[0]->getOperand(2);
4914 Function *FirstCallee = cast<Function>(CalleeVal);
4915 CalleeVal = ForkCalls[1]->getOperand(2);
4916 Function *SecondCallee = cast<Function>(CalleeVal);
4917 EXPECT_NE(FirstCallee, SecondCallee);
4918
4919 // Two different reduction functions must have been created.
4920 SmallVector<CallInst *> ReduceCalls;
4921 findCalls(FirstCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder,
4922 ReduceCalls);
4923 ASSERT_EQ(ReduceCalls.size(), 1u);
4924 auto *AddReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4925 ReduceCalls.clear();
4926 findCalls(SecondCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce,
4927 OMPBuilder, ReduceCalls);
4928 auto *XorReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4929 EXPECT_NE(AddReduction, XorReduction);
4930
4931 // Each reduction function does its own kind of reduction.
4932 BasicBlock *FnReductionBB = &AddReduction->getEntryBlock();
4933 Value *FirstLHSPtr = findSingleUserInBlock<GetElementPtrInst>(
4934 AddReduction->getArg(0), FnReductionBB);
4935 ASSERT_NE(FirstLHSPtr, nullptr);
4936 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4937 ASSERT_NE(Opaque, nullptr);
4938 Instruction::BinaryOps Opcode = Instruction::FAdd;
4939 EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB, &Opcode));
4940
4941 FnReductionBB = &XorReduction->getEntryBlock();
4942 Value *SecondLHSPtr = findSingleUserInBlock<GetElementPtrInst>(
4943 XorReduction->getArg(0), FnReductionBB);
4944 ASSERT_NE(FirstLHSPtr, nullptr);
4945 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4946 ASSERT_NE(Opaque, nullptr);
4947 Opcode = Instruction::Xor;
4948 EXPECT_TRUE(isSimpleBinaryReduction(Opaque, FnReductionBB, &Opcode));
4949 }
4950
4951 TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
4952 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4953 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4954 OpenMPIRBuilder OMPBuilder(*M);
4955 OMPBuilder.initialize();
4956 F->setName("func");
4957 IRBuilder<> Builder(BB);
4958
4959 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4960 Builder.CreateBr(EnterBB);
4961 Builder.SetInsertPoint(EnterBB);
4962 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4963
4964 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4965 llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4966
4967 auto FiniCB = [&](InsertPointTy IP) {};
4968 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4969 SectionCBVector.push_back(SectionCB);
4970
4971 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4972 llvm::Value &, llvm::Value &Val,
4973 llvm::Value *&ReplVal) { return CodeGenIP; };
4974 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4975 F->getEntryBlock().getFirstInsertionPt());
4976 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4977 PrivCB, FiniCB, false, false));
4978 Builder.CreateRetVoid(); // Required at the end of the function
4979 EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);
4980 EXPECT_FALSE(verifyModule(*M, &errs()));
4981 }
4982
4983 TEST_F(OpenMPIRBuilderTest, CreateSections) {
4984 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4985 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4986 OpenMPIRBuilder OMPBuilder(*M);
4987 OMPBuilder.initialize();
4988 F->setName("func");
4989 IRBuilder<> Builder(BB);
4990
4991 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4992 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4993 llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4994
4995 BasicBlock *SwitchBB = nullptr;
4996 AllocaInst *PrivAI = nullptr;
4997 SwitchInst *Switch = nullptr;
4998
4999 unsigned NumBodiesGenerated = 0;
5000 unsigned NumFiniCBCalls = 0;
5001 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
5002
5003 auto FiniCB = [&](InsertPointTy IP) {
5004 ++NumFiniCBCalls;
5005 BasicBlock *IPBB = IP.getBlock();
5006 EXPECT_NE(IPBB->end(), IP.getPoint());
5007 };
5008
5009 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5010 ++NumBodiesGenerated;
5011 CaseBBs.push_back(CodeGenIP.getBlock());
5012 SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();
5013 Builder.restoreIP(CodeGenIP);
5014 Builder.CreateStore(F->arg_begin(), PrivAI);
5015 Value *PrivLoad =
5016 Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca");
5017 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
5018 };
5019 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5020 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
5021 // TODO: Privatization not implemented yet
5022 return CodeGenIP;
5023 };
5024
5025 SectionCBVector.push_back(SectionCB);
5026 SectionCBVector.push_back(SectionCB);
5027
5028 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5029 F->getEntryBlock().getFirstInsertionPt());
5030 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
5031 PrivCB, FiniCB, false, false));
5032 Builder.CreateRetVoid(); // Required at the end of the function
5033
5034 // Switch BB's predecessor is loop condition BB, whose successor at index 1 is
5035 // loop's exit BB
5036 BasicBlock *ForExitBB =
5037 SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(1);
5038 EXPECT_NE(ForExitBB, nullptr);
5039
5040 EXPECT_NE(PrivAI, nullptr);
5041 Function *OutlinedFn = PrivAI->getFunction();
5042 EXPECT_EQ(F, OutlinedFn);
5043 EXPECT_FALSE(verifyModule(*M, &errs()));
5044 EXPECT_EQ(OutlinedFn->arg_size(), 1U);
5045
5046 BasicBlock *LoopPreheaderBB =
5047 OutlinedFn->getEntryBlock().getSingleSuccessor();
5048 // loop variables are 5 - lower bound, upper bound, stride, islastiter, and
5049 // iterator/counter
5050 bool FoundForInit = false;
5051 for (Instruction &Inst : *LoopPreheaderBB) {
5052 if (isa<CallInst>(Inst)) {
5053 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
5054 "__kmpc_for_static_init_4u") {
5055 FoundForInit = true;
5056 }
5057 }
5058 }
5059 EXPECT_EQ(FoundForInit, true);
5060
5061 bool FoundForExit = false;
5062 bool FoundBarrier = false;
5063 for (Instruction &Inst : *ForExitBB) {
5064 if (isa<CallInst>(Inst)) {
5065 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
5066 "__kmpc_for_static_fini") {
5067 FoundForExit = true;
5068 }
5069 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
5070 "__kmpc_barrier") {
5071 FoundBarrier = true;
5072 }
5073 if (FoundForExit && FoundBarrier)
5074 break;
5075 }
5076 }
5077 EXPECT_EQ(FoundForExit, true);
5078 EXPECT_EQ(FoundBarrier, true);
5079
5080 EXPECT_NE(SwitchBB, nullptr);
5081 EXPECT_NE(SwitchBB->getTerminator(), nullptr);
5082 EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true);
5083 Switch = cast<SwitchInst>(SwitchBB->getTerminator());
5084 EXPECT_EQ(Switch->getNumCases(), 2U);
5085
5086 EXPECT_EQ(CaseBBs.size(), 2U);
5087 for (auto *&CaseBB : CaseBBs) {
5088 EXPECT_EQ(CaseBB->getParent(), OutlinedFn);
5089 }
5090
5091 ASSERT_EQ(NumBodiesGenerated, 2U);
5092 ASSERT_EQ(NumFiniCBCalls, 1U);
5093 EXPECT_FALSE(verifyModule(*M, &errs()));
5094 }
5095
5096 TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {
5097 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5098 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
5099 OpenMPIRBuilder OMPBuilder(*M);
5100 OMPBuilder.initialize();
5101 F->setName("func");
5102 IRBuilder<> Builder(BB);
5103
5104 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
5105 Builder.CreateBr(EnterBB);
5106 Builder.SetInsertPoint(EnterBB);
5107 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5108
5109 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5110 F->getEntryBlock().getFirstInsertionPt());
5111 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
5112 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
5113 llvm::Value &, llvm::Value &Val,
5114 llvm::Value *&ReplVal) { return CodeGenIP; };
5115 auto FiniCB = [&](InsertPointTy IP) {};
5116
5117 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
5118 PrivCB, FiniCB, false, true));
5119 Builder.CreateRetVoid(); // Required at the end of the function
5120 for (auto &Inst : instructions(*F)) {
5121 EXPECT_FALSE(isa<CallInst>(Inst) &&
5122 cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
5123 "__kmpc_barrier" &&
5124 "call to function __kmpc_barrier found with nowait");
5125 }
5126 }
5127
5128 TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) {
5129 OpenMPIRBuilder OMPBuilder(*M);
5130 OMPBuilder.initialize();
5131
5132 IRBuilder<> Builder(BB);
5133
5134 SmallVector<uint64_t> Mappings = {0, 1};
5135 GlobalVariable *OffloadMaptypesGlobal =
5136 OMPBuilder.createOffloadMaptypes(Mappings, "offload_maptypes");
5137 EXPECT_FALSE(M->global_empty());
5138 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes");
5139 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
5140 EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
5141 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
5142 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
5143 Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
5144 EXPECT_TRUE(isa<ConstantDataArray>(Initializer));
5145 ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Initializer);
5146 EXPECT_EQ(MappingInit->getNumElements(), Mappings.size());
5147 EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64));
5148 Constant *CA = ConstantDataArray::get(Builder.getContext(), Mappings);
5149 EXPECT_EQ(MappingInit, CA);
5150 }
5151
5152 TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) {
5153 OpenMPIRBuilder OMPBuilder(*M);
5154 OMPBuilder.initialize();
5155
5156 IRBuilder<> Builder(BB);
5157
5158 uint32_t StrSize;
5159 Constant *Cst1 =
5160 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
5161 Constant *Cst2 =
5162 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
5163 SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
5164
5165 GlobalVariable *OffloadMaptypesGlobal =
5166 OMPBuilder.createOffloadMapnames(Names, "offload_mapnames");
5167 EXPECT_FALSE(M->global_empty());
5168 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames");
5169 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
5170 EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
5171 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
5172 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
5173 Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
5174 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts()));
5175 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts()));
5176
5177 GlobalVariable *Name1Gbl =
5178 cast<GlobalVariable>(Initializer->getOperand(0)->stripPointerCasts());
5179 EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer()));
5180 ConstantDataArray *Name1GblCA =
5181 dyn_cast<ConstantDataArray>(Name1Gbl->getInitializer());
5182 EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;");
5183
5184 GlobalVariable *Name2Gbl =
5185 cast<GlobalVariable>(Initializer->getOperand(1)->stripPointerCasts());
5186 EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer()));
5187 ConstantDataArray *Name2GblCA =
5188 dyn_cast<ConstantDataArray>(Name2Gbl->getInitializer());
5189 EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;");
5190
5191 EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy());
5192 EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size());
5193 }
5194
5195 TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) {
5196 OpenMPIRBuilder OMPBuilder(*M);
5197 OMPBuilder.initialize();
5198 F->setName("func");
5199 IRBuilder<> Builder(BB);
5200
5201 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5202
5203 unsigned TotalNbOperand = 2;
5204
5205 OpenMPIRBuilder::MapperAllocas MapperAllocas;
5206 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5207 F->getEntryBlock().getFirstInsertionPt());
5208 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
5209 EXPECT_NE(MapperAllocas.ArgsBase, nullptr);
5210 EXPECT_NE(MapperAllocas.Args, nullptr);
5211 EXPECT_NE(MapperAllocas.ArgSizes, nullptr);
5212 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy());
5213 ArrayType *ArrType =
5214 dyn_cast<ArrayType>(MapperAllocas.ArgsBase->getAllocatedType());
5215 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
5216 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()
5217 ->getArrayElementType()
5218 ->isPointerTy());
5219
5220 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy());
5221 ArrType = dyn_cast<ArrayType>(MapperAllocas.Args->getAllocatedType());
5222 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
5223 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()
5224 ->getArrayElementType()
5225 ->isPointerTy());
5226
5227 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy());
5228 ArrType = dyn_cast<ArrayType>(MapperAllocas.ArgSizes->getAllocatedType());
5229 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
5230 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()
5231 ->getArrayElementType()
5232 ->isIntegerTy(64));
5233 }
5234
5235 TEST_F(OpenMPIRBuilderTest, EmitMapperCall) {
5236 OpenMPIRBuilder OMPBuilder(*M);
5237 OMPBuilder.initialize();
5238 F->setName("func");
5239 IRBuilder<> Builder(BB);
5240 LLVMContext &Ctx = M->getContext();
5241
5242 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5243
5244 unsigned TotalNbOperand = 2;
5245
5246 OpenMPIRBuilder::MapperAllocas MapperAllocas;
5247 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5248 F->getEntryBlock().getFirstInsertionPt());
5249 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
5250
5251 auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
5252 omp::OMPRTL___tgt_target_data_begin_mapper);
5253
5254 SmallVector<uint64_t> Flags = {0, 2};
5255
5256 uint32_t StrSize;
5257 Constant *SrcLocCst =
5258 OMPBuilder.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize);
5259 Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocCst, StrSize);
5260
5261 Constant *Cst1 =
5262 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
5263 Constant *Cst2 =
5264 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
5265 SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
5266
5267 GlobalVariable *Maptypes =
5268 OMPBuilder.createOffloadMaptypes(Flags, ".offload_maptypes");
5269 Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32(
5270 ArrayType::get(Type::getInt64Ty(Ctx), TotalNbOperand), Maptypes,
5271 /*Idx0=*/0, /*Idx1=*/0);
5272
5273 GlobalVariable *Mapnames =
5274 OMPBuilder.createOffloadMapnames(Names, ".offload_mapnames");
5275 Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32(
5276 ArrayType::get(Type::getInt8PtrTy(Ctx), TotalNbOperand), Mapnames,
5277 /*Idx0=*/0, /*Idx1=*/0);
5278
5279 OMPBuilder.emitMapperCall(Builder.saveIP(), BeginMapperFunc, SrcLocInfo,
5280 MaptypesArg, MapnamesArg, MapperAllocas, -1,
5281 TotalNbOperand);
5282
5283 CallInst *MapperCall = dyn_cast<CallInst>(&BB->back());
5284 EXPECT_NE(MapperCall, nullptr);
5285 EXPECT_EQ(MapperCall->arg_size(), 9U);
5286 EXPECT_EQ(MapperCall->getCalledFunction()->getName(),
5287 "__tgt_target_data_begin_mapper");
5288 EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo);
5289 EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64));
5290 EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32));
5291
5292 EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg);
5293 EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg);
5294 EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy());
5295 }
5296
5297 TEST_F(OpenMPIRBuilderTest, TargetEnterData) {
5298 OpenMPIRBuilder OMPBuilder(*M);
5299 OMPBuilder.initialize();
5300 F->setName("func");
5301 IRBuilder<> Builder(BB);
5302 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5303
5304 int64_t DeviceID = 2;
5305
5306 AllocaInst *Val1 =
5307 Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));
5308 ASSERT_NE(Val1, nullptr);
5309
5310 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5311 F->getEntryBlock().getFirstInsertionPt());
5312
5313 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
5314 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5315 auto GenMapInfoCB =
5316 [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
5317 // Get map clause information.
5318 Builder.restoreIP(codeGenIP);
5319
5320 CombinedInfo.BasePointers.emplace_back(Val1);
5321 CombinedInfo.Pointers.emplace_back(Val1);
5322 CombinedInfo.DevicePointers.emplace_back(
5323 llvm::OpenMPIRBuilder::DeviceInfoTy::None);
5324 CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));
5325 CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(1));
5326 uint32_t temp;
5327 CombinedInfo.Names.emplace_back(
5328 OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
5329 return CombinedInfo;
5330 };
5331
5332 llvm::OpenMPIRBuilder::TargetDataInfo Info(
5333 /*RequiresDevicePointerInfo=*/false,
5334 /*SeparateBeginEndCalls=*/true);
5335
5336 OMPBuilder.Config.setIsGPU(true);
5337
5338 llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_begin_mapper;
5339 Builder.restoreIP(OMPBuilder.createTargetData(
5340 Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
5341 /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc));
5342
5343 CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
5344 EXPECT_NE(TargetDataCall, nullptr);
5345 EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5346 EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5347 "__tgt_target_data_begin_mapper");
5348 EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5349 EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5350 EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5351
5352 Builder.CreateRetVoid();
5353 EXPECT_FALSE(verifyModule(*M, &errs()));
5354 }
5355
5356 TEST_F(OpenMPIRBuilderTest, TargetExitData) {
5357 OpenMPIRBuilder OMPBuilder(*M);
5358 OMPBuilder.initialize();
5359 F->setName("func");
5360 IRBuilder<> Builder(BB);
5361 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5362
5363 int64_t DeviceID = 2;
5364
5365 AllocaInst *Val1 =
5366 Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));
5367 ASSERT_NE(Val1, nullptr);
5368
5369 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5370 F->getEntryBlock().getFirstInsertionPt());
5371
5372 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
5373 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5374 auto GenMapInfoCB =
5375 [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
5376 // Get map clause information.
5377 Builder.restoreIP(codeGenIP);
5378
5379 CombinedInfo.BasePointers.emplace_back(Val1);
5380 CombinedInfo.Pointers.emplace_back(Val1);
5381 CombinedInfo.DevicePointers.emplace_back(
5382 llvm::OpenMPIRBuilder::DeviceInfoTy::None);
5383 CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));
5384 CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(2));
5385 uint32_t temp;
5386 CombinedInfo.Names.emplace_back(
5387 OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
5388 return CombinedInfo;
5389 };
5390
5391 llvm::OpenMPIRBuilder::TargetDataInfo Info(
5392 /*RequiresDevicePointerInfo=*/false,
5393 /*SeparateBeginEndCalls=*/true);
5394
5395 OMPBuilder.Config.setIsGPU(true);
5396
5397 llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_end_mapper;
5398 Builder.restoreIP(OMPBuilder.createTargetData(
5399 Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
5400 /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc));
5401
5402 CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
5403 EXPECT_NE(TargetDataCall, nullptr);
5404 EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5405 EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5406 "__tgt_target_data_end_mapper");
5407 EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5408 EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5409 EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5410
5411 Builder.CreateRetVoid();
5412 EXPECT_FALSE(verifyModule(*M, &errs()));
5413 }
5414
5415 TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
5416 OpenMPIRBuilder OMPBuilder(*M);
5417 OMPBuilder.initialize();
5418 F->setName("func");
5419 IRBuilder<> Builder(BB);
5420 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5421
5422 int64_t DeviceID = 2;
5423
5424 AllocaInst *Val1 =
5425 Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));
5426 ASSERT_NE(Val1, nullptr);
5427
5428 AllocaInst *Val2 = Builder.CreateAlloca(Builder.getPtrTy());
5429 ASSERT_NE(Val2, nullptr);
5430
5431 AllocaInst *Val3 = Builder.CreateAlloca(Builder.getPtrTy());
5432 ASSERT_NE(Val3, nullptr);
5433
5434 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
5435 F->getEntryBlock().getFirstInsertionPt());
5436
5437 using DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;
5438 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
5439 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5440 auto GenMapInfoCB =
5441 [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
5442 // Get map clause information.
5443 Builder.restoreIP(codeGenIP);
5444 uint32_t temp;
5445
5446 CombinedInfo.BasePointers.emplace_back(Val1);
5447 CombinedInfo.Pointers.emplace_back(Val1);
5448 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::None);
5449 CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));
5450 CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(3));
5451 CombinedInfo.Names.emplace_back(
5452 OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
5453
5454 CombinedInfo.BasePointers.emplace_back(Val2);
5455 CombinedInfo.Pointers.emplace_back(Val2);
5456 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);
5457 CombinedInfo.Sizes.emplace_back(Builder.getInt64(8));
5458 CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(67));
5459 CombinedInfo.Names.emplace_back(
5460 OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
5461
5462 CombinedInfo.BasePointers.emplace_back(Val3);
5463 CombinedInfo.Pointers.emplace_back(Val3);
5464 CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Address);
5465 CombinedInfo.Sizes.emplace_back(Builder.getInt64(8));
5466 CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(67));
5467 CombinedInfo.Names.emplace_back(
5468 OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
5469 return CombinedInfo;
5470 };
5471
5472 llvm::OpenMPIRBuilder::TargetDataInfo Info(
5473 /*RequiresDevicePointerInfo=*/true,
5474 /*SeparateBeginEndCalls=*/true);
5475
5476 OMPBuilder.Config.setIsGPU(true);
5477
5478 using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
5479 auto BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {
5480 if (BodyGenType == BodyGenTy::Priv) {
5481 EXPECT_EQ(Info.DevicePtrInfoMap.size(), 2u);
5482 Builder.restoreIP(CodeGenIP);
5483 CallInst *TargetDataCall =
5484 dyn_cast<CallInst>(BB->back().getPrevNode()->getPrevNode());
5485 EXPECT_NE(TargetDataCall, nullptr);
5486 EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5487 EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5488 "__tgt_target_data_begin_mapper");
5489 EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5490 EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5491 EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5492
5493 LoadInst *LI = dyn_cast<LoadInst>(BB->back().getPrevNode());
5494 EXPECT_NE(LI, nullptr);
5495 StoreInst *SI = dyn_cast<StoreInst>(&BB->back());
5496 EXPECT_NE(SI, nullptr);
5497 EXPECT_EQ(SI->getValueOperand(), LI);
5498 EXPECT_EQ(SI->getPointerOperand(), Info.DevicePtrInfoMap[Val2].second);
5499 EXPECT_TRUE(isa<AllocaInst>(Info.DevicePtrInfoMap[Val2].second));
5500 EXPECT_TRUE(isa<GetElementPtrInst>(Info.DevicePtrInfoMap[Val3].second));
5501 Builder.CreateStore(Builder.getInt32(99), Val1);
5502 }
5503 return Builder.saveIP();
5504 };
5505
5506 Builder.restoreIP(OMPBuilder.createTargetData(
5507 Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
5508 /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyCB));
5509
5510 CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
5511 EXPECT_NE(TargetDataCall, nullptr);
5512 EXPECT_EQ(TargetDataCall->arg_size(), 9U);
5513 EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
5514 "__tgt_target_data_end_mapper");
5515 EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
5516 EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
5517 EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
5518
5519 Builder.CreateRetVoid();
5520 EXPECT_FALSE(verifyModule(*M, &errs()));
5521 }
5522
5523 namespace {
5524 // Some basic handling of argument mapping for the moment
5525 void CreateDefaultMapInfos(llvm::OpenMPIRBuilder &OmpBuilder,
5526 llvm::SmallVectorImpl<llvm::Value *> &Args,
5527 llvm::OpenMPIRBuilder::MapInfosTy &CombinedInfo) {
5528 for (auto Arg : Args) {
5529 CombinedInfo.BasePointers.emplace_back(Arg);
5530 CombinedInfo.Pointers.emplace_back(Arg);
5531 uint32_t SrcLocStrSize;
5532 CombinedInfo.Names.emplace_back(OmpBuilder.getOrCreateSrcLocStr(
5533 "Unknown loc - stub implementation", SrcLocStrSize));
5534 CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(
5535 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO |
5536 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM |
5537 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM));
5538 CombinedInfo.Sizes.emplace_back(OmpBuilder.Builder.getInt64(
5539 OmpBuilder.M.getDataLayout().getTypeAllocSize(Arg->getType())));
5540 }
5541 }
5542 } // namespace
5543
5544 TEST_F(OpenMPIRBuilderTest, TargetRegion) {
5545 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5546 OpenMPIRBuilder OMPBuilder(*M);
5547 OMPBuilder.initialize();
5548 OpenMPIRBuilderConfig Config(false, false, false, false, false, false, false);
5549 OMPBuilder.setConfig(Config);
5550 F->setName("func");
5551 IRBuilder<> Builder(BB);
5552 auto Int32Ty = Builder.getInt32Ty();
5553
5554 AllocaInst *APtr = Builder.CreateAlloca(Int32Ty, nullptr, "a_ptr");
5555 AllocaInst *BPtr = Builder.CreateAlloca(Int32Ty, nullptr, "b_ptr");
5556 AllocaInst *CPtr = Builder.CreateAlloca(Int32Ty, nullptr, "c_ptr");
5557
5558 Builder.CreateStore(Builder.getInt32(10), APtr);
5559 Builder.CreateStore(Builder.getInt32(20), BPtr);
5560 auto BodyGenCB = [&](InsertPointTy AllocaIP,
5561 InsertPointTy CodeGenIP) -> InsertPointTy {
5562 Builder.restoreIP(CodeGenIP);
5563 LoadInst *AVal = Builder.CreateLoad(Int32Ty, APtr);
5564 LoadInst *BVal = Builder.CreateLoad(Int32Ty, BPtr);
5565 Value *Sum = Builder.CreateAdd(AVal, BVal);
5566 Builder.CreateStore(Sum, CPtr);
5567 return Builder.saveIP();
5568 };
5569
5570 llvm::SmallVector<llvm::Value *> Inputs;
5571 Inputs.push_back(APtr);
5572 Inputs.push_back(BPtr);
5573 Inputs.push_back(CPtr);
5574
5575 auto SimpleArgAccessorCB =
5576 [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
5577 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
5578 llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
5579 if (!OMPBuilder.Config.isTargetDevice()) {
5580 RetVal = cast<llvm::Value>(&Arg);
5581 return CodeGenIP;
5582 }
5583
5584 Builder.restoreIP(AllocaIP);
5585
5586 llvm::Value *Addr = Builder.CreateAlloca(
5587 Arg.getType()->isPointerTy()
5588 ? Arg.getType()
5589 : Type::getInt64Ty(Builder.getContext()),
5590 OMPBuilder.M.getDataLayout().getAllocaAddrSpace());
5591 llvm::Value *AddrAscast =
5592 Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Input->getType());
5593 Builder.CreateStore(&Arg, AddrAscast);
5594
5595 Builder.restoreIP(CodeGenIP);
5596
5597 RetVal = Builder.CreateLoad(Arg.getType(), AddrAscast);
5598
5599 return Builder.saveIP();
5600 };
5601
5602 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;
5603 auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
5604 -> llvm::OpenMPIRBuilder::MapInfosTy & {
5605 CreateDefaultMapInfos(OMPBuilder, Inputs, CombinedInfos);
5606 return CombinedInfos;
5607 };
5608
5609 TargetRegionEntryInfo EntryInfo("func", 42, 4711, 17);
5610 OpenMPIRBuilder::LocationDescription OmpLoc({Builder.saveIP(), DL});
5611 Builder.restoreIP(OMPBuilder.createTarget(
5612 OmpLoc, Builder.saveIP(), Builder.saveIP(), EntryInfo, -1, 0, Inputs,
5613 GenMapInfoCB, BodyGenCB, SimpleArgAccessorCB));
5614 OMPBuilder.finalize();
5615 Builder.CreateRetVoid();
5616
5617 // Check the kernel launch sequence
5618 auto Iter = F->getEntryBlock().rbegin();
5619 EXPECT_TRUE(isa<BranchInst>(&*(Iter)));
5620 BranchInst *Branch = dyn_cast<BranchInst>(&*(Iter));
5621 EXPECT_TRUE(isa<CmpInst>(&*(++Iter)));
5622 EXPECT_TRUE(isa<CallInst>(&*(++Iter)));
5623 CallInst *Call = dyn_cast<CallInst>(&*(Iter));
5624
5625 // Check that the kernel launch function is called
5626 Function *KernelLaunchFunc = Call->getCalledFunction();
5627 EXPECT_NE(KernelLaunchFunc, nullptr);
5628 StringRef FunctionName = KernelLaunchFunc->getName();
5629 EXPECT_TRUE(FunctionName.startswith("__tgt_target_kernel"));
5630
5631 // Check the fallback call
5632 BasicBlock *FallbackBlock = Branch->getSuccessor(0);
5633 Iter = FallbackBlock->rbegin();
5634 CallInst *FCall = dyn_cast<CallInst>(&*(++Iter));
5635 EXPECT_NE(FCall, nullptr);
5636
5637 // Check that the correct aguments are passed in
5638 for (auto ArgInput : zip(FCall->args(), Inputs)) {
5639 EXPECT_EQ(std::get<0>(ArgInput), std::get<1>(ArgInput));
5640 }
5641
5642 // Check that the outlined function exists with the expected prefix
5643 Function *OutlinedFunc = FCall->getCalledFunction();
5644 EXPECT_NE(OutlinedFunc, nullptr);
5645 StringRef FunctionName2 = OutlinedFunc->getName();
5646 EXPECT_TRUE(FunctionName2.startswith("__omp_offloading"));
5647
5648 EXPECT_FALSE(verifyModule(*M, &errs()));
5649 }
5650
5651 TEST_F(OpenMPIRBuilderTest, TargetRegionDevice) {
5652 OpenMPIRBuilder OMPBuilder(*M);
5653 OMPBuilder.setConfig(
5654 OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
5655 OMPBuilder.initialize();
5656
5657 F->setName("func");
5658 IRBuilder<> Builder(BB);
5659 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
5660
5661 LoadInst *Value = nullptr;
5662 StoreInst *TargetStore = nullptr;
5663 llvm::SmallVector<llvm::Value *, 2> CapturedArgs = {
5664 Constant::getNullValue(PointerType::get(Ctx, 0)),
5665 Constant::getNullValue(PointerType::get(Ctx, 0))};
5666
5667 auto SimpleArgAccessorCB =
5668 [&](llvm::Argument &Arg, llvm::Value *Input, llvm::Value *&RetVal,
5669 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP,
5670 llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP) {
5671 if (!OMPBuilder.Config.isTargetDevice()) {
5672 RetVal = cast<llvm::Value>(&Arg);
5673 return CodeGenIP;
5674 }
5675
5676 Builder.restoreIP(AllocaIP);
5677
5678 llvm::Value *Addr = Builder.CreateAlloca(
5679 Arg.getType()->isPointerTy()
5680 ? Arg.getType()
5681 : Type::getInt64Ty(Builder.getContext()),
5682 OMPBuilder.M.getDataLayout().getAllocaAddrSpace());
5683 llvm::Value *AddrAscast =
5684 Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Input->getType());
5685 Builder.CreateStore(&Arg, AddrAscast);
5686
5687 Builder.restoreIP(CodeGenIP);
5688
5689 RetVal = Builder.CreateLoad(Arg.getType(), AddrAscast);
5690
5691 return Builder.saveIP();
5692 };
5693
5694 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos;
5695 auto GenMapInfoCB = [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP)
5696 -> llvm::OpenMPIRBuilder::MapInfosTy & {
5697 CreateDefaultMapInfos(OMPBuilder, CapturedArgs, CombinedInfos);
5698 return CombinedInfos;
5699 };
5700
5701 auto BodyGenCB = [&](OpenMPIRBuilder::InsertPointTy AllocaIP,
5702 OpenMPIRBuilder::InsertPointTy CodeGenIP)
5703 -> OpenMPIRBuilder::InsertPointTy {
5704 Builder.restoreIP(CodeGenIP);
5705 Value = Builder.CreateLoad(Type::getInt32Ty(Ctx), CapturedArgs[0]);
5706 TargetStore = Builder.CreateStore(Value, CapturedArgs[1]);
5707 return Builder.saveIP();
5708 };
5709
5710 IRBuilder<>::InsertPoint EntryIP(&F->getEntryBlock(),
5711 F->getEntryBlock().getFirstInsertionPt());
5712 TargetRegionEntryInfo EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
5713 /*Line=*/3, /*Count=*/0);
5714
5715 Builder.restoreIP(
5716 OMPBuilder.createTarget(Loc, EntryIP, EntryIP, EntryInfo, /*NumTeams=*/-1,
5717 /*NumThreads=*/0, CapturedArgs, GenMapInfoCB,
5718 BodyGenCB, SimpleArgAccessorCB));
5719
5720 Builder.CreateRetVoid();
5721 OMPBuilder.finalize();
5722
5723 // Check outlined function
5724 EXPECT_FALSE(verifyModule(*M, &errs()));
5725 EXPECT_NE(TargetStore, nullptr);
5726 Function *OutlinedFn = TargetStore->getFunction();
5727 EXPECT_NE(F, OutlinedFn);
5728
5729 EXPECT_TRUE(OutlinedFn->hasWeakODRLinkage());
5730 // Account for the "implicit" first argument.
5731 EXPECT_EQ(OutlinedFn->getName(), "__omp_offloading_1_2_parent_l3");
5732 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
5733 EXPECT_TRUE(OutlinedFn->getArg(1)->getType()->isPointerTy());
5734 EXPECT_TRUE(OutlinedFn->getArg(2)->getType()->isPointerTy());
5735
5736 // Check entry block
5737 auto &EntryBlock = OutlinedFn->getEntryBlock();
5738 Instruction *Alloca1 = EntryBlock.getFirstNonPHI();
5739 EXPECT_NE(Alloca1, nullptr);
5740
5741 EXPECT_TRUE(isa<AllocaInst>(Alloca1));
5742 auto *Store1 = Alloca1->getNextNode();
5743 EXPECT_TRUE(isa<StoreInst>(Store1));
5744 auto *Alloca2 = Store1->getNextNode();
5745 EXPECT_TRUE(isa<AllocaInst>(Alloca2));
5746 auto *Store2 = Alloca2->getNextNode();
5747 EXPECT_TRUE(isa<StoreInst>(Store2));
5748
5749 auto *InitCall = dyn_cast<CallInst>(Store2->getNextNode());
5750 EXPECT_NE(InitCall, nullptr);
5751 EXPECT_EQ(InitCall->getCalledFunction()->getName(), "__kmpc_target_init");
5752 EXPECT_EQ(InitCall->arg_size(), 2U);
5753 EXPECT_TRUE(isa<GlobalVariable>(InitCall->getArgOperand(0)));
5754 auto *KernelEnvGV = cast<GlobalVariable>(InitCall->getArgOperand(0));
5755 EXPECT_TRUE(isa<ConstantStruct>(KernelEnvGV->getInitializer()));
5756 auto *KernelEnvC = cast<ConstantStruct>(KernelEnvGV->getInitializer());
5757 EXPECT_TRUE(isa<ConstantStruct>(KernelEnvC->getAggregateElement(0U)));
5758 auto ConfigC = cast<ConstantStruct>(KernelEnvC->getAggregateElement(0U));
5759 EXPECT_EQ(ConfigC->getAggregateElement(0U),
5760 ConstantInt::get(Type::getInt8Ty(Ctx), true));
5761 EXPECT_EQ(ConfigC->getAggregateElement(1U),
5762 ConstantInt::get(Type::getInt8Ty(Ctx), true));
5763 EXPECT_EQ(ConfigC->getAggregateElement(2U),
5764 ConstantInt::get(Type::getInt8Ty(Ctx), OMP_TGT_EXEC_MODE_GENERIC));
5765
5766 auto *EntryBlockBranch = EntryBlock.getTerminator();
5767 EXPECT_NE(EntryBlockBranch, nullptr);
5768 EXPECT_EQ(EntryBlockBranch->getNumSuccessors(), 2U);
5769
5770 // Check user code block
5771 auto *UserCodeBlock = EntryBlockBranch->getSuccessor(0);
5772 EXPECT_EQ(UserCodeBlock->getName(), "user_code.entry");
5773 auto *Load1 = UserCodeBlock->getFirstNonPHI();
5774 EXPECT_TRUE(isa<LoadInst>(Load1));
5775 auto *Load2 = Load1->getNextNode();
5776 EXPECT_TRUE(isa<LoadInst>(Load2));
5777
5778 auto *Value1 = Load2->getNextNode();
5779 EXPECT_EQ(Value1, Value);
5780 EXPECT_EQ(Value1->getNextNode(), TargetStore);
5781 auto *Deinit = TargetStore->getNextNode();
5782 EXPECT_NE(Deinit, nullptr);
5783
5784 auto *DeinitCall = dyn_cast<CallInst>(Deinit);
5785 EXPECT_NE(DeinitCall, nullptr);
5786 EXPECT_EQ(DeinitCall->getCalledFunction()->getName(), "__kmpc_target_deinit");
5787 EXPECT_EQ(DeinitCall->arg_size(), 0U);
5788
5789 EXPECT_TRUE(isa<ReturnInst>(DeinitCall->getNextNode()));
5790
5791 // Check exit block
5792 auto *ExitBlock = EntryBlockBranch->getSuccessor(1);
5793 EXPECT_EQ(ExitBlock->getName(), "worker.exit");
5794 EXPECT_TRUE(isa<ReturnInst>(ExitBlock->getFirstNonPHI()));
5795 }
5796
5797 TEST_F(OpenMPIRBuilderTest, CreateTask) {
5798 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5799 OpenMPIRBuilder OMPBuilder(*M);
5800 OMPBuilder.initialize();
5801 F->setName("func");
5802 IRBuilder<> Builder(BB);
5803
5804 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
5805 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
5806 Value *Val128 =
5807 Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
5808
5809 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5810 Builder.restoreIP(AllocaIP);
5811 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
5812 "bodygen.alloca128");
5813
5814 Builder.restoreIP(CodeGenIP);
5815 // Loading and storing captured pointer and values
5816 Builder.CreateStore(Val128, Local128);
5817 Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
5818 "bodygen.load32");
5819
5820 LoadInst *PrivLoad128 = Builder.CreateLoad(
5821 Local128->getAllocatedType(), Local128, "bodygen.local.load128");
5822 Value *Cmp = Builder.CreateICmpNE(
5823 Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));
5824 Instruction *ThenTerm, *ElseTerm;
5825 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
5826 &ThenTerm, &ElseTerm);
5827 };
5828
5829 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5830 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5831 OpenMPIRBuilder::LocationDescription Loc(
5832 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5833 Builder.restoreIP(OMPBuilder.createTask(
5834 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
5835 BodyGenCB));
5836 OMPBuilder.finalize();
5837 Builder.CreateRetVoid();
5838
5839 EXPECT_FALSE(verifyModule(*M, &errs()));
5840
5841 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5842 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
5843 ->user_back());
5844
5845 // Verify the Ident argument
5846 GlobalVariable *Ident = cast<GlobalVariable>(TaskAllocCall->getArgOperand(0));
5847 ASSERT_NE(Ident, nullptr);
5848 EXPECT_TRUE(Ident->hasInitializer());
5849 Constant *Initializer = Ident->getInitializer();
5850 GlobalVariable *SrcStrGlob =
5851 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
5852 ASSERT_NE(SrcStrGlob, nullptr);
5853 ConstantDataArray *SrcSrc =
5854 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
5855 ASSERT_NE(SrcSrc, nullptr);
5856
5857 // Verify the num_threads argument.
5858 CallInst *GTID = dyn_cast<CallInst>(TaskAllocCall->getArgOperand(1));
5859 ASSERT_NE(GTID, nullptr);
5860 EXPECT_EQ(GTID->arg_size(), 1U);
5861 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
5862
5863 // Verify the flags
5864 // TODO: Check for others flags. Currently testing only for tiedness.
5865 ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
5866 ASSERT_NE(Flags, nullptr);
5867 EXPECT_EQ(Flags->getSExtValue(), 1);
5868
5869 // Verify the data size
5870 ConstantInt *DataSize =
5871 dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));
5872 ASSERT_NE(DataSize, nullptr);
5873 EXPECT_EQ(DataSize->getSExtValue(), 40);
5874
5875 ConstantInt *SharedsSize =
5876 dyn_cast<ConstantInt>(TaskAllocCall->getOperand(4));
5877 EXPECT_EQ(SharedsSize->getSExtValue(),
5878 24); // 64-bit pointer + 128-bit integer
5879
5880 // Verify Wrapper function
5881 Function *OutlinedFn =
5882 dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());
5883 ASSERT_NE(OutlinedFn, nullptr);
5884
5885 LoadInst *SharedsLoad = dyn_cast<LoadInst>(OutlinedFn->begin()->begin());
5886 ASSERT_NE(SharedsLoad, nullptr);
5887 EXPECT_EQ(SharedsLoad->getPointerOperand(), OutlinedFn->getArg(1));
5888
5889 EXPECT_FALSE(OutlinedFn->isDeclaration());
5890 EXPECT_EQ(OutlinedFn->getArg(0)->getType(), Builder.getInt32Ty());
5891
5892 // Verify that the data argument is used only once, and that too in the load
5893 // instruction that is then used for accessing shared data.
5894 Value *DataPtr = OutlinedFn->getArg(1);
5895 EXPECT_EQ(DataPtr->getNumUses(), 1U);
5896 EXPECT_TRUE(isa<LoadInst>(DataPtr->uses().begin()->getUser()));
5897 Value *Data = DataPtr->uses().begin()->getUser();
5898 EXPECT_TRUE(all_of(Data->uses(), [](Use &U) {
5899 return isa<GetElementPtrInst>(U.getUser());
5900 }));
5901
5902 // Verify the presence of `trunc` and `icmp` instructions in Outlined function
5903 EXPECT_TRUE(any_of(instructions(OutlinedFn),
5904 [](Instruction &inst) { return isa<TruncInst>(&inst); }));
5905 EXPECT_TRUE(any_of(instructions(OutlinedFn),
5906 [](Instruction &inst) { return isa<ICmpInst>(&inst); }));
5907
5908 // Verify the execution of the task
5909 CallInst *TaskCall = dyn_cast<CallInst>(
5910 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)
5911 ->user_back());
5912 ASSERT_NE(TaskCall, nullptr);
5913 EXPECT_EQ(TaskCall->getArgOperand(0), Ident);
5914 EXPECT_EQ(TaskCall->getArgOperand(1), GTID);
5915 EXPECT_EQ(TaskCall->getArgOperand(2), TaskAllocCall);
5916
5917 // Verify that the argument data has been copied
5918 for (User *in : TaskAllocCall->users()) {
5919 if (MemCpyInst *memCpyInst = dyn_cast<MemCpyInst>(in)) {
5920 EXPECT_EQ(memCpyInst->getDest(), TaskAllocCall);
5921 }
5922 }
5923 }
5924
5925 TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
5926 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5927 OpenMPIRBuilder OMPBuilder(*M);
5928 OMPBuilder.initialize();
5929 F->setName("func");
5930 IRBuilder<> Builder(BB);
5931
5932 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5933
5934 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5935 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5936 OpenMPIRBuilder::LocationDescription Loc(
5937 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5938 Builder.restoreIP(OMPBuilder.createTask(
5939 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
5940 BodyGenCB));
5941 OMPBuilder.finalize();
5942 Builder.CreateRetVoid();
5943
5944 EXPECT_FALSE(verifyModule(*M, &errs()));
5945
5946 // Check that the outlined function has only one argument.
5947 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5948 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
5949 ->user_back());
5950 Function *OutlinedFn = dyn_cast<Function>(TaskAllocCall->getArgOperand(5));
5951 ASSERT_NE(OutlinedFn, nullptr);
5952 ASSERT_EQ(OutlinedFn->arg_size(), 1U);
5953 }
5954
5955 TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
5956 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5957 OpenMPIRBuilder OMPBuilder(*M);
5958 OMPBuilder.initialize();
5959 F->setName("func");
5960 IRBuilder<> Builder(BB);
5961 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5962 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5963 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5964 OpenMPIRBuilder::LocationDescription Loc(
5965 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5966 Builder.restoreIP(OMPBuilder.createTask(
5967 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
5968 /*Tied=*/false));
5969 OMPBuilder.finalize();
5970 Builder.CreateRetVoid();
5971
5972 // Check for the `Tied` argument
5973 CallInst *TaskAllocCall = dyn_cast<CallInst>(
5974 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
5975 ->user_back());
5976 ASSERT_NE(TaskAllocCall, nullptr);
5977 ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
5978 ASSERT_NE(Flags, nullptr);
5979 EXPECT_EQ(Flags->getZExtValue() & 1U, 0U);
5980
5981 EXPECT_FALSE(verifyModule(*M, &errs()));
5982 }
5983
5984 TEST_F(OpenMPIRBuilderTest, CreateTaskDepend) {
5985 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5986 OpenMPIRBuilder OMPBuilder(*M);
5987 OMPBuilder.initialize();
5988 F->setName("func");
5989 IRBuilder<> Builder(BB);
5990 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
5991 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5992 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5993 OpenMPIRBuilder::LocationDescription Loc(
5994 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5995 AllocaInst *InDep = Builder.CreateAlloca(Type::getInt32Ty(M->getContext()));
5996 SmallVector<OpenMPIRBuilder::DependData> DDS;
5997 {
5998 OpenMPIRBuilder::DependData DDIn(RTLDependenceKindTy::DepIn,
5999 Type::getInt32Ty(M->getContext()), InDep);
6000 DDS.push_back(DDIn);
6001 }
6002 Builder.restoreIP(OMPBuilder.createTask(
6003 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
6004 /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS));
6005 OMPBuilder.finalize();
6006 Builder.CreateRetVoid();
6007
6008 // Check for the `NumDeps` argument
6009 CallInst *TaskAllocCall = dyn_cast<CallInst>(
6010 OMPBuilder
6011 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps)
6012 ->user_back());
6013 ASSERT_NE(TaskAllocCall, nullptr);
6014 ConstantInt *NumDeps = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));
6015 ASSERT_NE(NumDeps, nullptr);
6016 EXPECT_EQ(NumDeps->getZExtValue(), 1U);
6017
6018 // Check for the `DepInfo` array argument
6019 AllocaInst *DepArray = dyn_cast<AllocaInst>(TaskAllocCall->getOperand(4));
6020 ASSERT_NE(DepArray, nullptr);
6021 Value::user_iterator DepArrayI = DepArray->user_begin();
6022 ++DepArrayI;
6023 Value::user_iterator DepInfoI = DepArrayI->user_begin();
6024 // Check for the `DependKind` flag in the `DepInfo` array
6025 Value *Flag = findStoredValue<GetElementPtrInst>(*DepInfoI);
6026 ASSERT_NE(Flag, nullptr);
6027 ConstantInt *FlagInt = dyn_cast<ConstantInt>(Flag);
6028 ASSERT_NE(FlagInt, nullptr);
6029 EXPECT_EQ(FlagInt->getZExtValue(),
6030 static_cast<unsigned int>(RTLDependenceKindTy::DepIn));
6031 ++DepInfoI;
6032 // Check for the size in the `DepInfo` array
6033 Value *Size = findStoredValue<GetElementPtrInst>(*DepInfoI);
6034 ASSERT_NE(Size, nullptr);
6035 ConstantInt *SizeInt = dyn_cast<ConstantInt>(Size);
6036 ASSERT_NE(SizeInt, nullptr);
6037 EXPECT_EQ(SizeInt->getZExtValue(), 4U);
6038 ++DepInfoI;
6039 // Check for the variable address in the `DepInfo` array
6040 Value *AddrStored = findStoredValue<GetElementPtrInst>(*DepInfoI);
6041 ASSERT_NE(AddrStored, nullptr);
6042 PtrToIntInst *AddrInt = dyn_cast<PtrToIntInst>(AddrStored);
6043 ASSERT_NE(AddrInt, nullptr);
6044 Value *Addr = AddrInt->getPointerOperand();
6045 EXPECT_EQ(Addr, InDep);
6046
6047 ConstantInt *NumDepsNoAlias =
6048 dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(5));
6049 ASSERT_NE(NumDepsNoAlias, nullptr);
6050 EXPECT_EQ(NumDepsNoAlias->getZExtValue(), 0U);
6051 EXPECT_EQ(TaskAllocCall->getOperand(6),
6052 ConstantPointerNull::get(Type::getInt8PtrTy(M->getContext())));
6053
6054 EXPECT_FALSE(verifyModule(*M, &errs()));
6055 }
6056
6057 TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
6058 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6059 OpenMPIRBuilder OMPBuilder(*M);
6060 OMPBuilder.initialize();
6061 F->setName("func");
6062 IRBuilder<> Builder(BB);
6063 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6064 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
6065 IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
6066 Builder.SetInsertPoint(BodyBB);
6067 Value *Final = Builder.CreateICmp(
6068 CmpInst::Predicate::ICMP_EQ, F->getArg(0),
6069 ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
6070 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
6071 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
6072 /*Tied=*/false, Final));
6073 OMPBuilder.finalize();
6074 Builder.CreateRetVoid();
6075
6076 // Check for the `Tied` argument
6077 CallInst *TaskAllocCall = dyn_cast<CallInst>(
6078 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
6079 ->user_back());
6080 ASSERT_NE(TaskAllocCall, nullptr);
6081 BinaryOperator *OrInst =
6082 dyn_cast<BinaryOperator>(TaskAllocCall->getArgOperand(2));
6083 ASSERT_NE(OrInst, nullptr);
6084 EXPECT_EQ(OrInst->getOpcode(), BinaryOperator::BinaryOps::Or);
6085
6086 // One of the arguments to `or` instruction is the tied flag, which is equal
6087 // to zero.
6088 EXPECT_TRUE(any_of(OrInst->operands(), [](Value *op) {
6089 if (ConstantInt *TiedValue = dyn_cast<ConstantInt>(op))
6090 return TiedValue->getSExtValue() == 0;
6091 return false;
6092 }));
6093
6094 // One of the arguments to `or` instruction is the final condition.
6095 EXPECT_TRUE(any_of(OrInst->operands(), [Final](Value *op) {
6096 if (SelectInst *Select = dyn_cast<SelectInst>(op)) {
6097 ConstantInt *TrueValue = dyn_cast<ConstantInt>(Select->getTrueValue());
6098 ConstantInt *FalseValue = dyn_cast<ConstantInt>(Select->getFalseValue());
6099 if (!TrueValue || !FalseValue)
6100 return false;
6101 return Select->getCondition() == Final &&
6102 TrueValue->getSExtValue() == 2 && FalseValue->getSExtValue() == 0;
6103 }
6104 return false;
6105 }));
6106
6107 EXPECT_FALSE(verifyModule(*M, &errs()));
6108 }
6109
6110 TEST_F(OpenMPIRBuilderTest, CreateTaskIfCondition) {
6111 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6112 OpenMPIRBuilder OMPBuilder(*M);
6113 OMPBuilder.initialize();
6114 F->setName("func");
6115 IRBuilder<> Builder(BB);
6116 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
6117 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
6118 IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
6119 Builder.SetInsertPoint(BodyBB);
6120 Value *IfCondition = Builder.CreateICmp(
6121 CmpInst::Predicate::ICMP_EQ, F->getArg(0),
6122 ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
6123 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
6124 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
6125 /*Tied=*/false, /*Final=*/nullptr,
6126 IfCondition));
6127 OMPBuilder.finalize();
6128 Builder.CreateRetVoid();
6129
6130 EXPECT_FALSE(verifyModule(*M, &errs()));
6131
6132 CallInst *TaskAllocCall = dyn_cast<CallInst>(
6133 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
6134 ->user_back());
6135 ASSERT_NE(TaskAllocCall, nullptr);
6136
6137 // Check the branching is based on the if condition argument.
6138 BranchInst *IfConditionBranchInst =
6139 dyn_cast<BranchInst>(TaskAllocCall->getParent()->getTerminator());
6140 ASSERT_NE(IfConditionBranchInst, nullptr);
6141 ASSERT_TRUE(IfConditionBranchInst->isConditional());
6142 EXPECT_EQ(IfConditionBranchInst->getCondition(), IfCondition);
6143
6144 // Check that the `__kmpc_omp_task` executes only in the then branch.
6145 CallInst *TaskCall = dyn_cast<CallInst>(
6146 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)
6147 ->user_back());
6148 ASSERT_NE(TaskCall, nullptr);
6149 EXPECT_EQ(TaskCall->getParent(), IfConditionBranchInst->getSuccessor(0));
6150
6151 // Check that the OpenMP Runtime Functions specific to `if` clause execute
6152 // only in the else branch. Also check that the function call is between the
6153 // `__kmpc_omp_task_begin_if0` and `__kmpc_omp_task_complete_if0` calls.
6154 CallInst *TaskBeginIfCall = dyn_cast<CallInst>(
6155 OMPBuilder
6156 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0)
6157 ->user_back());
6158 CallInst *TaskCompleteCall = dyn_cast<CallInst>(
6159 OMPBuilder
6160 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0)
6161 ->user_back());
6162 ASSERT_NE(TaskBeginIfCall, nullptr);
6163 ASSERT_NE(TaskCompleteCall, nullptr);
6164 Function *OulinedFn =
6165 dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());
6166 ASSERT_NE(OulinedFn, nullptr);
6167 CallInst *OulinedFnCall = dyn_cast<CallInst>(OulinedFn->user_back());
6168 ASSERT_NE(OulinedFnCall, nullptr);
6169 EXPECT_EQ(TaskBeginIfCall->getParent(),
6170 IfConditionBranchInst->getSuccessor(1));
6171
6172 EXPECT_EQ(TaskBeginIfCall->getNextNonDebugInstruction(), OulinedFnCall);
6173 EXPECT_EQ(OulinedFnCall->getNextNonDebugInstruction(), TaskCompleteCall);
6174 }
6175
6176 TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
6177 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6178 OpenMPIRBuilder OMPBuilder(*M);
6179 OMPBuilder.initialize();
6180 F->setName("func");
6181 IRBuilder<> Builder(BB);
6182
6183 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
6184 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
6185 Value *Val128 =
6186 Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
6187 Instruction *ThenTerm, *ElseTerm;
6188
6189 Value *InternalStoreInst, *InternalLoad32, *InternalLoad128, *InternalIfCmp;
6190
6191 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6192 Builder.restoreIP(AllocaIP);
6193 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
6194 "bodygen.alloca128");
6195
6196 Builder.restoreIP(CodeGenIP);
6197 // Loading and storing captured pointer and values
6198 InternalStoreInst = Builder.CreateStore(Val128, Local128);
6199 InternalLoad32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
6200 "bodygen.load32");
6201
6202 InternalLoad128 = Builder.CreateLoad(Local128->getAllocatedType(), Local128,
6203 "bodygen.local.load128");
6204 InternalIfCmp = Builder.CreateICmpNE(
6205 InternalLoad32,
6206 Builder.CreateTrunc(InternalLoad128, InternalLoad32->getType()));
6207 SplitBlockAndInsertIfThenElse(InternalIfCmp,
6208 CodeGenIP.getBlock()->getTerminator(),
6209 &ThenTerm, &ElseTerm);
6210 };
6211
6212 BasicBlock *AllocaBB = Builder.GetInsertBlock();
6213 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
6214 OpenMPIRBuilder::LocationDescription Loc(
6215 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6216 Builder.restoreIP(OMPBuilder.createTaskgroup(
6217 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
6218 BodyGenCB));
6219 OMPBuilder.finalize();
6220 Builder.CreateRetVoid();
6221
6222 EXPECT_FALSE(verifyModule(*M, &errs()));
6223
6224 CallInst *TaskgroupCall = dyn_cast<CallInst>(
6225 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)
6226 ->user_back());
6227 ASSERT_NE(TaskgroupCall, nullptr);
6228 CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
6229 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)
6230 ->user_back());
6231 ASSERT_NE(EndTaskgroupCall, nullptr);
6232
6233 // Verify the Ident argument
6234 GlobalVariable *Ident = cast<GlobalVariable>(TaskgroupCall->getArgOperand(0));
6235 ASSERT_NE(Ident, nullptr);
6236 EXPECT_TRUE(Ident->hasInitializer());
6237 Constant *Initializer = Ident->getInitializer();
6238 GlobalVariable *SrcStrGlob =
6239 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
6240 ASSERT_NE(SrcStrGlob, nullptr);
6241 ConstantDataArray *SrcSrc =
6242 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
6243 ASSERT_NE(SrcSrc, nullptr);
6244
6245 // Verify the num_threads argument.
6246 CallInst *GTID = dyn_cast<CallInst>(TaskgroupCall->getArgOperand(1));
6247 ASSERT_NE(GTID, nullptr);
6248 EXPECT_EQ(GTID->arg_size(), 1U);
6249 EXPECT_EQ(GTID->getCalledFunction(), OMPBuilder.getOrCreateRuntimeFunctionPtr(
6250 OMPRTL___kmpc_global_thread_num));
6251
6252 // Checking the general structure of the IR generated is same as expected.
6253 Instruction *GeneratedStoreInst = TaskgroupCall->getNextNonDebugInstruction();
6254 EXPECT_EQ(GeneratedStoreInst, InternalStoreInst);
6255 Instruction *GeneratedLoad32 =
6256 GeneratedStoreInst->getNextNonDebugInstruction();
6257 EXPECT_EQ(GeneratedLoad32, InternalLoad32);
6258 Instruction *GeneratedLoad128 = GeneratedLoad32->getNextNonDebugInstruction();
6259 EXPECT_EQ(GeneratedLoad128, InternalLoad128);
6260
6261 // Checking the ordering because of the if statements and that
6262 // `__kmp_end_taskgroup` call is after the if branching.
6263 BasicBlock *RefOrder[] = {TaskgroupCall->getParent(), ThenTerm->getParent(),
6264 ThenTerm->getSuccessor(0),
6265 EndTaskgroupCall->getParent(),
6266 ElseTerm->getParent()};
6267 verifyDFSOrder(F, RefOrder);
6268 }
6269
6270 TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
6271 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
6272 OpenMPIRBuilder OMPBuilder(*M);
6273 OMPBuilder.initialize();
6274 F->setName("func");
6275 IRBuilder<> Builder(BB);
6276
6277 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6278 Builder.restoreIP(AllocaIP);
6279 AllocaInst *Alloca32 =
6280 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, "bodygen.alloca32");
6281 AllocaInst *Alloca64 =
6282 Builder.CreateAlloca(Builder.getInt64Ty(), nullptr, "bodygen.alloca64");
6283 Builder.restoreIP(CodeGenIP);
6284 auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6285 Builder.restoreIP(CodeGenIP);
6286 LoadInst *LoadValue =
6287 Builder.CreateLoad(Alloca64->getAllocatedType(), Alloca64);
6288 Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64));
6289 Builder.CreateStore(AddInst, Alloca64);
6290 };
6291 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
6292 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1));
6293
6294 auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
6295 Builder.restoreIP(CodeGenIP);
6296 LoadInst *LoadValue =
6297 Builder.CreateLoad(Alloca32->getAllocatedType(), Alloca32);
6298 Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32));
6299 Builder.CreateStore(AddInst, Alloca32);
6300 };
6301 OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);
6302 Builder.restoreIP(OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2));
6303 };
6304
6305 BasicBlock *AllocaBB = Builder.GetInsertBlock();
6306 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
6307 OpenMPIRBuilder::LocationDescription Loc(
6308 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
6309 Builder.restoreIP(OMPBuilder.createTaskgroup(
6310 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
6311 BodyGenCB));
6312 OMPBuilder.finalize();
6313 Builder.CreateRetVoid();
6314
6315 EXPECT_FALSE(verifyModule(*M, &errs()));
6316
6317 CallInst *TaskgroupCall = dyn_cast<CallInst>(
6318 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)
6319 ->user_back());
6320 ASSERT_NE(TaskgroupCall, nullptr);
6321 CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
6322 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)
6323 ->user_back());
6324 ASSERT_NE(EndTaskgroupCall, nullptr);
6325
6326 Function *TaskAllocFn =
6327 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
6328 ASSERT_EQ(TaskAllocFn->getNumUses(), 2u);
6329
6330 CallInst *FirstTaskAllocCall =
6331 dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin());
6332 CallInst *SecondTaskAllocCall =
6333 dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin()++);
6334 ASSERT_NE(FirstTaskAllocCall, nullptr);
6335 ASSERT_NE(SecondTaskAllocCall, nullptr);
6336
6337 // Verify that the tasks have been generated in order and inside taskgroup
6338 // construct.
6339 BasicBlock *RefOrder[] = {
6340 TaskgroupCall->getParent(), FirstTaskAllocCall->getParent(),
6341 SecondTaskAllocCall->getParent(), EndTaskgroupCall->getParent()};
6342 verifyDFSOrder(F, RefOrder);
6343 }
6344
6345 TEST_F(OpenMPIRBuilderTest, EmitOffloadingArraysArguments) {
6346 OpenMPIRBuilder OMPBuilder(*M);
6347 OMPBuilder.initialize();
6348
6349 IRBuilder<> Builder(BB);
6350
6351 OpenMPIRBuilder::TargetDataRTArgs RTArgs;
6352 OpenMPIRBuilder::TargetDataInfo Info(true, false);
6353
6354 auto VoidPtrTy = PointerType::getUnqual(Builder.getContext());
6355 auto VoidPtrPtrTy = PointerType::getUnqual(Builder.getContext());
6356 auto Int64Ty = Type::getInt64Ty(Builder.getContext());
6357 auto Int64PtrTy = PointerType::getUnqual(Builder.getContext());
6358 auto Array4VoidPtrTy = ArrayType::get(VoidPtrTy, 4);
6359 auto Array4Int64PtrTy = ArrayType::get(Int64Ty, 4);
6360
6361 Info.RTArgs.BasePointersArray =
6362 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo(0));
6363 Info.RTArgs.PointersArray =
6364 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo());
6365 Info.RTArgs.SizesArray =
6366 ConstantPointerNull::get(Array4Int64PtrTy->getPointerTo());
6367 Info.RTArgs.MapTypesArray =
6368 ConstantPointerNull::get(Array4Int64PtrTy->getPointerTo());
6369 Info.RTArgs.MapNamesArray =
6370 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo());
6371 Info.RTArgs.MappersArray =
6372 ConstantPointerNull::get(Array4VoidPtrTy->getPointerTo());
6373 Info.NumberOfPtrs = 4;
6374
6375 OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info, false, false);
6376
6377 EXPECT_NE(RTArgs.BasePointersArray, nullptr);
6378 EXPECT_NE(RTArgs.PointersArray, nullptr);
6379 EXPECT_NE(RTArgs.SizesArray, nullptr);
6380 EXPECT_NE(RTArgs.MapTypesArray, nullptr);
6381 EXPECT_NE(RTArgs.MappersArray, nullptr);
6382 EXPECT_NE(RTArgs.MapNamesArray, nullptr);
6383 EXPECT_EQ(RTArgs.MapTypesArrayEnd, nullptr);
6384
6385 EXPECT_EQ(RTArgs.BasePointersArray->getType(), VoidPtrPtrTy);
6386 EXPECT_EQ(RTArgs.PointersArray->getType(), VoidPtrPtrTy);
6387 EXPECT_EQ(RTArgs.SizesArray->getType(), Int64PtrTy);
6388 EXPECT_EQ(RTArgs.MapTypesArray->getType(), Int64PtrTy);
6389 EXPECT_EQ(RTArgs.MappersArray->getType(), VoidPtrPtrTy);
6390 EXPECT_EQ(RTArgs.MapNamesArray->getType(), VoidPtrPtrTy);
6391 }
6392
6393 TEST_F(OpenMPIRBuilderTest, OffloadEntriesInfoManager) {
6394 OpenMPIRBuilder OMPBuilder(*M);
6395 OMPBuilder.setConfig(
6396 OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
6397 OffloadEntriesInfoManager &InfoManager = OMPBuilder.OffloadInfoManager;
6398 TargetRegionEntryInfo EntryInfo("parent", 1, 2, 4, 0);
6399 InfoManager.initializeTargetRegionEntryInfo(EntryInfo, 0);
6400 EXPECT_TRUE(InfoManager.hasTargetRegionEntryInfo(EntryInfo));
6401 InfoManager.initializeDeviceGlobalVarEntryInfo(
6402 "gvar", OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo, 0);
6403 InfoManager.registerTargetRegionEntryInfo(
6404 EntryInfo, nullptr, nullptr,
6405 OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
6406 InfoManager.registerDeviceGlobalVarEntryInfo(
6407 "gvar", 0x0, 8, OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,
6408 GlobalValue::WeakAnyLinkage);
6409 EXPECT_TRUE(InfoManager.hasDeviceGlobalVarEntryInfo("gvar"));
6410 }
6411
6412 // Tests both registerTargetGlobalVariable and getAddrOfDeclareTargetVar as they
6413 // call each other (recursively in some cases). The test case test these
6414 // functions by utilising them for host code generation for declare target
6415 // global variables
6416 TEST_F(OpenMPIRBuilderTest, registerTargetGlobalVariable) {
6417 OpenMPIRBuilder OMPBuilder(*M);
6418 OMPBuilder.initialize();
6419 OpenMPIRBuilderConfig Config(false, false, false, false, false, false, false);
6420 OMPBuilder.setConfig(Config);
6421
6422 std::vector<llvm::Triple> TargetTriple;
6423 TargetTriple.emplace_back("amdgcn-amd-amdhsa");
6424
6425 TargetRegionEntryInfo EntryInfo("", 42, 4711, 17);
6426 std::vector<GlobalVariable *> RefsGathered;
6427
6428 std::vector<Constant *> Globals;
6429 auto *IntTy = Type::getInt32Ty(Ctx);
6430 for (int I = 0; I < 2; ++I) {
6431 Globals.push_back(M->getOrInsertGlobal(
6432 "test_data_int_" + std::to_string(I), IntTy, [&]() -> GlobalVariable * {
6433 return new GlobalVariable(
6434 *M, IntTy, false, GlobalValue::LinkageTypes::WeakAnyLinkage,
6435 ConstantInt::get(IntTy, I), "test_data_int_" + std::to_string(I));
6436 }));
6437 }
6438
6439 OMPBuilder.registerTargetGlobalVariable(
6440 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo,
6441 OffloadEntriesInfoManager::OMPTargetDeviceClauseAny, false, true,
6442 EntryInfo, Globals[0]->getName(), RefsGathered, false, TargetTriple,
6443 nullptr, nullptr, Globals[0]->getType(), Globals[0]);
6444
6445 OMPBuilder.registerTargetGlobalVariable(
6446 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink,
6447 OffloadEntriesInfoManager::OMPTargetDeviceClauseAny, false, true,
6448 EntryInfo, Globals[1]->getName(), RefsGathered, false, TargetTriple,
6449 nullptr, nullptr, Globals[1]->getType(), Globals[1]);
6450
6451 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportfn =
6452 [](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,
6453 const llvm::TargetRegionEntryInfo &EntryInfo) -> void {
6454 // If this is invoked, then we want to emit an error, even if it is not
6455 // neccesarily the most readable, as something has went wrong. The
6456 // test-suite unfortunately eats up all error output
6457 ASSERT_EQ(Kind, Kind);
6458 };
6459
6460 OMPBuilder.createOffloadEntriesAndInfoMetadata(ErrorReportfn);
6461
6462 // Clauses for data_int_0 with To + Any clauses for the host
6463 std::vector<GlobalVariable *> OffloadEntries;
6464 OffloadEntries.push_back(M->getNamedGlobal(".omp_offloading.entry_name"));
6465 OffloadEntries.push_back(
6466 M->getNamedGlobal(".omp_offloading.entry.test_data_int_0"));
6467
6468 // Clauses for data_int_1 with Link + Any clauses for the host
6469 OffloadEntries.push_back(
6470 M->getNamedGlobal("test_data_int_1_decl_tgt_ref_ptr"));
6471 OffloadEntries.push_back(M->getNamedGlobal(".omp_offloading.entry_name.1"));
6472 OffloadEntries.push_back(M->getNamedGlobal(
6473 ".omp_offloading.entry.test_data_int_1_decl_tgt_ref_ptr"));
6474
6475 for (unsigned I = 0; I < OffloadEntries.size(); ++I)
6476 EXPECT_NE(OffloadEntries[I], nullptr);
6477
6478 // Metadata generated for the host offload module
6479 NamedMDNode *OffloadMetadata = M->getNamedMetadata("omp_offload.info");
6480 ASSERT_THAT(OffloadMetadata, testing::NotNull());
6481 StringRef Nodes[2] = {
6482 cast<MDString>(OffloadMetadata->getOperand(0)->getOperand(1))
6483 ->getString(),
6484 cast<MDString>(OffloadMetadata->getOperand(1)->getOperand(1))
6485 ->getString()};
6486 EXPECT_THAT(
6487 Nodes, testing::UnorderedElementsAre("test_data_int_0",
6488 "test_data_int_1_decl_tgt_ref_ptr"));
6489 }
6490
6491 TEST_F(OpenMPIRBuilderTest, createGPUOffloadEntry) {
6492 OpenMPIRBuilder OMPBuilder(*M);
6493 OMPBuilder.initialize();
6494 OpenMPIRBuilderConfig Config(/* IsTargetDevice = */ true,
6495 /* IsGPU = */ true,
6496 /* OpenMPOffloadMandatory = */ false,
6497 /* HasRequiresReverseOffload = */ false,
6498 /* HasRequiresUnifiedAddress = */ false,
6499 /* HasRequiresUnifiedSharedMemory = */ false,
6500 /* HasRequiresDynamicAllocators = */ false);
6501 OMPBuilder.setConfig(Config);
6502
6503 FunctionCallee FnTypeAndCallee =
6504 M->getOrInsertFunction("test_kernel", Type::getVoidTy(Ctx));
6505
6506 auto *Fn = cast<Function>(FnTypeAndCallee.getCallee());
6507 OMPBuilder.createOffloadEntry(/* ID = */ nullptr, Fn,
6508 /* Size = */ 0,
6509 /* Flags = */ 0, GlobalValue::WeakAnyLinkage);
6510
6511 // Check nvvm.annotations only created for GPU kernels
6512 NamedMDNode *MD = M->getNamedMetadata("nvvm.annotations");
6513 EXPECT_NE(MD, nullptr);
6514 EXPECT_EQ(MD->getNumOperands(), 1u);
6515
6516 MDNode *Annotations = MD->getOperand(0);
6517 EXPECT_EQ(Annotations->getNumOperands(), 3u);
6518
6519 Constant *ConstVal =
6520 dyn_cast<ConstantAsMetadata>(Annotations->getOperand(0))->getValue();
6521 EXPECT_TRUE(isa<Function>(Fn));
6522 EXPECT_EQ(ConstVal, cast<Function>(Fn));
6523
6524 EXPECT_TRUE(Annotations->getOperand(1).equalsStr("kernel"));
6525
6526 EXPECT_TRUE(mdconst::hasa<ConstantInt>(Annotations->getOperand(2)));
6527 APInt IntVal =
6528 mdconst::extract<ConstantInt>(Annotations->getOperand(2))->getValue();
6529 EXPECT_EQ(IntVal, 1);
6530
6531 // Check kernel attributes
6532 EXPECT_TRUE(Fn->hasFnAttribute("kernel"));
6533 EXPECT_TRUE(Fn->hasFnAttribute(Attribute::MustProgress));
6534 }
6535
6536 TEST_F(OpenMPIRBuilderTest, CreateRegisterRequires) {
6537 OpenMPIRBuilder OMPBuilder(*M);
6538 OMPBuilder.initialize();
6539
6540 OpenMPIRBuilderConfig Config(/* IsTargetDevice = */ false,
6541 /* IsGPU = */ false,
6542 /* OpenMPOffloadMandatory = */ false,
6543 /* HasRequiresReverseOffload = */ true,
6544 /* HasRequiresUnifiedAddress = */ false,
6545 /* HasRequiresUnifiedSharedMemory = */ true,
6546 /* HasRequiresDynamicAllocators = */ false);
6547 OMPBuilder.setConfig(Config);
6548
6549 auto FName =
6550 OMPBuilder.createPlatformSpecificName({"omp_offloading", "requires_reg"});
6551 EXPECT_EQ(FName, ".omp_offloading.requires_reg");
6552
6553 Function *Fn = OMPBuilder.createRegisterRequires(FName);
6554 EXPECT_NE(Fn, nullptr);
6555 EXPECT_EQ(FName, Fn->getName());
6556
6557 EXPECT_EQ(Fn->getSection(), ".text.startup");
6558 EXPECT_TRUE(Fn->hasInternalLinkage());
6559 EXPECT_TRUE(Fn->hasFnAttribute(Attribute::NoInline));
6560 EXPECT_TRUE(Fn->hasFnAttribute(Attribute::NoUnwind));
6561 EXPECT_EQ(Fn->size(), 1u);
6562
6563 BasicBlock *Entry = &Fn->getEntryBlock();
6564 EXPECT_FALSE(Entry->empty());
6565 EXPECT_EQ(Fn->getReturnType()->getTypeID(), Type::VoidTyID);
6566
6567 CallInst *Call = &cast<CallInst>(*Entry->begin());
6568 EXPECT_EQ(Call->getCalledFunction()->getName(), "__tgt_register_requires");
6569 EXPECT_EQ(Call->getNumOperands(), 2u);
6570
6571 Value *Flags = Call->getArgOperand(0);
6572 EXPECT_EQ(cast<ConstantInt>(Flags)->getSExtValue(),
6573 OMPBuilder.Config.getRequiresFlags());
6574 }
6575
6576 } // namespace
LLVM monorepo