1 //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "llvm/Frontend/OpenMP/OMPConstants.h"
10 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
11 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
12 #include "llvm/IR/BasicBlock.h"
13 #include "llvm/IR/DIBuilder.h"
14 #include "llvm/IR/Function.h"
15 #include "llvm/IR/InstIterator.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/IR/LLVMContext.h"
18 #include "llvm/IR/Module.h"
19 #include "llvm/IR/Verifier.h"
20 #include "llvm/Passes/PassBuilder.h"
21 #include "llvm/Support/Casting.h"
22 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
23 #include "gmock/gmock.h"
24 #include "gtest/gtest.h"
30 // Wrapper lambdas to allow using EXPECT*() macros inside of error-returning
32 #define FINICB_WRAPPER(cb) \
33 [&cb](InsertPointTy IP) -> Error { \
35 return Error::success(); \
38 #define BODYGENCB_WRAPPER(cb) \
39 [&cb](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) -> Error { \
40 cb(AllocaIP, CodeGenIP); \
41 return Error::success(); \
46 /// Create an instruction that uses the values in \p Values. We use "printf"
47 /// just because it is often used for this purpose in test code, but it is never
49 static CallInst
*createPrintfCall(IRBuilder
<> &Builder
, StringRef FormatStr
,
50 ArrayRef
<Value
*> Values
) {
51 Module
*M
= Builder
.GetInsertBlock()->getParent()->getParent();
53 GlobalVariable
*GV
= Builder
.CreateGlobalString(FormatStr
, "", 0, M
);
54 Constant
*Zero
= ConstantInt::get(Type::getInt32Ty(M
->getContext()), 0);
55 Constant
*Indices
[] = {Zero
, Zero
};
56 Constant
*FormatStrConst
=
57 ConstantExpr::getInBoundsGetElementPtr(GV
->getValueType(), GV
, Indices
);
59 Function
*PrintfDecl
= M
->getFunction("printf");
61 GlobalValue::LinkageTypes Linkage
= Function::ExternalLinkage
;
62 FunctionType
*Ty
= FunctionType::get(Builder
.getInt32Ty(), true);
63 PrintfDecl
= Function::Create(Ty
, Linkage
, "printf", M
);
66 SmallVector
<Value
*, 4> Args
;
67 Args
.push_back(FormatStrConst
);
68 Args
.append(Values
.begin(), Values
.end());
69 return Builder
.CreateCall(PrintfDecl
, Args
);
72 /// Verify that blocks in \p RefOrder are corresponds to the depth-first visit
73 /// order the control flow of \p F.
75 /// This is an easy way to verify the branching structure of the CFG without
76 /// checking every branch instruction individually. For the CFG of a
77 /// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering
78 /// the body, i.e. the DFS order corresponds to the execution order with one
80 static testing::AssertionResult
81 verifyDFSOrder(Function
*F
, ArrayRef
<BasicBlock
*> RefOrder
) {
82 ArrayRef
<BasicBlock
*>::iterator It
= RefOrder
.begin();
83 ArrayRef
<BasicBlock
*>::iterator E
= RefOrder
.end();
85 df_iterator_default_set
<BasicBlock
*, 16> Visited
;
86 auto DFS
= llvm::depth_first_ext(&F
->getEntryBlock(), Visited
);
88 BasicBlock
*Prev
= nullptr;
89 for (BasicBlock
*BB
: DFS
) {
90 if (It
!= E
&& BB
== *It
) {
97 return testing::AssertionSuccess();
99 return testing::AssertionFailure()
100 << "Did not find " << (*It
)->getName() << " in control flow";
101 return testing::AssertionFailure()
102 << "Expected " << Prev
->getName() << " before " << (*It
)->getName()
103 << " in control flow";
106 /// Verify that blocks in \p RefOrder are in the same relative order in the
107 /// linked lists of blocks in \p F. The linked list may contain additional
108 /// blocks in-between.
110 /// While the order in the linked list is not relevant for semantics, keeping
111 /// the order roughly in execution order makes its printout easier to read.
112 static testing::AssertionResult
113 verifyListOrder(Function
*F
, ArrayRef
<BasicBlock
*> RefOrder
) {
114 ArrayRef
<BasicBlock
*>::iterator It
= RefOrder
.begin();
115 ArrayRef
<BasicBlock
*>::iterator E
= RefOrder
.end();
117 BasicBlock
*Prev
= nullptr;
118 for (BasicBlock
&BB
: *F
) {
119 if (It
!= E
&& &BB
== *It
) {
126 return testing::AssertionSuccess();
128 return testing::AssertionFailure() << "Did not find " << (*It
)->getName()
129 << " in function " << F
->getName();
130 return testing::AssertionFailure()
131 << "Expected " << Prev
->getName() << " before " << (*It
)->getName()
132 << " in function " << F
->getName();
135 /// Populate Calls with call instructions calling the function with the given
136 /// FnID from the given function F.
137 static void findCalls(Function
*F
, omp::RuntimeFunction FnID
,
138 OpenMPIRBuilder
&OMPBuilder
,
139 SmallVectorImpl
<CallInst
*> &Calls
) {
140 Function
*Fn
= OMPBuilder
.getOrCreateRuntimeFunctionPtr(FnID
);
141 for (BasicBlock
&BB
: *F
) {
142 for (Instruction
&I
: BB
) {
143 auto *Call
= dyn_cast
<CallInst
>(&I
);
144 if (Call
&& Call
->getCalledFunction() == Fn
)
145 Calls
.push_back(Call
);
150 /// Assuming \p F contains only one call to the function with the given \p FnID,
151 /// return that call.
152 static CallInst
*findSingleCall(Function
*F
, omp::RuntimeFunction FnID
,
153 OpenMPIRBuilder
&OMPBuilder
) {
154 SmallVector
<CallInst
*, 1> Calls
;
155 findCalls(F
, FnID
, OMPBuilder
, Calls
);
156 EXPECT_EQ(1u, Calls
.size());
157 if (Calls
.size() != 1)
159 return Calls
.front();
162 static omp::ScheduleKind
getSchedKind(omp::OMPScheduleType SchedType
) {
163 switch (SchedType
& ~omp::OMPScheduleType::ModifierMask
) {
164 case omp::OMPScheduleType::BaseDynamicChunked
:
165 return omp::OMP_SCHEDULE_Dynamic
;
166 case omp::OMPScheduleType::BaseGuidedChunked
:
167 return omp::OMP_SCHEDULE_Guided
;
168 case omp::OMPScheduleType::BaseAuto
:
169 return omp::OMP_SCHEDULE_Auto
;
170 case omp::OMPScheduleType::BaseRuntime
:
171 return omp::OMP_SCHEDULE_Runtime
;
173 llvm_unreachable("unknown type for this test");
177 class OpenMPIRBuilderTest
: public testing::Test
{
179 void SetUp() override
{
180 M
.reset(new Module("MyModule", Ctx
));
182 FunctionType::get(Type::getVoidTy(Ctx
), {Type::getInt32Ty(Ctx
)},
184 F
= Function::Create(FTy
, Function::ExternalLinkage
, "", M
.get());
185 BB
= BasicBlock::Create(Ctx
, "", F
);
188 auto File
= DIB
.createFile("test.dbg", "/src", std::nullopt
,
189 std::optional
<StringRef
>("/src/test.dbg"));
191 DIB
.createCompileUnit(dwarf::DW_LANG_C
, File
, "llvm-C", true, "", 0);
192 auto Type
= DIB
.createSubroutineType(DIB
.getOrCreateTypeArray({}));
193 auto SP
= DIB
.createFunction(
194 CU
, "foo", "", File
, 1, Type
, 1, DINode::FlagZero
,
195 DISubprogram::SPFlagDefinition
| DISubprogram::SPFlagOptimized
);
196 F
->setSubprogram(SP
);
197 auto Scope
= DIB
.createLexicalBlockFile(SP
, File
, 0);
199 DL
= DILocation::get(Ctx
, 3, 7, Scope
);
202 void TearDown() override
{
207 /// Create a function with a simple loop that calls printf using the logical
208 /// loop counter for use with tests that need a CanonicalLoopInfo object.
209 CanonicalLoopInfo
*buildSingleLoopFunction(DebugLoc DL
,
210 OpenMPIRBuilder
&OMPBuilder
,
212 CallInst
**Call
= nullptr,
213 BasicBlock
**BodyCode
= nullptr) {
214 OMPBuilder
.initialize();
217 IRBuilder
<> Builder(BB
);
218 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
219 Value
*TripCount
= F
->getArg(0);
221 Type
*IVType
= Type::getIntNTy(Builder
.getContext(), UseIVBits
);
222 Value
*CastedTripCount
=
223 Builder
.CreateZExtOrTrunc(TripCount
, IVType
, "tripcount");
225 auto LoopBodyGenCB
= [&](OpenMPIRBuilder::InsertPointTy CodeGenIP
,
227 Builder
.restoreIP(CodeGenIP
);
229 *BodyCode
= Builder
.GetInsertBlock();
231 // Add something that consumes the induction variable to the body.
232 CallInst
*CallInst
= createPrintfCall(Builder
, "%d\\n", {LC
});
236 return Error::success();
238 Expected
<CanonicalLoopInfo
*> LoopResult
=
239 OMPBuilder
.createCanonicalLoop(Loc
, LoopBodyGenCB
, CastedTripCount
);
240 assert(LoopResult
&& "unexpected error");
241 CanonicalLoopInfo
*Loop
= *LoopResult
;
243 // Finalize the function.
244 Builder
.restoreIP(Loop
->getAfterIP());
245 Builder
.CreateRetVoid();
251 std::unique_ptr
<Module
> M
;
257 class OpenMPIRBuilderTestWithParams
258 : public OpenMPIRBuilderTest
,
259 public ::testing::WithParamInterface
<omp::OMPScheduleType
> {};
261 class OpenMPIRBuilderTestWithIVBits
262 : public OpenMPIRBuilderTest
,
263 public ::testing::WithParamInterface
<int> {};
265 // Returns the value stored in the given allocation. Returns null if the given
266 // value is not a result of an InstTy instruction, if no value is stored or if
267 // there is more than one store.
268 template <typename InstTy
> static Value
*findStoredValue(Value
*AllocaValue
) {
269 Instruction
*Inst
= dyn_cast
<InstTy
>(AllocaValue
);
272 StoreInst
*Store
= nullptr;
273 for (Use
&U
: Inst
->uses()) {
274 if (auto *CandidateStore
= dyn_cast
<StoreInst
>(U
.getUser())) {
275 EXPECT_EQ(Store
, nullptr);
276 Store
= CandidateStore
;
281 return Store
->getValueOperand();
284 // Returns the value stored in the aggregate argument of an outlined function,
285 // or nullptr if it is not found.
286 static Value
*findStoredValueInAggregateAt(LLVMContext
&Ctx
, Value
*Aggregate
,
288 GetElementPtrInst
*GEPAtIdx
= nullptr;
289 // Find GEP instruction at that index.
290 for (User
*Usr
: Aggregate
->users()) {
291 GetElementPtrInst
*GEP
= dyn_cast
<GetElementPtrInst
>(Usr
);
295 if (GEP
->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx
), Idx
))
298 EXPECT_EQ(GEPAtIdx
, nullptr);
302 EXPECT_NE(GEPAtIdx
, nullptr);
303 EXPECT_EQ(GEPAtIdx
->getNumUses(), 1U);
305 // Find the value stored to the aggregate.
306 StoreInst
*StoreToAgg
= dyn_cast
<StoreInst
>(*GEPAtIdx
->user_begin());
307 Value
*StoredAggValue
= StoreToAgg
->getValueOperand();
309 Value
*StoredValue
= nullptr;
311 // Find the value stored to the value stored in the aggregate.
312 for (User
*Usr
: StoredAggValue
->users()) {
313 StoreInst
*Store
= dyn_cast
<StoreInst
>(Usr
);
317 if (Store
->getPointerOperand() != StoredAggValue
)
320 EXPECT_EQ(StoredValue
, nullptr);
321 StoredValue
= Store
->getValueOperand();
327 // Returns the aggregate that the value is originating from.
328 static Value
*findAggregateFromValue(Value
*V
) {
329 // Expects a load instruction that loads from the aggregate.
330 LoadInst
*Load
= dyn_cast
<LoadInst
>(V
);
331 EXPECT_NE(Load
, nullptr);
332 // Find the GEP instruction used in the load instruction.
333 GetElementPtrInst
*GEP
=
334 dyn_cast
<GetElementPtrInst
>(Load
->getPointerOperand());
335 EXPECT_NE(GEP
, nullptr);
336 // Find the aggregate used in the GEP instruction.
337 Value
*Aggregate
= GEP
->getPointerOperand();
342 TEST_F(OpenMPIRBuilderTest
, CreateBarrier
) {
343 OpenMPIRBuilder
OMPBuilder(*M
);
344 OMPBuilder
.initialize();
346 IRBuilder
<> Builder(BB
);
348 OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP1
=
349 OMPBuilder
.createBarrier({IRBuilder
<>::InsertPoint()}, OMPD_for
);
350 assert(BarrierIP1
&& "unexpected error");
351 EXPECT_TRUE(M
->global_empty());
352 EXPECT_EQ(M
->size(), 1U);
353 EXPECT_EQ(F
->size(), 1U);
354 EXPECT_EQ(BB
->size(), 0U);
356 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP()});
357 OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP2
=
358 OMPBuilder
.createBarrier(Loc
, OMPD_for
);
359 assert(BarrierIP2
&& "unexpected error");
360 EXPECT_FALSE(M
->global_empty());
361 EXPECT_EQ(M
->size(), 3U);
362 EXPECT_EQ(F
->size(), 1U);
363 EXPECT_EQ(BB
->size(), 2U);
365 CallInst
*GTID
= dyn_cast
<CallInst
>(&BB
->front());
366 EXPECT_NE(GTID
, nullptr);
367 EXPECT_EQ(GTID
->arg_size(), 1U);
368 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
369 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotAccessMemory());
370 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotFreeMemory());
372 CallInst
*Barrier
= dyn_cast
<CallInst
>(GTID
->getNextNode());
373 EXPECT_NE(Barrier
, nullptr);
374 EXPECT_EQ(Barrier
->arg_size(), 2U);
375 EXPECT_EQ(Barrier
->getCalledFunction()->getName(), "__kmpc_barrier");
376 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotAccessMemory());
377 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotFreeMemory());
379 EXPECT_EQ(cast
<CallInst
>(Barrier
)->getArgOperand(1), GTID
);
381 Builder
.CreateUnreachable();
382 EXPECT_FALSE(verifyModule(*M
, &errs()));
385 TEST_F(OpenMPIRBuilderTest
, CreateCancel
) {
386 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
387 OpenMPIRBuilder
OMPBuilder(*M
);
388 OMPBuilder
.initialize();
390 BasicBlock
*CBB
= BasicBlock::Create(Ctx
, "", F
);
391 new UnreachableInst(Ctx
, CBB
);
392 auto FiniCB
= [&](InsertPointTy IP
) {
393 ASSERT_NE(IP
.getBlock(), nullptr);
394 ASSERT_EQ(IP
.getBlock()->end(), IP
.getPoint());
395 BranchInst::Create(CBB
, IP
.getBlock());
397 OMPBuilder
.pushFinalizationCB({FINICB_WRAPPER(FiniCB
), OMPD_parallel
, true});
399 IRBuilder
<> Builder(BB
);
401 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP()});
402 OpenMPIRBuilder::InsertPointOrErrorTy NewIP
=
403 OMPBuilder
.createCancel(Loc
, nullptr, OMPD_parallel
);
404 assert(NewIP
&& "unexpected error");
405 Builder
.restoreIP(*NewIP
);
406 EXPECT_FALSE(M
->global_empty());
407 EXPECT_EQ(M
->size(), 4U);
408 EXPECT_EQ(F
->size(), 4U);
409 EXPECT_EQ(BB
->size(), 4U);
411 CallInst
*GTID
= dyn_cast
<CallInst
>(&BB
->front());
412 EXPECT_NE(GTID
, nullptr);
413 EXPECT_EQ(GTID
->arg_size(), 1U);
414 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
415 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotAccessMemory());
416 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotFreeMemory());
418 CallInst
*Cancel
= dyn_cast
<CallInst
>(GTID
->getNextNode());
419 EXPECT_NE(Cancel
, nullptr);
420 EXPECT_EQ(Cancel
->arg_size(), 3U);
421 EXPECT_EQ(Cancel
->getCalledFunction()->getName(), "__kmpc_cancel");
422 EXPECT_FALSE(Cancel
->getCalledFunction()->doesNotAccessMemory());
423 EXPECT_FALSE(Cancel
->getCalledFunction()->doesNotFreeMemory());
424 EXPECT_EQ(Cancel
->getNumUses(), 1U);
425 Instruction
*CancelBBTI
= Cancel
->getParent()->getTerminator();
426 EXPECT_EQ(CancelBBTI
->getNumSuccessors(), 2U);
427 EXPECT_EQ(CancelBBTI
->getSuccessor(0), NewIP
->getBlock());
428 EXPECT_EQ(CancelBBTI
->getSuccessor(1)->size(), 3U);
429 CallInst
*GTID1
= dyn_cast
<CallInst
>(&CancelBBTI
->getSuccessor(1)->front());
430 EXPECT_NE(GTID1
, nullptr);
431 EXPECT_EQ(GTID1
->arg_size(), 1U);
432 EXPECT_EQ(GTID1
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
433 EXPECT_FALSE(GTID1
->getCalledFunction()->doesNotAccessMemory());
434 EXPECT_FALSE(GTID1
->getCalledFunction()->doesNotFreeMemory());
435 CallInst
*Barrier
= dyn_cast
<CallInst
>(GTID1
->getNextNode());
436 EXPECT_NE(Barrier
, nullptr);
437 EXPECT_EQ(Barrier
->arg_size(), 2U);
438 EXPECT_EQ(Barrier
->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
439 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotAccessMemory());
440 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotFreeMemory());
441 EXPECT_EQ(Barrier
->getNumUses(), 0U);
442 EXPECT_EQ(CancelBBTI
->getSuccessor(1)->getTerminator()->getNumSuccessors(),
444 EXPECT_EQ(CancelBBTI
->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB
);
446 EXPECT_EQ(cast
<CallInst
>(Cancel
)->getArgOperand(1), GTID
);
448 OMPBuilder
.popFinalizationCB();
450 Builder
.CreateUnreachable();
451 EXPECT_FALSE(verifyModule(*M
, &errs()));
454 TEST_F(OpenMPIRBuilderTest
, CreateCancelIfCond
) {
455 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
456 OpenMPIRBuilder
OMPBuilder(*M
);
457 OMPBuilder
.initialize();
459 BasicBlock
*CBB
= BasicBlock::Create(Ctx
, "", F
);
460 new UnreachableInst(Ctx
, CBB
);
461 auto FiniCB
= [&](InsertPointTy IP
) {
462 ASSERT_NE(IP
.getBlock(), nullptr);
463 ASSERT_EQ(IP
.getBlock()->end(), IP
.getPoint());
464 BranchInst::Create(CBB
, IP
.getBlock());
466 OMPBuilder
.pushFinalizationCB({FINICB_WRAPPER(FiniCB
), OMPD_parallel
, true});
468 IRBuilder
<> Builder(BB
);
470 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP()});
471 OpenMPIRBuilder::InsertPointOrErrorTy NewIP
=
472 OMPBuilder
.createCancel(Loc
, Builder
.getTrue(), OMPD_parallel
);
473 assert(NewIP
&& "unexpected error");
474 Builder
.restoreIP(*NewIP
);
475 EXPECT_FALSE(M
->global_empty());
476 EXPECT_EQ(M
->size(), 4U);
477 EXPECT_EQ(F
->size(), 7U);
478 EXPECT_EQ(BB
->size(), 1U);
479 ASSERT_TRUE(isa
<BranchInst
>(BB
->getTerminator()));
480 ASSERT_EQ(BB
->getTerminator()->getNumSuccessors(), 2U);
481 BB
= BB
->getTerminator()->getSuccessor(0);
482 EXPECT_EQ(BB
->size(), 4U);
484 CallInst
*GTID
= dyn_cast
<CallInst
>(&BB
->front());
485 EXPECT_NE(GTID
, nullptr);
486 EXPECT_EQ(GTID
->arg_size(), 1U);
487 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
488 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotAccessMemory());
489 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotFreeMemory());
491 CallInst
*Cancel
= dyn_cast
<CallInst
>(GTID
->getNextNode());
492 EXPECT_NE(Cancel
, nullptr);
493 EXPECT_EQ(Cancel
->arg_size(), 3U);
494 EXPECT_EQ(Cancel
->getCalledFunction()->getName(), "__kmpc_cancel");
495 EXPECT_FALSE(Cancel
->getCalledFunction()->doesNotAccessMemory());
496 EXPECT_FALSE(Cancel
->getCalledFunction()->doesNotFreeMemory());
497 EXPECT_EQ(Cancel
->getNumUses(), 1U);
498 Instruction
*CancelBBTI
= Cancel
->getParent()->getTerminator();
499 EXPECT_EQ(CancelBBTI
->getNumSuccessors(), 2U);
500 EXPECT_EQ(CancelBBTI
->getSuccessor(0)->size(), 1U);
501 EXPECT_EQ(CancelBBTI
->getSuccessor(0)->getUniqueSuccessor(),
503 EXPECT_EQ(CancelBBTI
->getSuccessor(1)->size(), 3U);
504 CallInst
*GTID1
= dyn_cast
<CallInst
>(&CancelBBTI
->getSuccessor(1)->front());
505 EXPECT_NE(GTID1
, nullptr);
506 EXPECT_EQ(GTID1
->arg_size(), 1U);
507 EXPECT_EQ(GTID1
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
508 EXPECT_FALSE(GTID1
->getCalledFunction()->doesNotAccessMemory());
509 EXPECT_FALSE(GTID1
->getCalledFunction()->doesNotFreeMemory());
510 CallInst
*Barrier
= dyn_cast
<CallInst
>(GTID1
->getNextNode());
511 EXPECT_NE(Barrier
, nullptr);
512 EXPECT_EQ(Barrier
->arg_size(), 2U);
513 EXPECT_EQ(Barrier
->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
514 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotAccessMemory());
515 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotFreeMemory());
516 EXPECT_EQ(Barrier
->getNumUses(), 0U);
517 EXPECT_EQ(CancelBBTI
->getSuccessor(1)->getTerminator()->getNumSuccessors(),
519 EXPECT_EQ(CancelBBTI
->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB
);
521 EXPECT_EQ(cast
<CallInst
>(Cancel
)->getArgOperand(1), GTID
);
523 OMPBuilder
.popFinalizationCB();
525 Builder
.CreateUnreachable();
526 EXPECT_FALSE(verifyModule(*M
, &errs()));
529 TEST_F(OpenMPIRBuilderTest
, CreateCancelBarrier
) {
530 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
531 OpenMPIRBuilder
OMPBuilder(*M
);
532 OMPBuilder
.initialize();
534 BasicBlock
*CBB
= BasicBlock::Create(Ctx
, "", F
);
535 new UnreachableInst(Ctx
, CBB
);
536 auto FiniCB
= [&](InsertPointTy IP
) {
537 ASSERT_NE(IP
.getBlock(), nullptr);
538 ASSERT_EQ(IP
.getBlock()->end(), IP
.getPoint());
539 BranchInst::Create(CBB
, IP
.getBlock());
541 OMPBuilder
.pushFinalizationCB({FINICB_WRAPPER(FiniCB
), OMPD_parallel
, true});
543 IRBuilder
<> Builder(BB
);
545 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP()});
546 OpenMPIRBuilder::InsertPointOrErrorTy NewIP
=
547 OMPBuilder
.createBarrier(Loc
, OMPD_for
);
548 assert(NewIP
&& "unexpected error");
549 Builder
.restoreIP(*NewIP
);
550 EXPECT_FALSE(M
->global_empty());
551 EXPECT_EQ(M
->size(), 3U);
552 EXPECT_EQ(F
->size(), 4U);
553 EXPECT_EQ(BB
->size(), 4U);
555 CallInst
*GTID
= dyn_cast
<CallInst
>(&BB
->front());
556 EXPECT_NE(GTID
, nullptr);
557 EXPECT_EQ(GTID
->arg_size(), 1U);
558 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
559 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotAccessMemory());
560 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotFreeMemory());
562 CallInst
*Barrier
= dyn_cast
<CallInst
>(GTID
->getNextNode());
563 EXPECT_NE(Barrier
, nullptr);
564 EXPECT_EQ(Barrier
->arg_size(), 2U);
565 EXPECT_EQ(Barrier
->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
566 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotAccessMemory());
567 EXPECT_FALSE(Barrier
->getCalledFunction()->doesNotFreeMemory());
568 EXPECT_EQ(Barrier
->getNumUses(), 1U);
569 Instruction
*BarrierBBTI
= Barrier
->getParent()->getTerminator();
570 EXPECT_EQ(BarrierBBTI
->getNumSuccessors(), 2U);
571 EXPECT_EQ(BarrierBBTI
->getSuccessor(0), NewIP
->getBlock());
572 EXPECT_EQ(BarrierBBTI
->getSuccessor(1)->size(), 1U);
573 EXPECT_EQ(BarrierBBTI
->getSuccessor(1)->getTerminator()->getNumSuccessors(),
575 EXPECT_EQ(BarrierBBTI
->getSuccessor(1)->getTerminator()->getSuccessor(0),
578 EXPECT_EQ(cast
<CallInst
>(Barrier
)->getArgOperand(1), GTID
);
580 OMPBuilder
.popFinalizationCB();
582 Builder
.CreateUnreachable();
583 EXPECT_FALSE(verifyModule(*M
, &errs()));
586 TEST_F(OpenMPIRBuilderTest
, DbgLoc
) {
587 OpenMPIRBuilder
OMPBuilder(*M
);
588 OMPBuilder
.initialize();
591 IRBuilder
<> Builder(BB
);
593 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
594 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
595 OMPBuilder
.createBarrier(Loc
, OMPD_for
);
596 assert(AfterIP
&& "unexpected error");
597 CallInst
*GTID
= dyn_cast
<CallInst
>(&BB
->front());
598 CallInst
*Barrier
= dyn_cast
<CallInst
>(GTID
->getNextNode());
599 EXPECT_EQ(GTID
->getDebugLoc(), DL
);
600 EXPECT_EQ(Barrier
->getDebugLoc(), DL
);
601 EXPECT_TRUE(isa
<GlobalVariable
>(Barrier
->getOperand(0)));
602 if (!isa
<GlobalVariable
>(Barrier
->getOperand(0)))
604 GlobalVariable
*Ident
= cast
<GlobalVariable
>(Barrier
->getOperand(0));
605 EXPECT_TRUE(Ident
->hasInitializer());
606 if (!Ident
->hasInitializer())
608 Constant
*Initializer
= Ident
->getInitializer();
610 isa
<GlobalVariable
>(Initializer
->getOperand(4)->stripPointerCasts()));
611 GlobalVariable
*SrcStrGlob
=
612 cast
<GlobalVariable
>(Initializer
->getOperand(4)->stripPointerCasts());
615 EXPECT_TRUE(isa
<ConstantDataArray
>(SrcStrGlob
->getInitializer()));
616 ConstantDataArray
*SrcSrc
=
617 dyn_cast
<ConstantDataArray
>(SrcStrGlob
->getInitializer());
620 EXPECT_EQ(SrcSrc
->getAsCString(), ";/src/test.dbg;foo;3;7;;");
623 TEST_F(OpenMPIRBuilderTest
, ParallelSimpleGPU
) {
624 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
625 std::string oldDLStr
= M
->getDataLayoutStr();
627 "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:"
628 "256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:"
629 "256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8");
630 OpenMPIRBuilder
OMPBuilder(*M
);
631 OMPBuilder
.Config
.IsTargetDevice
= true;
632 OMPBuilder
.initialize();
634 IRBuilder
<> Builder(BB
);
635 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
636 Builder
.CreateBr(EnterBB
);
637 Builder
.SetInsertPoint(EnterBB
);
638 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
640 AllocaInst
*PrivAI
= nullptr;
642 unsigned NumBodiesGenerated
= 0;
643 unsigned NumPrivatizedVars
= 0;
644 unsigned NumFinalizationPoints
= 0;
646 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
647 ++NumBodiesGenerated
;
649 Builder
.restoreIP(AllocaIP
);
650 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
651 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
653 Builder
.restoreIP(CodeGenIP
);
655 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
656 Value
*Cmp
= Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
657 Instruction
*ThenTerm
, *ElseTerm
;
658 SplitBlockAndInsertIfThenElse(Cmp
, CodeGenIP
.getBlock()->getTerminator(),
659 &ThenTerm
, &ElseTerm
);
660 return Error::success();
663 auto PrivCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
664 Value
&Orig
, Value
&Inner
,
665 Value
*&ReplacementValue
) -> InsertPointTy
{
668 if (!isa
<AllocaInst
>(Orig
)) {
669 EXPECT_EQ(&Orig
, F
->arg_begin());
670 ReplacementValue
= &Inner
;
674 // Since the original value is an allocation, it has a pointer type and
675 // therefore no additional wrapping should happen.
676 EXPECT_EQ(&Orig
, &Inner
);
678 // Trivial copy (=firstprivate).
679 Builder
.restoreIP(AllocaIP
);
680 Type
*VTy
= ReplacementValue
->getType();
681 Value
*V
= Builder
.CreateLoad(VTy
, &Inner
, Orig
.getName() + ".reload");
682 ReplacementValue
= Builder
.CreateAlloca(VTy
, 0, Orig
.getName() + ".copy");
683 Builder
.restoreIP(CodeGenIP
);
684 Builder
.CreateStore(V
, ReplacementValue
);
688 auto FiniCB
= [&](InsertPointTy CodeGenIP
) {
689 ++NumFinalizationPoints
;
690 return Error::success();
693 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
694 F
->getEntryBlock().getFirstInsertionPt());
695 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
696 OMPBuilder
.createParallel(Loc
, AllocaIP
, BodyGenCB
, PrivCB
, FiniCB
,
697 nullptr, nullptr, OMP_PROC_BIND_default
, false);
698 assert(AfterIP
&& "unexpected error");
700 EXPECT_EQ(NumBodiesGenerated
, 1U);
701 EXPECT_EQ(NumPrivatizedVars
, 1U);
702 EXPECT_EQ(NumFinalizationPoints
, 1U);
704 Builder
.restoreIP(*AfterIP
);
705 Builder
.CreateRetVoid();
707 OMPBuilder
.finalize();
708 Function
*OutlinedFn
= PrivAI
->getFunction();
709 EXPECT_FALSE(verifyModule(*M
, &errs()));
710 EXPECT_NE(OutlinedFn
, F
);
711 EXPECT_TRUE(OutlinedFn
->hasFnAttribute(Attribute::NoUnwind
));
712 EXPECT_TRUE(OutlinedFn
->hasParamAttribute(0, Attribute::NoAlias
));
713 EXPECT_TRUE(OutlinedFn
->hasParamAttribute(1, Attribute::NoAlias
));
715 EXPECT_TRUE(OutlinedFn
->hasInternalLinkage());
716 EXPECT_EQ(OutlinedFn
->arg_size(), 3U);
717 // Make sure that arguments are pointers in 0 address address space
718 EXPECT_EQ(OutlinedFn
->getArg(0)->getType(),
719 PointerType::get(M
->getContext(), 0));
720 EXPECT_EQ(OutlinedFn
->getArg(1)->getType(),
721 PointerType::get(M
->getContext(), 0));
722 EXPECT_EQ(OutlinedFn
->getArg(2)->getType(),
723 PointerType::get(M
->getContext(), 0));
724 EXPECT_EQ(&OutlinedFn
->getEntryBlock(), PrivAI
->getParent());
725 EXPECT_EQ(OutlinedFn
->getNumUses(), 1U);
726 User
*Usr
= OutlinedFn
->user_back();
727 ASSERT_TRUE(isa
<CallInst
>(Usr
));
728 CallInst
*Parallel51CI
= dyn_cast
<CallInst
>(Usr
);
729 ASSERT_NE(Parallel51CI
, nullptr);
731 EXPECT_EQ(Parallel51CI
->getCalledFunction()->getName(), "__kmpc_parallel_51");
732 EXPECT_EQ(Parallel51CI
->arg_size(), 9U);
733 EXPECT_EQ(Parallel51CI
->getArgOperand(5), OutlinedFn
);
735 isa
<GlobalVariable
>(Parallel51CI
->getArgOperand(0)->stripPointerCasts()));
736 EXPECT_EQ(Parallel51CI
, Usr
);
737 M
->setDataLayout(oldDLStr
);
740 TEST_F(OpenMPIRBuilderTest
, ParallelSimple
) {
741 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
742 OpenMPIRBuilder
OMPBuilder(*M
);
743 OMPBuilder
.Config
.IsTargetDevice
= false;
744 OMPBuilder
.initialize();
746 IRBuilder
<> Builder(BB
);
748 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
749 Builder
.CreateBr(EnterBB
);
750 Builder
.SetInsertPoint(EnterBB
);
751 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
753 AllocaInst
*PrivAI
= nullptr;
755 unsigned NumBodiesGenerated
= 0;
756 unsigned NumPrivatizedVars
= 0;
757 unsigned NumFinalizationPoints
= 0;
759 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
760 ++NumBodiesGenerated
;
762 Builder
.restoreIP(AllocaIP
);
763 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
764 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
766 Builder
.restoreIP(CodeGenIP
);
768 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
769 Value
*Cmp
= Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
770 Instruction
*ThenTerm
, *ElseTerm
;
771 SplitBlockAndInsertIfThenElse(Cmp
, CodeGenIP
.getBlock()->getTerminator(),
772 &ThenTerm
, &ElseTerm
);
773 return Error::success();
776 auto PrivCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
777 Value
&Orig
, Value
&Inner
,
778 Value
*&ReplacementValue
) -> InsertPointTy
{
781 if (!isa
<AllocaInst
>(Orig
)) {
782 EXPECT_EQ(&Orig
, F
->arg_begin());
783 ReplacementValue
= &Inner
;
787 // Since the original value is an allocation, it has a pointer type and
788 // therefore no additional wrapping should happen.
789 EXPECT_EQ(&Orig
, &Inner
);
791 // Trivial copy (=firstprivate).
792 Builder
.restoreIP(AllocaIP
);
793 Type
*VTy
= ReplacementValue
->getType();
794 Value
*V
= Builder
.CreateLoad(VTy
, &Inner
, Orig
.getName() + ".reload");
795 ReplacementValue
= Builder
.CreateAlloca(VTy
, 0, Orig
.getName() + ".copy");
796 Builder
.restoreIP(CodeGenIP
);
797 Builder
.CreateStore(V
, ReplacementValue
);
801 auto FiniCB
= [&](InsertPointTy CodeGenIP
) {
802 ++NumFinalizationPoints
;
803 return Error::success();
806 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
807 F
->getEntryBlock().getFirstInsertionPt());
808 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
809 OMPBuilder
.createParallel(Loc
, AllocaIP
, BodyGenCB
, PrivCB
, FiniCB
,
810 nullptr, nullptr, OMP_PROC_BIND_default
, false);
811 assert(AfterIP
&& "unexpected error");
812 EXPECT_EQ(NumBodiesGenerated
, 1U);
813 EXPECT_EQ(NumPrivatizedVars
, 1U);
814 EXPECT_EQ(NumFinalizationPoints
, 1U);
816 Builder
.restoreIP(*AfterIP
);
817 Builder
.CreateRetVoid();
819 OMPBuilder
.finalize();
821 EXPECT_NE(PrivAI
, nullptr);
822 Function
*OutlinedFn
= PrivAI
->getFunction();
823 EXPECT_NE(F
, OutlinedFn
);
824 EXPECT_FALSE(verifyModule(*M
, &errs()));
825 EXPECT_TRUE(OutlinedFn
->hasFnAttribute(Attribute::NoUnwind
));
826 EXPECT_TRUE(OutlinedFn
->hasParamAttribute(0, Attribute::NoAlias
));
827 EXPECT_TRUE(OutlinedFn
->hasParamAttribute(1, Attribute::NoAlias
));
829 EXPECT_TRUE(OutlinedFn
->hasInternalLinkage());
830 EXPECT_EQ(OutlinedFn
->arg_size(), 3U);
832 EXPECT_EQ(&OutlinedFn
->getEntryBlock(), PrivAI
->getParent());
833 EXPECT_EQ(OutlinedFn
->getNumUses(), 1U);
834 User
*Usr
= OutlinedFn
->user_back();
835 ASSERT_TRUE(isa
<CallInst
>(Usr
));
836 CallInst
*ForkCI
= dyn_cast
<CallInst
>(Usr
);
837 ASSERT_NE(ForkCI
, nullptr);
839 EXPECT_EQ(ForkCI
->getCalledFunction()->getName(), "__kmpc_fork_call");
840 EXPECT_EQ(ForkCI
->arg_size(), 4U);
841 EXPECT_TRUE(isa
<GlobalVariable
>(ForkCI
->getArgOperand(0)));
842 EXPECT_EQ(ForkCI
->getArgOperand(1),
843 ConstantInt::get(Type::getInt32Ty(Ctx
), 1U));
844 EXPECT_EQ(ForkCI
, Usr
);
846 findStoredValueInAggregateAt(Ctx
, ForkCI
->getArgOperand(3), 0);
847 EXPECT_EQ(StoredValue
, F
->arg_begin());
850 TEST_F(OpenMPIRBuilderTest
, ParallelNested
) {
851 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
852 OpenMPIRBuilder
OMPBuilder(*M
);
853 OMPBuilder
.Config
.IsTargetDevice
= false;
854 OMPBuilder
.initialize();
856 IRBuilder
<> Builder(BB
);
858 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
859 Builder
.CreateBr(EnterBB
);
860 Builder
.SetInsertPoint(EnterBB
);
861 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
863 unsigned NumInnerBodiesGenerated
= 0;
864 unsigned NumOuterBodiesGenerated
= 0;
865 unsigned NumFinalizationPoints
= 0;
867 auto InnerBodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
868 ++NumInnerBodiesGenerated
;
869 return Error::success();
872 auto PrivCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
873 Value
&Orig
, Value
&Inner
,
874 Value
*&ReplacementValue
) -> InsertPointTy
{
875 // Trivial copy (=firstprivate).
876 Builder
.restoreIP(AllocaIP
);
877 Type
*VTy
= ReplacementValue
->getType();
878 Value
*V
= Builder
.CreateLoad(VTy
, &Inner
, Orig
.getName() + ".reload");
879 ReplacementValue
= Builder
.CreateAlloca(VTy
, 0, Orig
.getName() + ".copy");
880 Builder
.restoreIP(CodeGenIP
);
881 Builder
.CreateStore(V
, ReplacementValue
);
885 auto FiniCB
= [&](InsertPointTy CodeGenIP
) {
886 ++NumFinalizationPoints
;
887 return Error::success();
890 auto OuterBodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
891 ++NumOuterBodiesGenerated
;
892 Builder
.restoreIP(CodeGenIP
);
893 BasicBlock
*CGBB
= CodeGenIP
.getBlock();
894 BasicBlock
*NewBB
= SplitBlock(CGBB
, &*CodeGenIP
.getPoint());
895 CGBB
->getTerminator()->eraseFromParent();
897 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createParallel(
898 InsertPointTy(CGBB
, CGBB
->end()), AllocaIP
, InnerBodyGenCB
, PrivCB
,
899 FiniCB
, nullptr, nullptr, OMP_PROC_BIND_default
, false);
900 assert(AfterIP
&& "unexpected error");
902 Builder
.restoreIP(*AfterIP
);
903 Builder
.CreateBr(NewBB
);
904 return Error::success();
907 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
908 F
->getEntryBlock().getFirstInsertionPt());
909 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
910 OMPBuilder
.createParallel(Loc
, AllocaIP
, OuterBodyGenCB
, PrivCB
, FiniCB
,
911 nullptr, nullptr, OMP_PROC_BIND_default
, false);
912 assert(AfterIP
&& "unexpected error");
914 EXPECT_EQ(NumInnerBodiesGenerated
, 1U);
915 EXPECT_EQ(NumOuterBodiesGenerated
, 1U);
916 EXPECT_EQ(NumFinalizationPoints
, 2U);
918 Builder
.restoreIP(*AfterIP
);
919 Builder
.CreateRetVoid();
921 OMPBuilder
.finalize();
923 EXPECT_EQ(M
->size(), 5U);
924 for (Function
&OutlinedFn
: *M
) {
925 if (F
== &OutlinedFn
|| OutlinedFn
.isDeclaration())
927 EXPECT_FALSE(verifyModule(*M
, &errs()));
928 EXPECT_TRUE(OutlinedFn
.hasFnAttribute(Attribute::NoUnwind
));
929 EXPECT_TRUE(OutlinedFn
.hasParamAttribute(0, Attribute::NoAlias
));
930 EXPECT_TRUE(OutlinedFn
.hasParamAttribute(1, Attribute::NoAlias
));
932 EXPECT_TRUE(OutlinedFn
.hasInternalLinkage());
933 EXPECT_EQ(OutlinedFn
.arg_size(), 2U);
935 EXPECT_EQ(OutlinedFn
.getNumUses(), 1U);
936 User
*Usr
= OutlinedFn
.user_back();
937 ASSERT_TRUE(isa
<CallInst
>(Usr
));
938 CallInst
*ForkCI
= dyn_cast
<CallInst
>(Usr
);
939 ASSERT_NE(ForkCI
, nullptr);
941 EXPECT_EQ(ForkCI
->getCalledFunction()->getName(), "__kmpc_fork_call");
942 EXPECT_EQ(ForkCI
->arg_size(), 3U);
943 EXPECT_TRUE(isa
<GlobalVariable
>(ForkCI
->getArgOperand(0)));
944 EXPECT_EQ(ForkCI
->getArgOperand(1),
945 ConstantInt::get(Type::getInt32Ty(Ctx
), 0U));
946 EXPECT_EQ(ForkCI
, Usr
);
950 TEST_F(OpenMPIRBuilderTest
, ParallelNested2Inner
) {
951 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
952 OpenMPIRBuilder
OMPBuilder(*M
);
953 OMPBuilder
.Config
.IsTargetDevice
= false;
954 OMPBuilder
.initialize();
956 IRBuilder
<> Builder(BB
);
958 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
959 Builder
.CreateBr(EnterBB
);
960 Builder
.SetInsertPoint(EnterBB
);
961 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
963 unsigned NumInnerBodiesGenerated
= 0;
964 unsigned NumOuterBodiesGenerated
= 0;
965 unsigned NumFinalizationPoints
= 0;
967 auto InnerBodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
968 ++NumInnerBodiesGenerated
;
969 return Error::success();
972 auto PrivCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
973 Value
&Orig
, Value
&Inner
,
974 Value
*&ReplacementValue
) -> InsertPointTy
{
975 // Trivial copy (=firstprivate).
976 Builder
.restoreIP(AllocaIP
);
977 Type
*VTy
= ReplacementValue
->getType();
978 Value
*V
= Builder
.CreateLoad(VTy
, &Inner
, Orig
.getName() + ".reload");
979 ReplacementValue
= Builder
.CreateAlloca(VTy
, 0, Orig
.getName() + ".copy");
980 Builder
.restoreIP(CodeGenIP
);
981 Builder
.CreateStore(V
, ReplacementValue
);
985 auto FiniCB
= [&](InsertPointTy CodeGenIP
) {
986 ++NumFinalizationPoints
;
987 return Error::success();
990 auto OuterBodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
991 ++NumOuterBodiesGenerated
;
992 Builder
.restoreIP(CodeGenIP
);
993 BasicBlock
*CGBB
= CodeGenIP
.getBlock();
994 BasicBlock
*NewBB1
= SplitBlock(CGBB
, &*CodeGenIP
.getPoint());
995 BasicBlock
*NewBB2
= SplitBlock(NewBB1
, &*NewBB1
->getFirstInsertionPt());
996 CGBB
->getTerminator()->eraseFromParent();
998 NewBB1
->getTerminator()->eraseFromParent();
1001 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP1
= OMPBuilder
.createParallel(
1002 InsertPointTy(CGBB
, CGBB
->end()), AllocaIP
, InnerBodyGenCB
, PrivCB
,
1003 FiniCB
, nullptr, nullptr, OMP_PROC_BIND_default
, false);
1004 assert(AfterIP1
&& "unexpected error");
1006 Builder
.restoreIP(*AfterIP1
);
1007 Builder
.CreateBr(NewBB1
);
1009 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP2
= OMPBuilder
.createParallel(
1010 InsertPointTy(NewBB1
, NewBB1
->end()), AllocaIP
, InnerBodyGenCB
, PrivCB
,
1011 FiniCB
, nullptr, nullptr, OMP_PROC_BIND_default
, false);
1012 assert(AfterIP2
&& "unexpected error");
1014 Builder
.restoreIP(*AfterIP2
);
1015 Builder
.CreateBr(NewBB2
);
1016 return Error::success();
1019 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
1020 F
->getEntryBlock().getFirstInsertionPt());
1021 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
1022 OMPBuilder
.createParallel(Loc
, AllocaIP
, OuterBodyGenCB
, PrivCB
, FiniCB
,
1023 nullptr, nullptr, OMP_PROC_BIND_default
, false);
1024 assert(AfterIP
&& "unexpected error");
1026 EXPECT_EQ(NumInnerBodiesGenerated
, 2U);
1027 EXPECT_EQ(NumOuterBodiesGenerated
, 1U);
1028 EXPECT_EQ(NumFinalizationPoints
, 3U);
1030 Builder
.restoreIP(*AfterIP
);
1031 Builder
.CreateRetVoid();
1033 OMPBuilder
.finalize();
1035 EXPECT_EQ(M
->size(), 6U);
1036 for (Function
&OutlinedFn
: *M
) {
1037 if (F
== &OutlinedFn
|| OutlinedFn
.isDeclaration())
1039 EXPECT_FALSE(verifyModule(*M
, &errs()));
1040 EXPECT_TRUE(OutlinedFn
.hasFnAttribute(Attribute::NoUnwind
));
1041 EXPECT_TRUE(OutlinedFn
.hasParamAttribute(0, Attribute::NoAlias
));
1042 EXPECT_TRUE(OutlinedFn
.hasParamAttribute(1, Attribute::NoAlias
));
1044 EXPECT_TRUE(OutlinedFn
.hasInternalLinkage());
1045 EXPECT_EQ(OutlinedFn
.arg_size(), 2U);
1047 unsigned NumAllocas
= 0;
1048 for (Instruction
&I
: instructions(OutlinedFn
))
1049 NumAllocas
+= isa
<AllocaInst
>(I
);
1050 EXPECT_EQ(NumAllocas
, 1U);
1052 EXPECT_EQ(OutlinedFn
.getNumUses(), 1U);
1053 User
*Usr
= OutlinedFn
.user_back();
1054 ASSERT_TRUE(isa
<CallInst
>(Usr
));
1055 CallInst
*ForkCI
= dyn_cast
<CallInst
>(Usr
);
1056 ASSERT_NE(ForkCI
, nullptr);
1058 EXPECT_EQ(ForkCI
->getCalledFunction()->getName(), "__kmpc_fork_call");
1059 EXPECT_EQ(ForkCI
->arg_size(), 3U);
1060 EXPECT_TRUE(isa
<GlobalVariable
>(ForkCI
->getArgOperand(0)));
1061 EXPECT_EQ(ForkCI
->getArgOperand(1),
1062 ConstantInt::get(Type::getInt32Ty(Ctx
), 0U));
1063 EXPECT_EQ(ForkCI
, Usr
);
1067 TEST_F(OpenMPIRBuilderTest
, ParallelIfCond
) {
1068 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1069 OpenMPIRBuilder
OMPBuilder(*M
);
1070 OMPBuilder
.Config
.IsTargetDevice
= false;
1071 OMPBuilder
.initialize();
1073 IRBuilder
<> Builder(BB
);
1075 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
1076 Builder
.CreateBr(EnterBB
);
1077 Builder
.SetInsertPoint(EnterBB
);
1078 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
1080 AllocaInst
*PrivAI
= nullptr;
1082 unsigned NumBodiesGenerated
= 0;
1083 unsigned NumPrivatizedVars
= 0;
1084 unsigned NumFinalizationPoints
= 0;
1086 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
1087 ++NumBodiesGenerated
;
1089 Builder
.restoreIP(AllocaIP
);
1090 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
1091 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
1093 Builder
.restoreIP(CodeGenIP
);
1095 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
1096 Value
*Cmp
= Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
1097 Instruction
*ThenTerm
, *ElseTerm
;
1098 SplitBlockAndInsertIfThenElse(Cmp
, &*Builder
.GetInsertPoint(), &ThenTerm
,
1100 return Error::success();
1103 auto PrivCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
1104 Value
&Orig
, Value
&Inner
,
1105 Value
*&ReplacementValue
) -> InsertPointTy
{
1106 ++NumPrivatizedVars
;
1108 if (!isa
<AllocaInst
>(Orig
)) {
1109 EXPECT_EQ(&Orig
, F
->arg_begin());
1110 ReplacementValue
= &Inner
;
1114 // Since the original value is an allocation, it has a pointer type and
1115 // therefore no additional wrapping should happen.
1116 EXPECT_EQ(&Orig
, &Inner
);
1118 // Trivial copy (=firstprivate).
1119 Builder
.restoreIP(AllocaIP
);
1120 Type
*VTy
= ReplacementValue
->getType();
1121 Value
*V
= Builder
.CreateLoad(VTy
, &Inner
, Orig
.getName() + ".reload");
1122 ReplacementValue
= Builder
.CreateAlloca(VTy
, 0, Orig
.getName() + ".copy");
1123 Builder
.restoreIP(CodeGenIP
);
1124 Builder
.CreateStore(V
, ReplacementValue
);
1128 auto FiniCB
= [&](InsertPointTy CodeGenIP
) {
1129 ++NumFinalizationPoints
;
1131 return Error::success();
1134 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
1135 F
->getEntryBlock().getFirstInsertionPt());
1136 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
1137 OMPBuilder
.createParallel(Loc
, AllocaIP
, BodyGenCB
, PrivCB
, FiniCB
,
1138 Builder
.CreateIsNotNull(F
->arg_begin()),
1139 nullptr, OMP_PROC_BIND_default
, false);
1140 assert(AfterIP
&& "unexpected error");
1142 EXPECT_EQ(NumBodiesGenerated
, 1U);
1143 EXPECT_EQ(NumPrivatizedVars
, 1U);
1144 EXPECT_EQ(NumFinalizationPoints
, 1U);
1146 Builder
.restoreIP(*AfterIP
);
1147 Builder
.CreateRetVoid();
1148 OMPBuilder
.finalize();
1150 EXPECT_NE(PrivAI
, nullptr);
1151 Function
*OutlinedFn
= PrivAI
->getFunction();
1152 EXPECT_NE(F
, OutlinedFn
);
1153 EXPECT_FALSE(verifyModule(*M
, &errs()));
1155 EXPECT_TRUE(OutlinedFn
->hasInternalLinkage());
1156 EXPECT_EQ(OutlinedFn
->arg_size(), 3U);
1158 EXPECT_EQ(&OutlinedFn
->getEntryBlock(), PrivAI
->getParent());
1159 ASSERT_EQ(OutlinedFn
->getNumUses(), 1U);
1161 CallInst
*ForkCI
= nullptr;
1162 for (User
*Usr
: OutlinedFn
->users()) {
1163 ASSERT_TRUE(isa
<CallInst
>(Usr
));
1164 ForkCI
= cast
<CallInst
>(Usr
);
1167 EXPECT_EQ(ForkCI
->getCalledFunction()->getName(), "__kmpc_fork_call_if");
1168 EXPECT_EQ(ForkCI
->arg_size(), 5U);
1169 EXPECT_TRUE(isa
<GlobalVariable
>(ForkCI
->getArgOperand(0)));
1170 EXPECT_EQ(ForkCI
->getArgOperand(1),
1171 ConstantInt::get(Type::getInt32Ty(Ctx
), 1));
1172 EXPECT_EQ(ForkCI
->getArgOperand(3)->getType(), Type::getInt32Ty(Ctx
));
1175 TEST_F(OpenMPIRBuilderTest
, ParallelCancelBarrier
) {
1176 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1177 OpenMPIRBuilder
OMPBuilder(*M
);
1178 OMPBuilder
.Config
.IsTargetDevice
= false;
1179 OMPBuilder
.initialize();
1181 IRBuilder
<> Builder(BB
);
1183 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
1184 Builder
.CreateBr(EnterBB
);
1185 Builder
.SetInsertPoint(EnterBB
);
1186 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
1188 unsigned NumBodiesGenerated
= 0;
1189 unsigned NumPrivatizedVars
= 0;
1190 unsigned NumFinalizationPoints
= 0;
1192 CallInst
*CheckedBarrier
= nullptr;
1193 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
1194 ++NumBodiesGenerated
;
1196 Builder
.restoreIP(CodeGenIP
);
1198 // Create three barriers, two cancel barriers but only one checked.
1199 Function
*CBFn
, *BFn
;
1201 OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP1
=
1202 OMPBuilder
.createBarrier(Builder
.saveIP(), OMPD_parallel
);
1203 assert(BarrierIP1
&& "unexpected error");
1204 Builder
.restoreIP(*BarrierIP1
);
1206 CBFn
= M
->getFunction("__kmpc_cancel_barrier");
1207 BFn
= M
->getFunction("__kmpc_barrier");
1208 ASSERT_NE(CBFn
, nullptr);
1209 ASSERT_EQ(BFn
, nullptr);
1210 ASSERT_EQ(CBFn
->getNumUses(), 1U);
1211 ASSERT_TRUE(isa
<CallInst
>(CBFn
->user_back()));
1212 ASSERT_EQ(CBFn
->user_back()->getNumUses(), 1U);
1213 CheckedBarrier
= cast
<CallInst
>(CBFn
->user_back());
1215 OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP2
=
1216 OMPBuilder
.createBarrier(Builder
.saveIP(), OMPD_parallel
, true);
1217 assert(BarrierIP2
&& "unexpected error");
1218 Builder
.restoreIP(*BarrierIP2
);
1219 CBFn
= M
->getFunction("__kmpc_cancel_barrier");
1220 BFn
= M
->getFunction("__kmpc_barrier");
1221 ASSERT_NE(CBFn
, nullptr);
1222 ASSERT_NE(BFn
, nullptr);
1223 ASSERT_EQ(CBFn
->getNumUses(), 1U);
1224 ASSERT_EQ(BFn
->getNumUses(), 1U);
1225 ASSERT_TRUE(isa
<CallInst
>(BFn
->user_back()));
1226 ASSERT_EQ(BFn
->user_back()->getNumUses(), 0U);
1228 OpenMPIRBuilder::InsertPointOrErrorTy BarrierIP3
=
1229 OMPBuilder
.createBarrier(Builder
.saveIP(), OMPD_parallel
, false, false);
1230 assert(BarrierIP3
&& "unexpected error");
1231 Builder
.restoreIP(*BarrierIP3
);
1232 ASSERT_EQ(CBFn
->getNumUses(), 2U);
1233 ASSERT_EQ(BFn
->getNumUses(), 1U);
1234 ASSERT_TRUE(CBFn
->user_back() != CheckedBarrier
);
1235 ASSERT_TRUE(isa
<CallInst
>(CBFn
->user_back()));
1236 ASSERT_EQ(CBFn
->user_back()->getNumUses(), 0U);
1239 auto PrivCB
= [&](InsertPointTy
, InsertPointTy
, Value
&V
, Value
&,
1240 Value
*&) -> InsertPointTy
{
1241 ++NumPrivatizedVars
;
1242 llvm_unreachable("No privatization callback call expected!");
1245 FunctionType
*FakeDestructorTy
=
1246 FunctionType::get(Type::getVoidTy(Ctx
), {Type::getInt32Ty(Ctx
)},
1247 /*isVarArg=*/false);
1248 auto *FakeDestructor
= Function::Create(
1249 FakeDestructorTy
, Function::ExternalLinkage
, "fakeDestructor", M
.get());
1251 auto FiniCB
= [&](InsertPointTy IP
) {
1252 ++NumFinalizationPoints
;
1253 Builder
.restoreIP(IP
);
1254 Builder
.CreateCall(FakeDestructor
,
1255 {Builder
.getInt32(NumFinalizationPoints
)});
1256 return Error::success();
1259 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
1260 F
->getEntryBlock().getFirstInsertionPt());
1261 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createParallel(
1262 Loc
, AllocaIP
, BODYGENCB_WRAPPER(BodyGenCB
), PrivCB
, FiniCB
,
1263 Builder
.CreateIsNotNull(F
->arg_begin()), nullptr, OMP_PROC_BIND_default
,
1265 assert(AfterIP
&& "unexpected error");
1267 EXPECT_EQ(NumBodiesGenerated
, 1U);
1268 EXPECT_EQ(NumPrivatizedVars
, 0U);
1269 EXPECT_EQ(NumFinalizationPoints
, 2U);
1270 EXPECT_EQ(FakeDestructor
->getNumUses(), 2U);
1272 Builder
.restoreIP(*AfterIP
);
1273 Builder
.CreateRetVoid();
1274 OMPBuilder
.finalize();
1276 EXPECT_FALSE(verifyModule(*M
, &errs()));
1278 BasicBlock
*ExitBB
= nullptr;
1279 for (const User
*Usr
: FakeDestructor
->users()) {
1280 const CallInst
*CI
= dyn_cast
<CallInst
>(Usr
);
1281 ASSERT_EQ(CI
->getCalledFunction(), FakeDestructor
);
1282 ASSERT_TRUE(isa
<BranchInst
>(CI
->getNextNode()));
1283 ASSERT_EQ(CI
->getNextNode()->getNumSuccessors(), 1U);
1285 ASSERT_EQ(CI
->getNextNode()->getSuccessor(0), ExitBB
);
1287 ExitBB
= CI
->getNextNode()->getSuccessor(0);
1288 ASSERT_EQ(ExitBB
->size(), 1U);
1289 if (!isa
<ReturnInst
>(ExitBB
->front())) {
1290 ASSERT_TRUE(isa
<BranchInst
>(ExitBB
->front()));
1291 ASSERT_EQ(cast
<BranchInst
>(ExitBB
->front()).getNumSuccessors(), 1U);
1292 ASSERT_TRUE(isa
<ReturnInst
>(
1293 cast
<BranchInst
>(ExitBB
->front()).getSuccessor(0)->front()));
1298 TEST_F(OpenMPIRBuilderTest
, ParallelForwardAsPointers
) {
1299 OpenMPIRBuilder
OMPBuilder(*M
);
1300 OMPBuilder
.Config
.IsTargetDevice
= false;
1301 OMPBuilder
.initialize();
1303 IRBuilder
<> Builder(BB
);
1304 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
1305 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1307 Type
*I32Ty
= Type::getInt32Ty(M
->getContext());
1308 Type
*PtrTy
= PointerType::get(M
->getContext(), 0);
1309 Type
*StructTy
= StructType::get(I32Ty
, PtrTy
);
1310 Type
*VoidTy
= Type::getVoidTy(M
->getContext());
1311 FunctionCallee RetI32Func
= M
->getOrInsertFunction("ret_i32", I32Ty
);
1312 FunctionCallee TakeI32Func
=
1313 M
->getOrInsertFunction("take_i32", VoidTy
, I32Ty
);
1314 FunctionCallee RetI32PtrFunc
= M
->getOrInsertFunction("ret_i32ptr", PtrTy
);
1315 FunctionCallee TakeI32PtrFunc
=
1316 M
->getOrInsertFunction("take_i32ptr", VoidTy
, PtrTy
);
1317 FunctionCallee RetStructFunc
= M
->getOrInsertFunction("ret_struct", StructTy
);
1318 FunctionCallee TakeStructFunc
=
1319 M
->getOrInsertFunction("take_struct", VoidTy
, StructTy
);
1320 FunctionCallee RetStructPtrFunc
=
1321 M
->getOrInsertFunction("ret_structptr", PtrTy
);
1322 FunctionCallee TakeStructPtrFunc
=
1323 M
->getOrInsertFunction("take_structPtr", VoidTy
, PtrTy
);
1324 Value
*I32Val
= Builder
.CreateCall(RetI32Func
);
1325 Value
*I32PtrVal
= Builder
.CreateCall(RetI32PtrFunc
);
1326 Value
*StructVal
= Builder
.CreateCall(RetStructFunc
);
1327 Value
*StructPtrVal
= Builder
.CreateCall(RetStructPtrFunc
);
1329 Instruction
*Internal
;
1330 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
1331 IRBuilder
<>::InsertPointGuard
Guard(Builder
);
1332 Builder
.restoreIP(CodeGenIP
);
1333 Internal
= Builder
.CreateCall(TakeI32Func
, I32Val
);
1334 Builder
.CreateCall(TakeI32PtrFunc
, I32PtrVal
);
1335 Builder
.CreateCall(TakeStructFunc
, StructVal
);
1336 Builder
.CreateCall(TakeStructPtrFunc
, StructPtrVal
);
1337 return Error::success();
1339 auto PrivCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
, Value
&,
1340 Value
&Inner
, Value
*&ReplacementValue
) {
1341 ReplacementValue
= &Inner
;
1344 auto FiniCB
= [](InsertPointTy
) { return Error::success(); };
1346 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
1347 F
->getEntryBlock().getFirstInsertionPt());
1348 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
1349 OMPBuilder
.createParallel(Loc
, AllocaIP
, BodyGenCB
, PrivCB
, FiniCB
,
1350 nullptr, nullptr, OMP_PROC_BIND_default
, false);
1351 assert(AfterIP
&& "unexpected error");
1352 Builder
.restoreIP(*AfterIP
);
1353 Builder
.CreateRetVoid();
1355 OMPBuilder
.finalize();
1357 EXPECT_FALSE(verifyModule(*M
, &errs()));
1358 Function
*OutlinedFn
= Internal
->getFunction();
1360 Type
*Arg2Type
= OutlinedFn
->getArg(2)->getType();
1361 EXPECT_TRUE(Arg2Type
->isPointerTy());
1364 TEST_F(OpenMPIRBuilderTest
, CanonicalLoopSimple
) {
1365 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1366 OpenMPIRBuilder
OMPBuilder(*M
);
1367 OMPBuilder
.initialize();
1368 IRBuilder
<> Builder(BB
);
1369 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
1370 Value
*TripCount
= F
->getArg(0);
1372 unsigned NumBodiesGenerated
= 0;
1373 auto LoopBodyGenCB
= [&](InsertPointTy CodeGenIP
, llvm::Value
*LC
) {
1374 NumBodiesGenerated
+= 1;
1376 Builder
.restoreIP(CodeGenIP
);
1378 Value
*Cmp
= Builder
.CreateICmpEQ(LC
, TripCount
);
1379 Instruction
*ThenTerm
, *ElseTerm
;
1380 SplitBlockAndInsertIfThenElse(Cmp
, CodeGenIP
.getBlock()->getTerminator(),
1381 &ThenTerm
, &ElseTerm
);
1382 return Error::success();
1385 Expected
<CanonicalLoopInfo
*> LoopResult
=
1386 OMPBuilder
.createCanonicalLoop(Loc
, LoopBodyGenCB
, TripCount
);
1387 assert(LoopResult
&& "unexpected error");
1388 CanonicalLoopInfo
*Loop
= *LoopResult
;
1390 Builder
.restoreIP(Loop
->getAfterIP());
1391 ReturnInst
*RetInst
= Builder
.CreateRetVoid();
1392 OMPBuilder
.finalize();
1395 EXPECT_FALSE(verifyModule(*M
, &errs()));
1397 EXPECT_EQ(NumBodiesGenerated
, 1U);
1399 // Verify control flow structure (in addition to Loop->assertOK()).
1400 EXPECT_EQ(Loop
->getPreheader()->getSinglePredecessor(), &F
->getEntryBlock());
1401 EXPECT_EQ(Loop
->getAfter(), Builder
.GetInsertBlock());
1403 Instruction
*IndVar
= Loop
->getIndVar();
1404 EXPECT_TRUE(isa
<PHINode
>(IndVar
));
1405 EXPECT_EQ(IndVar
->getType(), TripCount
->getType());
1406 EXPECT_EQ(IndVar
->getParent(), Loop
->getHeader());
1408 EXPECT_EQ(Loop
->getTripCount(), TripCount
);
1410 BasicBlock
*Body
= Loop
->getBody();
1411 Instruction
*CmpInst
= &Body
->front();
1412 EXPECT_TRUE(isa
<ICmpInst
>(CmpInst
));
1413 EXPECT_EQ(CmpInst
->getOperand(0), IndVar
);
1415 BasicBlock
*LatchPred
= Loop
->getLatch()->getSinglePredecessor();
1416 EXPECT_TRUE(llvm::all_of(successors(Body
), [=](BasicBlock
*SuccBB
) {
1417 return SuccBB
->getSingleSuccessor() == LatchPred
;
1420 EXPECT_EQ(&Loop
->getAfter()->front(), RetInst
);
1423 TEST_F(OpenMPIRBuilderTest
, CanonicalLoopBounds
) {
1424 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1425 OpenMPIRBuilder
OMPBuilder(*M
);
1426 OMPBuilder
.initialize();
1427 IRBuilder
<> Builder(BB
);
1429 // Check the trip count is computed correctly. We generate the canonical loop
1430 // but rely on the IRBuilder's constant folder to compute the final result
1431 // since all inputs are constant. To verify overflow situations, limit the
1432 // trip count / loop counter widths to 16 bits.
1433 auto EvalTripCount
= [&](int64_t Start
, int64_t Stop
, int64_t Step
,
1434 bool IsSigned
, bool InclusiveStop
) -> int64_t {
1435 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
1436 Type
*LCTy
= Type::getInt16Ty(Ctx
);
1437 Value
*StartVal
= ConstantInt::get(LCTy
, Start
);
1438 Value
*StopVal
= ConstantInt::get(LCTy
, Stop
);
1439 Value
*StepVal
= ConstantInt::get(LCTy
, Step
);
1440 auto LoopBodyGenCB
= [&](InsertPointTy CodeGenIP
, llvm::Value
*LC
) {
1441 return Error::success();
1443 Expected
<CanonicalLoopInfo
*> LoopResult
=
1444 OMPBuilder
.createCanonicalLoop(Loc
, LoopBodyGenCB
, StartVal
, StopVal
,
1445 StepVal
, IsSigned
, InclusiveStop
);
1446 assert(LoopResult
&& "unexpected error");
1447 CanonicalLoopInfo
*Loop
= *LoopResult
;
1449 Builder
.restoreIP(Loop
->getAfterIP());
1450 Value
*TripCount
= Loop
->getTripCount();
1451 return cast
<ConstantInt
>(TripCount
)->getValue().getZExtValue();
1454 EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
1455 EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
1456 EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
1457 EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
1458 EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
1459 EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
1460 EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
1461 EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
1462 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
1463 EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
1464 EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
1465 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
1466 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
1468 EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
1469 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
1470 EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
1471 EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
1472 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
1473 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
1475 EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
1476 EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
1477 EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
1478 EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
1479 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
1480 EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
1481 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
1483 EXPECT_EQ(EvalTripCount(INT16_MIN
, 0, 1, true, false), 0x8000);
1484 EXPECT_EQ(EvalTripCount(INT16_MIN
, 0, 1, true, true), 0x8001);
1485 EXPECT_EQ(EvalTripCount(INT16_MIN
, 0x7FFF, 1, true, false), 0xFFFF);
1486 EXPECT_EQ(EvalTripCount(INT16_MIN
+ 1, 0x7FFF, 1, true, true), 0xFFFF);
1487 EXPECT_EQ(EvalTripCount(INT16_MIN
, 0, 0x7FFF, true, false), 2);
1488 EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
1489 EXPECT_EQ(EvalTripCount(0, INT16_MIN
, -1, true, false), 0x8000);
1490 EXPECT_EQ(EvalTripCount(0, INT16_MIN
, -16, true, false), 0x800);
1491 EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN
, -1, true, false), 0xFFFF);
1492 EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN
, true, false), 1);
1493 EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN
, true, true), 2);
1495 // Finalize the function and verify it.
1496 Builder
.CreateRetVoid();
1497 OMPBuilder
.finalize();
1498 EXPECT_FALSE(verifyModule(*M
, &errs()));
1501 TEST_F(OpenMPIRBuilderTest
, CollapseNestedLoops
) {
1502 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1503 OpenMPIRBuilder
OMPBuilder(*M
);
1504 OMPBuilder
.initialize();
1507 IRBuilder
<> Builder(BB
);
1509 Type
*LCTy
= F
->getArg(0)->getType();
1510 Constant
*One
= ConstantInt::get(LCTy
, 1);
1511 Constant
*Two
= ConstantInt::get(LCTy
, 2);
1512 Value
*OuterTripCount
=
1513 Builder
.CreateAdd(F
->getArg(0), Two
, "tripcount.outer");
1514 Value
*InnerTripCount
=
1515 Builder
.CreateAdd(F
->getArg(0), One
, "tripcount.inner");
1517 // Fix an insertion point for ComputeIP.
1518 BasicBlock
*LoopNextEnter
=
1519 BasicBlock::Create(M
->getContext(), "loopnest.enter", F
,
1520 Builder
.GetInsertBlock()->getNextNode());
1521 BranchInst
*EnterBr
= Builder
.CreateBr(LoopNextEnter
);
1522 InsertPointTy ComputeIP
{EnterBr
->getParent(), EnterBr
->getIterator()};
1524 Builder
.SetInsertPoint(LoopNextEnter
);
1525 OpenMPIRBuilder::LocationDescription
OuterLoc(Builder
.saveIP(), DL
);
1527 CanonicalLoopInfo
*InnerLoop
= nullptr;
1528 CallInst
*InbetweenLead
= nullptr;
1529 CallInst
*InbetweenTrail
= nullptr;
1530 CallInst
*Call
= nullptr;
1531 auto OuterLoopBodyGenCB
= [&](InsertPointTy OuterCodeGenIP
, Value
*OuterLC
) {
1532 Builder
.restoreIP(OuterCodeGenIP
);
1534 createPrintfCall(Builder
, "In-between lead i=%d\\n", {OuterLC
});
1536 auto InnerLoopBodyGenCB
= [&](InsertPointTy InnerCodeGenIP
,
1538 Builder
.restoreIP(InnerCodeGenIP
);
1539 Call
= createPrintfCall(Builder
, "body i=%d j=%d\\n", {OuterLC
, InnerLC
});
1540 return Error::success();
1542 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
1543 Builder
.saveIP(), InnerLoopBodyGenCB
, InnerTripCount
, "inner");
1544 assert(LoopResult
&& "unexpected error");
1545 InnerLoop
= *LoopResult
;
1547 Builder
.restoreIP(InnerLoop
->getAfterIP());
1549 createPrintfCall(Builder
, "In-between trail i=%d\\n", {OuterLC
});
1550 return Error::success();
1552 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
1553 OuterLoc
, OuterLoopBodyGenCB
, OuterTripCount
, "outer");
1554 assert(LoopResult
&& "unexpected error");
1555 CanonicalLoopInfo
*OuterLoop
= *LoopResult
;
1557 // Finish the function.
1558 Builder
.restoreIP(OuterLoop
->getAfterIP());
1559 Builder
.CreateRetVoid();
1561 CanonicalLoopInfo
*Collapsed
=
1562 OMPBuilder
.collapseLoops(DL
, {OuterLoop
, InnerLoop
}, ComputeIP
);
1564 OMPBuilder
.finalize();
1565 EXPECT_FALSE(verifyModule(*M
, &errs()));
1567 // Verify control flow and BB order.
1568 BasicBlock
*RefOrder
[] = {
1569 Collapsed
->getPreheader(), Collapsed
->getHeader(),
1570 Collapsed
->getCond(), Collapsed
->getBody(),
1571 InbetweenLead
->getParent(), Call
->getParent(),
1572 InbetweenTrail
->getParent(), Collapsed
->getLatch(),
1573 Collapsed
->getExit(), Collapsed
->getAfter(),
1575 EXPECT_TRUE(verifyDFSOrder(F
, RefOrder
));
1576 EXPECT_TRUE(verifyListOrder(F
, RefOrder
));
1578 // Verify the total trip count.
1579 auto *TripCount
= cast
<MulOperator
>(Collapsed
->getTripCount());
1580 EXPECT_EQ(TripCount
->getOperand(0), OuterTripCount
);
1581 EXPECT_EQ(TripCount
->getOperand(1), InnerTripCount
);
1583 // Verify the changed indvar.
1584 auto *OuterIV
= cast
<BinaryOperator
>(Call
->getOperand(1));
1585 EXPECT_EQ(OuterIV
->getOpcode(), Instruction::UDiv
);
1586 EXPECT_EQ(OuterIV
->getParent(), Collapsed
->getBody());
1587 EXPECT_EQ(OuterIV
->getOperand(1), InnerTripCount
);
1588 EXPECT_EQ(OuterIV
->getOperand(0), Collapsed
->getIndVar());
1590 auto *InnerIV
= cast
<BinaryOperator
>(Call
->getOperand(2));
1591 EXPECT_EQ(InnerIV
->getOpcode(), Instruction::URem
);
1592 EXPECT_EQ(InnerIV
->getParent(), Collapsed
->getBody());
1593 EXPECT_EQ(InnerIV
->getOperand(0), Collapsed
->getIndVar());
1594 EXPECT_EQ(InnerIV
->getOperand(1), InnerTripCount
);
1596 EXPECT_EQ(InbetweenLead
->getOperand(1), OuterIV
);
1597 EXPECT_EQ(InbetweenTrail
->getOperand(1), OuterIV
);
1600 TEST_F(OpenMPIRBuilderTest
, TileSingleLoop
) {
1601 OpenMPIRBuilder
OMPBuilder(*M
);
1603 BasicBlock
*BodyCode
;
1604 CanonicalLoopInfo
*Loop
=
1605 buildSingleLoopFunction(DL
, OMPBuilder
, 32, &Call
, &BodyCode
);
1607 Instruction
*OrigIndVar
= Loop
->getIndVar();
1608 EXPECT_EQ(Call
->getOperand(1), OrigIndVar
);
1611 Constant
*TileSize
= ConstantInt::get(Loop
->getIndVarType(), APInt(32, 7));
1612 std::vector
<CanonicalLoopInfo
*> GenLoops
=
1613 OMPBuilder
.tileLoops(DL
, {Loop
}, {TileSize
});
1615 OMPBuilder
.finalize();
1616 EXPECT_FALSE(verifyModule(*M
, &errs()));
1618 EXPECT_EQ(GenLoops
.size(), 2u);
1619 CanonicalLoopInfo
*Floor
= GenLoops
[0];
1620 CanonicalLoopInfo
*Tile
= GenLoops
[1];
1622 BasicBlock
*RefOrder
[] = {
1623 Floor
->getPreheader(), Floor
->getHeader(), Floor
->getCond(),
1624 Floor
->getBody(), Tile
->getPreheader(), Tile
->getHeader(),
1625 Tile
->getCond(), Tile
->getBody(), BodyCode
,
1626 Tile
->getLatch(), Tile
->getExit(), Tile
->getAfter(),
1627 Floor
->getLatch(), Floor
->getExit(), Floor
->getAfter(),
1629 EXPECT_TRUE(verifyDFSOrder(F
, RefOrder
));
1630 EXPECT_TRUE(verifyListOrder(F
, RefOrder
));
1632 // Check the induction variable.
1633 EXPECT_EQ(Call
->getParent(), BodyCode
);
1634 auto *Shift
= cast
<AddOperator
>(Call
->getOperand(1));
1635 EXPECT_EQ(cast
<Instruction
>(Shift
)->getParent(), Tile
->getBody());
1636 EXPECT_EQ(Shift
->getOperand(1), Tile
->getIndVar());
1637 auto *Scale
= cast
<MulOperator
>(Shift
->getOperand(0));
1638 EXPECT_EQ(cast
<Instruction
>(Scale
)->getParent(), Tile
->getBody());
1639 EXPECT_EQ(Scale
->getOperand(0), TileSize
);
1640 EXPECT_EQ(Scale
->getOperand(1), Floor
->getIndVar());
1643 TEST_F(OpenMPIRBuilderTest
, TileNestedLoops
) {
1644 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1645 OpenMPIRBuilder
OMPBuilder(*M
);
1646 OMPBuilder
.initialize();
1649 IRBuilder
<> Builder(BB
);
1650 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
1651 Value
*TripCount
= F
->getArg(0);
1652 Type
*LCTy
= TripCount
->getType();
1654 BasicBlock
*BodyCode
= nullptr;
1655 CanonicalLoopInfo
*InnerLoop
= nullptr;
1656 auto OuterLoopBodyGenCB
= [&](InsertPointTy OuterCodeGenIP
,
1657 llvm::Value
*OuterLC
) {
1658 auto InnerLoopBodyGenCB
= [&](InsertPointTy InnerCodeGenIP
,
1659 llvm::Value
*InnerLC
) {
1660 Builder
.restoreIP(InnerCodeGenIP
);
1661 BodyCode
= Builder
.GetInsertBlock();
1663 // Add something that consumes the induction variables to the body.
1664 createPrintfCall(Builder
, "i=%d j=%d\\n", {OuterLC
, InnerLC
});
1665 return Error::success();
1667 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
1668 OuterCodeGenIP
, InnerLoopBodyGenCB
, TripCount
, "inner");
1669 assert(LoopResult
&& "unexpected error");
1670 InnerLoop
= *LoopResult
;
1671 return Error::success();
1673 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
1674 Loc
, OuterLoopBodyGenCB
, TripCount
, "outer");
1675 assert(LoopResult
&& "unexpected error");
1676 CanonicalLoopInfo
*OuterLoop
= *LoopResult
;
1678 // Finalize the function.
1679 Builder
.restoreIP(OuterLoop
->getAfterIP());
1680 Builder
.CreateRetVoid();
1682 // Tile to loop nest.
1683 Constant
*OuterTileSize
= ConstantInt::get(LCTy
, APInt(32, 11));
1684 Constant
*InnerTileSize
= ConstantInt::get(LCTy
, APInt(32, 7));
1685 std::vector
<CanonicalLoopInfo
*> GenLoops
= OMPBuilder
.tileLoops(
1686 DL
, {OuterLoop
, InnerLoop
}, {OuterTileSize
, InnerTileSize
});
1688 OMPBuilder
.finalize();
1689 EXPECT_FALSE(verifyModule(*M
, &errs()));
1691 EXPECT_EQ(GenLoops
.size(), 4u);
1692 CanonicalLoopInfo
*Floor1
= GenLoops
[0];
1693 CanonicalLoopInfo
*Floor2
= GenLoops
[1];
1694 CanonicalLoopInfo
*Tile1
= GenLoops
[2];
1695 CanonicalLoopInfo
*Tile2
= GenLoops
[3];
1697 BasicBlock
*RefOrder
[] = {
1698 Floor1
->getPreheader(),
1699 Floor1
->getHeader(),
1702 Floor2
->getPreheader(),
1703 Floor2
->getHeader(),
1706 Tile1
->getPreheader(),
1710 Tile2
->getPreheader(),
1728 EXPECT_TRUE(verifyDFSOrder(F
, RefOrder
));
1729 EXPECT_TRUE(verifyListOrder(F
, RefOrder
));
1732 TEST_F(OpenMPIRBuilderTest
, TileNestedLoopsWithBounds
) {
1733 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1734 OpenMPIRBuilder
OMPBuilder(*M
);
1735 OMPBuilder
.initialize();
1738 IRBuilder
<> Builder(BB
);
1739 Value
*TripCount
= F
->getArg(0);
1740 Type
*LCTy
= TripCount
->getType();
1742 Value
*OuterStartVal
= ConstantInt::get(LCTy
, 2);
1743 Value
*OuterStopVal
= TripCount
;
1744 Value
*OuterStep
= ConstantInt::get(LCTy
, 5);
1745 Value
*InnerStartVal
= ConstantInt::get(LCTy
, 13);
1746 Value
*InnerStopVal
= TripCount
;
1747 Value
*InnerStep
= ConstantInt::get(LCTy
, 3);
1749 // Fix an insertion point for ComputeIP.
1750 BasicBlock
*LoopNextEnter
=
1751 BasicBlock::Create(M
->getContext(), "loopnest.enter", F
,
1752 Builder
.GetInsertBlock()->getNextNode());
1753 BranchInst
*EnterBr
= Builder
.CreateBr(LoopNextEnter
);
1754 InsertPointTy ComputeIP
{EnterBr
->getParent(), EnterBr
->getIterator()};
1756 InsertPointTy LoopIP
{LoopNextEnter
, LoopNextEnter
->begin()};
1757 OpenMPIRBuilder::LocationDescription
Loc({LoopIP
, DL
});
1759 BasicBlock
*BodyCode
= nullptr;
1760 CanonicalLoopInfo
*InnerLoop
= nullptr;
1761 CallInst
*Call
= nullptr;
1762 auto OuterLoopBodyGenCB
= [&](InsertPointTy OuterCodeGenIP
,
1763 llvm::Value
*OuterLC
) {
1764 auto InnerLoopBodyGenCB
= [&](InsertPointTy InnerCodeGenIP
,
1765 llvm::Value
*InnerLC
) {
1766 Builder
.restoreIP(InnerCodeGenIP
);
1767 BodyCode
= Builder
.GetInsertBlock();
1769 // Add something that consumes the induction variable to the body.
1770 Call
= createPrintfCall(Builder
, "i=%d j=%d\\n", {OuterLC
, InnerLC
});
1771 return Error::success();
1773 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
1774 OuterCodeGenIP
, InnerLoopBodyGenCB
, InnerStartVal
, InnerStopVal
,
1775 InnerStep
, false, false, ComputeIP
, "inner");
1776 assert(LoopResult
&& "unexpected error");
1777 InnerLoop
= *LoopResult
;
1778 return Error::success();
1780 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
1781 Loc
, OuterLoopBodyGenCB
, OuterStartVal
, OuterStopVal
, OuterStep
, false,
1782 false, ComputeIP
, "outer");
1783 assert(LoopResult
&& "unexpected error");
1784 CanonicalLoopInfo
*OuterLoop
= *LoopResult
;
1786 // Finalize the function
1787 Builder
.restoreIP(OuterLoop
->getAfterIP());
1788 Builder
.CreateRetVoid();
1790 // Tile the loop nest.
1791 Constant
*TileSize0
= ConstantInt::get(LCTy
, APInt(32, 11));
1792 Constant
*TileSize1
= ConstantInt::get(LCTy
, APInt(32, 7));
1793 std::vector
<CanonicalLoopInfo
*> GenLoops
=
1794 OMPBuilder
.tileLoops(DL
, {OuterLoop
, InnerLoop
}, {TileSize0
, TileSize1
});
1796 OMPBuilder
.finalize();
1797 EXPECT_FALSE(verifyModule(*M
, &errs()));
1799 EXPECT_EQ(GenLoops
.size(), 4u);
1800 CanonicalLoopInfo
*Floor0
= GenLoops
[0];
1801 CanonicalLoopInfo
*Floor1
= GenLoops
[1];
1802 CanonicalLoopInfo
*Tile0
= GenLoops
[2];
1803 CanonicalLoopInfo
*Tile1
= GenLoops
[3];
1805 BasicBlock
*RefOrder
[] = {
1806 Floor0
->getPreheader(),
1807 Floor0
->getHeader(),
1810 Floor1
->getPreheader(),
1811 Floor1
->getHeader(),
1814 Tile0
->getPreheader(),
1818 Tile1
->getPreheader(),
1836 EXPECT_TRUE(verifyDFSOrder(F
, RefOrder
));
1837 EXPECT_TRUE(verifyListOrder(F
, RefOrder
));
1839 EXPECT_EQ(Call
->getParent(), BodyCode
);
1841 auto *RangeShift0
= cast
<AddOperator
>(Call
->getOperand(1));
1842 EXPECT_EQ(RangeShift0
->getOperand(1), OuterStartVal
);
1843 auto *RangeScale0
= cast
<MulOperator
>(RangeShift0
->getOperand(0));
1844 EXPECT_EQ(RangeScale0
->getOperand(1), OuterStep
);
1845 auto *TileShift0
= cast
<AddOperator
>(RangeScale0
->getOperand(0));
1846 EXPECT_EQ(cast
<Instruction
>(TileShift0
)->getParent(), Tile1
->getBody());
1847 EXPECT_EQ(TileShift0
->getOperand(1), Tile0
->getIndVar());
1848 auto *TileScale0
= cast
<MulOperator
>(TileShift0
->getOperand(0));
1849 EXPECT_EQ(cast
<Instruction
>(TileScale0
)->getParent(), Tile1
->getBody());
1850 EXPECT_EQ(TileScale0
->getOperand(0), TileSize0
);
1851 EXPECT_EQ(TileScale0
->getOperand(1), Floor0
->getIndVar());
1853 auto *RangeShift1
= cast
<AddOperator
>(Call
->getOperand(2));
1854 EXPECT_EQ(cast
<Instruction
>(RangeShift1
)->getParent(), BodyCode
);
1855 EXPECT_EQ(RangeShift1
->getOperand(1), InnerStartVal
);
1856 auto *RangeScale1
= cast
<MulOperator
>(RangeShift1
->getOperand(0));
1857 EXPECT_EQ(cast
<Instruction
>(RangeScale1
)->getParent(), BodyCode
);
1858 EXPECT_EQ(RangeScale1
->getOperand(1), InnerStep
);
1859 auto *TileShift1
= cast
<AddOperator
>(RangeScale1
->getOperand(0));
1860 EXPECT_EQ(cast
<Instruction
>(TileShift1
)->getParent(), Tile1
->getBody());
1861 EXPECT_EQ(TileShift1
->getOperand(1), Tile1
->getIndVar());
1862 auto *TileScale1
= cast
<MulOperator
>(TileShift1
->getOperand(0));
1863 EXPECT_EQ(cast
<Instruction
>(TileScale1
)->getParent(), Tile1
->getBody());
1864 EXPECT_EQ(TileScale1
->getOperand(0), TileSize1
);
1865 EXPECT_EQ(TileScale1
->getOperand(1), Floor1
->getIndVar());
1868 TEST_F(OpenMPIRBuilderTest
, TileSingleLoopCounts
) {
1869 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
1870 OpenMPIRBuilder
OMPBuilder(*M
);
1871 OMPBuilder
.initialize();
1872 IRBuilder
<> Builder(BB
);
1874 // Create a loop, tile it, and extract its trip count. All input values are
1875 // constant and IRBuilder evaluates all-constant arithmetic inplace, such that
1876 // the floor trip count itself will be a ConstantInt. Unfortunately we cannot
1877 // do the same for the tile loop.
1878 auto GetFloorCount
= [&](int64_t Start
, int64_t Stop
, int64_t Step
,
1879 bool IsSigned
, bool InclusiveStop
,
1880 int64_t TileSize
) -> uint64_t {
1881 OpenMPIRBuilder::LocationDescription
Loc(Builder
.saveIP(), DL
);
1882 Type
*LCTy
= Type::getInt16Ty(Ctx
);
1883 Value
*StartVal
= ConstantInt::get(LCTy
, Start
);
1884 Value
*StopVal
= ConstantInt::get(LCTy
, Stop
);
1885 Value
*StepVal
= ConstantInt::get(LCTy
, Step
);
1888 auto LoopBodyGenCB
= [&](InsertPointTy CodeGenIP
, llvm::Value
*LC
) {
1889 return Error::success();
1891 Expected
<CanonicalLoopInfo
*> LoopResult
=
1892 OMPBuilder
.createCanonicalLoop(Loc
, LoopBodyGenCB
, StartVal
, StopVal
,
1893 StepVal
, IsSigned
, InclusiveStop
);
1894 assert(LoopResult
&& "unexpected error");
1895 CanonicalLoopInfo
*Loop
= *LoopResult
;
1896 InsertPointTy AfterIP
= Loop
->getAfterIP();
1899 Value
*TileSizeVal
= ConstantInt::get(LCTy
, TileSize
);
1900 std::vector
<CanonicalLoopInfo
*> GenLoops
=
1901 OMPBuilder
.tileLoops(Loc
.DL
, {Loop
}, {TileSizeVal
});
1903 // Set the insertion pointer to after loop, where the next loop will be
1905 Builder
.restoreIP(AfterIP
);
1907 // Extract the trip count.
1908 CanonicalLoopInfo
*FloorLoop
= GenLoops
[0];
1909 Value
*FloorTripCount
= FloorLoop
->getTripCount();
1910 return cast
<ConstantInt
>(FloorTripCount
)->getValue().getZExtValue();
1913 // Empty iteration domain.
1914 EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);
1915 EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);
1916 EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);
1917 EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);
1918 EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);
1920 // Only complete tiles.
1921 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1922 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1923 EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);
1924 EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);
1925 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);
1926 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);
1928 // Only a partial tile.
1929 EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);
1930 EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);
1931 EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);
1932 EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);
1933 EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);
1935 // Complete and partial tiles.
1936 EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);
1937 EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);
1938 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);
1939 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);
1940 EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);
1942 // Close to 16-bit integer range.
1943 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu
);
1944 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu
/ 7 + 1);
1945 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu
/ 7 + 1);
1946 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu
/ 7 + 1);
1947 EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu
/ 7 + 1);
1948 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);
1949 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);
1951 // Finalize the function.
1952 Builder
.CreateRetVoid();
1953 OMPBuilder
.finalize();
1955 EXPECT_FALSE(verifyModule(*M
, &errs()));
1958 TEST_F(OpenMPIRBuilderTest
, ApplySimd
) {
1959 OpenMPIRBuilder
OMPBuilder(*M
);
1960 MapVector
<Value
*, Value
*> AlignedVars
;
1961 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
1963 // Simd-ize the loop.
1964 OMPBuilder
.applySimd(CLI
, AlignedVars
, /* IfCond */ nullptr,
1965 OrderKind::OMP_ORDER_unknown
,
1966 /* Simdlen */ nullptr,
1967 /* Safelen */ nullptr);
1969 OMPBuilder
.finalize();
1970 EXPECT_FALSE(verifyModule(*M
, &errs()));
1973 FunctionAnalysisManager FAM
;
1974 PB
.registerFunctionAnalyses(FAM
);
1975 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
1977 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
1978 EXPECT_EQ(TopLvl
.size(), 1u);
1980 Loop
*L
= TopLvl
.front();
1981 EXPECT_TRUE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
1982 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
1984 // Check for llvm.access.group metadata attached to the printf
1985 // function in the loop body.
1986 BasicBlock
*LoopBody
= CLI
->getBody();
1987 EXPECT_TRUE(any_of(*LoopBody
, [](Instruction
&I
) {
1988 return I
.getMetadata("llvm.access.group") != nullptr;
1992 TEST_F(OpenMPIRBuilderTest
, ApplySimdCustomAligned
) {
1993 OpenMPIRBuilder
OMPBuilder(*M
);
1994 IRBuilder
<> Builder(BB
);
1995 const int AlignmentValue
= 32;
1996 AllocaInst
*Alloc1
=
1997 Builder
.CreateAlloca(Builder
.getPtrTy(), Builder
.getInt64(1));
1998 LoadInst
*Load1
= Builder
.CreateLoad(Alloc1
->getAllocatedType(), Alloc1
);
1999 MapVector
<Value
*, Value
*> AlignedVars
;
2000 AlignedVars
.insert({Load1
, Builder
.getInt64(AlignmentValue
)});
2002 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2004 // Simd-ize the loop.
2005 OMPBuilder
.applySimd(CLI
, AlignedVars
, /* IfCond */ nullptr,
2006 OrderKind::OMP_ORDER_unknown
,
2007 /* Simdlen */ nullptr,
2008 /* Safelen */ nullptr);
2010 OMPBuilder
.finalize();
2011 EXPECT_FALSE(verifyModule(*M
, &errs()));
2014 FunctionAnalysisManager FAM
;
2015 PB
.registerFunctionAnalyses(FAM
);
2016 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2018 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2019 EXPECT_EQ(TopLvl
.size(), 1u);
2021 Loop
*L
= TopLvl
.front();
2022 EXPECT_TRUE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2023 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2025 // Check for llvm.access.group metadata attached to the printf
2026 // function in the loop body.
2027 BasicBlock
*LoopBody
= CLI
->getBody();
2028 EXPECT_TRUE(any_of(*LoopBody
, [](Instruction
&I
) {
2029 return I
.getMetadata("llvm.access.group") != nullptr;
2032 // Check if number of assumption instructions is equal to number of aligned
2034 BasicBlock
*LoopPreheader
= CLI
->getPreheader();
2035 size_t NumAssummptionCallsInPreheader
= count_if(
2036 *LoopPreheader
, [](Instruction
&I
) { return isa
<AssumeInst
>(I
); });
2037 EXPECT_EQ(NumAssummptionCallsInPreheader
, AlignedVars
.size());
2039 // Check if variables are correctly aligned
2040 for (Instruction
&Instr
: *LoopPreheader
) {
2041 if (!isa
<AssumeInst
>(Instr
))
2043 AssumeInst
*AssumeInstruction
= cast
<AssumeInst
>(&Instr
);
2044 if (AssumeInstruction
->getNumTotalBundleOperands()) {
2045 auto Bundle
= AssumeInstruction
->getOperandBundleAt(0);
2046 if (Bundle
.getTagName() == "align") {
2047 EXPECT_TRUE(isa
<ConstantInt
>(Bundle
.Inputs
[1]));
2048 auto ConstIntVal
= dyn_cast
<ConstantInt
>(Bundle
.Inputs
[1]);
2049 EXPECT_EQ(ConstIntVal
->getSExtValue(), AlignmentValue
);
2054 TEST_F(OpenMPIRBuilderTest
, ApplySimdlen
) {
2055 OpenMPIRBuilder
OMPBuilder(*M
);
2056 MapVector
<Value
*, Value
*> AlignedVars
;
2057 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2059 // Simd-ize the loop.
2060 OMPBuilder
.applySimd(CLI
, AlignedVars
,
2061 /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown
,
2062 ConstantInt::get(Type::getInt32Ty(Ctx
), 3),
2063 /* Safelen */ nullptr);
2065 OMPBuilder
.finalize();
2066 EXPECT_FALSE(verifyModule(*M
, &errs()));
2069 FunctionAnalysisManager FAM
;
2070 PB
.registerFunctionAnalyses(FAM
);
2071 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2073 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2074 EXPECT_EQ(TopLvl
.size(), 1u);
2076 Loop
*L
= TopLvl
.front();
2077 EXPECT_TRUE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2078 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2079 EXPECT_EQ(getIntLoopAttribute(L
, "llvm.loop.vectorize.width"), 3);
2081 // Check for llvm.access.group metadata attached to the printf
2082 // function in the loop body.
2083 BasicBlock
*LoopBody
= CLI
->getBody();
2084 EXPECT_TRUE(any_of(*LoopBody
, [](Instruction
&I
) {
2085 return I
.getMetadata("llvm.access.group") != nullptr;
2089 TEST_F(OpenMPIRBuilderTest
, ApplySafelenOrderConcurrent
) {
2090 OpenMPIRBuilder
OMPBuilder(*M
);
2091 MapVector
<Value
*, Value
*> AlignedVars
;
2093 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2095 // Simd-ize the loop.
2096 OMPBuilder
.applySimd(
2097 CLI
, AlignedVars
, /* IfCond */ nullptr, OrderKind::OMP_ORDER_concurrent
,
2098 /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx
), 3));
2100 OMPBuilder
.finalize();
2101 EXPECT_FALSE(verifyModule(*M
, &errs()));
2104 FunctionAnalysisManager FAM
;
2105 PB
.registerFunctionAnalyses(FAM
);
2106 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2108 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2109 EXPECT_EQ(TopLvl
.size(), 1u);
2111 Loop
*L
= TopLvl
.front();
2112 // Parallel metadata shoudl be attached because of presence of
2113 // the order(concurrent) OpenMP clause
2114 EXPECT_TRUE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2115 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2116 EXPECT_EQ(getIntLoopAttribute(L
, "llvm.loop.vectorize.width"), 3);
2118 // Check for llvm.access.group metadata attached to the printf
2119 // function in the loop body.
2120 BasicBlock
*LoopBody
= CLI
->getBody();
2121 EXPECT_TRUE(any_of(*LoopBody
, [](Instruction
&I
) {
2122 return I
.getMetadata("llvm.access.group") != nullptr;
2126 TEST_F(OpenMPIRBuilderTest
, ApplySafelen
) {
2127 OpenMPIRBuilder
OMPBuilder(*M
);
2128 MapVector
<Value
*, Value
*> AlignedVars
;
2130 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2132 OMPBuilder
.applySimd(
2133 CLI
, AlignedVars
, /* IfCond */ nullptr, OrderKind::OMP_ORDER_unknown
,
2134 /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx
), 3));
2136 OMPBuilder
.finalize();
2137 EXPECT_FALSE(verifyModule(*M
, &errs()));
2140 FunctionAnalysisManager FAM
;
2141 PB
.registerFunctionAnalyses(FAM
);
2142 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2144 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2145 EXPECT_EQ(TopLvl
.size(), 1u);
2147 Loop
*L
= TopLvl
.front();
2148 EXPECT_FALSE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2149 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2150 EXPECT_EQ(getIntLoopAttribute(L
, "llvm.loop.vectorize.width"), 3);
2152 // Check for llvm.access.group metadata attached to the printf
2153 // function in the loop body.
2154 BasicBlock
*LoopBody
= CLI
->getBody();
2155 EXPECT_FALSE(any_of(*LoopBody
, [](Instruction
&I
) {
2156 return I
.getMetadata("llvm.access.group") != nullptr;
2160 TEST_F(OpenMPIRBuilderTest
, ApplySimdlenSafelen
) {
2161 OpenMPIRBuilder
OMPBuilder(*M
);
2162 MapVector
<Value
*, Value
*> AlignedVars
;
2164 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2166 OMPBuilder
.applySimd(CLI
, AlignedVars
, /* IfCond */ nullptr,
2167 OrderKind::OMP_ORDER_unknown
,
2168 ConstantInt::get(Type::getInt32Ty(Ctx
), 2),
2169 ConstantInt::get(Type::getInt32Ty(Ctx
), 3));
2171 OMPBuilder
.finalize();
2172 EXPECT_FALSE(verifyModule(*M
, &errs()));
2175 FunctionAnalysisManager FAM
;
2176 PB
.registerFunctionAnalyses(FAM
);
2177 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2179 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2180 EXPECT_EQ(TopLvl
.size(), 1u);
2182 Loop
*L
= TopLvl
.front();
2183 EXPECT_FALSE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2184 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2185 EXPECT_EQ(getIntLoopAttribute(L
, "llvm.loop.vectorize.width"), 2);
2187 // Check for llvm.access.group metadata attached to the printf
2188 // function in the loop body.
2189 BasicBlock
*LoopBody
= CLI
->getBody();
2190 EXPECT_FALSE(any_of(*LoopBody
, [](Instruction
&I
) {
2191 return I
.getMetadata("llvm.access.group") != nullptr;
2195 TEST_F(OpenMPIRBuilderTest
, ApplySimdIf
) {
2196 OpenMPIRBuilder
OMPBuilder(*M
);
2197 IRBuilder
<> Builder(BB
);
2198 MapVector
<Value
*, Value
*> AlignedVars
;
2199 AllocaInst
*Alloc1
= Builder
.CreateAlloca(Builder
.getInt32Ty());
2200 AllocaInst
*Alloc2
= Builder
.CreateAlloca(Builder
.getInt32Ty());
2202 // Generation of if condition
2203 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 0U), Alloc1
);
2204 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 1U), Alloc2
);
2205 LoadInst
*Load1
= Builder
.CreateLoad(Alloc1
->getAllocatedType(), Alloc1
);
2206 LoadInst
*Load2
= Builder
.CreateLoad(Alloc2
->getAllocatedType(), Alloc2
);
2208 Value
*IfCmp
= Builder
.CreateICmpNE(Load1
, Load2
);
2210 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2212 // Simd-ize the loop with if condition
2213 OMPBuilder
.applySimd(CLI
, AlignedVars
, IfCmp
, OrderKind::OMP_ORDER_unknown
,
2214 ConstantInt::get(Type::getInt32Ty(Ctx
), 3),
2215 /* Safelen */ nullptr);
2217 OMPBuilder
.finalize();
2218 EXPECT_FALSE(verifyModule(*M
, &errs()));
2221 FunctionAnalysisManager FAM
;
2222 PB
.registerFunctionAnalyses(FAM
);
2223 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2225 // Check if there are two loops (one with enabled vectorization)
2226 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2227 EXPECT_EQ(TopLvl
.size(), 2u);
2229 Loop
*L
= TopLvl
[0];
2230 EXPECT_TRUE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2231 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2232 EXPECT_EQ(getIntLoopAttribute(L
, "llvm.loop.vectorize.width"), 3);
2234 // The second loop should have disabled vectorization
2236 EXPECT_FALSE(findStringMetadataForLoop(L
, "llvm.loop.parallel_accesses"));
2237 EXPECT_FALSE(getBooleanLoopAttribute(L
, "llvm.loop.vectorize.enable"));
2238 // Check for llvm.access.group metadata attached to the printf
2239 // function in the loop body.
2240 BasicBlock
*LoopBody
= CLI
->getBody();
2241 EXPECT_TRUE(any_of(*LoopBody
, [](Instruction
&I
) {
2242 return I
.getMetadata("llvm.access.group") != nullptr;
2246 TEST_F(OpenMPIRBuilderTest
, UnrollLoopFull
) {
2247 OpenMPIRBuilder
OMPBuilder(*M
);
2249 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2252 OMPBuilder
.unrollLoopFull(DL
, CLI
);
2254 OMPBuilder
.finalize();
2255 EXPECT_FALSE(verifyModule(*M
, &errs()));
2258 FunctionAnalysisManager FAM
;
2259 PB
.registerFunctionAnalyses(FAM
);
2260 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2262 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2263 EXPECT_EQ(TopLvl
.size(), 1u);
2265 Loop
*L
= TopLvl
.front();
2266 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.unroll.enable"));
2267 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.unroll.full"));
2270 TEST_F(OpenMPIRBuilderTest
, UnrollLoopPartial
) {
2271 OpenMPIRBuilder
OMPBuilder(*M
);
2272 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2275 CanonicalLoopInfo
*UnrolledLoop
= nullptr;
2276 OMPBuilder
.unrollLoopPartial(DL
, CLI
, 5, &UnrolledLoop
);
2277 ASSERT_NE(UnrolledLoop
, nullptr);
2279 OMPBuilder
.finalize();
2280 EXPECT_FALSE(verifyModule(*M
, &errs()));
2281 UnrolledLoop
->assertOK();
2284 FunctionAnalysisManager FAM
;
2285 PB
.registerFunctionAnalyses(FAM
);
2286 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2288 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2289 EXPECT_EQ(TopLvl
.size(), 1u);
2290 Loop
*Outer
= TopLvl
.front();
2291 EXPECT_EQ(Outer
->getHeader(), UnrolledLoop
->getHeader());
2292 EXPECT_EQ(Outer
->getLoopLatch(), UnrolledLoop
->getLatch());
2293 EXPECT_EQ(Outer
->getExitingBlock(), UnrolledLoop
->getCond());
2294 EXPECT_EQ(Outer
->getExitBlock(), UnrolledLoop
->getExit());
2296 EXPECT_EQ(Outer
->getSubLoops().size(), 1u);
2297 Loop
*Inner
= Outer
->getSubLoops().front();
2299 EXPECT_TRUE(getBooleanLoopAttribute(Inner
, "llvm.loop.unroll.enable"));
2300 EXPECT_EQ(getIntLoopAttribute(Inner
, "llvm.loop.unroll.count"), 5);
2303 TEST_F(OpenMPIRBuilderTest
, UnrollLoopHeuristic
) {
2304 OpenMPIRBuilder
OMPBuilder(*M
);
2306 CanonicalLoopInfo
*CLI
= buildSingleLoopFunction(DL
, OMPBuilder
, 32);
2309 OMPBuilder
.unrollLoopHeuristic(DL
, CLI
);
2311 OMPBuilder
.finalize();
2312 EXPECT_FALSE(verifyModule(*M
, &errs()));
2315 FunctionAnalysisManager FAM
;
2316 PB
.registerFunctionAnalyses(FAM
);
2317 LoopInfo
&LI
= FAM
.getResult
<LoopAnalysis
>(*F
);
2319 const std::vector
<Loop
*> &TopLvl
= LI
.getTopLevelLoops();
2320 EXPECT_EQ(TopLvl
.size(), 1u);
2322 Loop
*L
= TopLvl
.front();
2323 EXPECT_TRUE(getBooleanLoopAttribute(L
, "llvm.loop.unroll.enable"));
2326 TEST_F(OpenMPIRBuilderTest
, StaticWorkshareLoopTarget
) {
2327 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2328 std::string oldDLStr
= M
->getDataLayoutStr();
2330 "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:"
2331 "256:256:32-p8:128:128-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:"
2332 "256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8");
2333 OpenMPIRBuilder
OMPBuilder(*M
);
2334 OMPBuilder
.Config
.IsTargetDevice
= true;
2335 OMPBuilder
.initialize();
2336 IRBuilder
<> Builder(BB
);
2337 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2338 InsertPointTy AllocaIP
= Builder
.saveIP();
2340 Type
*LCTy
= Type::getInt32Ty(Ctx
);
2341 Value
*StartVal
= ConstantInt::get(LCTy
, 10);
2342 Value
*StopVal
= ConstantInt::get(LCTy
, 52);
2343 Value
*StepVal
= ConstantInt::get(LCTy
, 2);
2344 auto LoopBodyGen
= [&](InsertPointTy
, Value
*) { return Error::success(); };
2346 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
2347 Loc
, LoopBodyGen
, StartVal
, StopVal
, StepVal
, false, false);
2348 assert(LoopResult
&& "unexpected error");
2349 CanonicalLoopInfo
*CLI
= *LoopResult
;
2350 BasicBlock
*Preheader
= CLI
->getPreheader();
2351 Value
*TripCount
= CLI
->getTripCount();
2353 Builder
.SetInsertPoint(BB
, BB
->getFirstInsertionPt());
2355 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.applyWorkshareLoop(
2356 DL
, CLI
, AllocaIP
, true, OMP_SCHEDULE_Static
, nullptr, false, false,
2357 false, false, WorksharingLoopType::ForStaticLoop
);
2358 assert(AfterIP
&& "unexpected error");
2359 Builder
.restoreIP(*AfterIP
);
2360 Builder
.CreateRetVoid();
2362 OMPBuilder
.finalize();
2363 EXPECT_FALSE(verifyModule(*M
, &errs()));
2365 CallInst
*WorkshareLoopRuntimeCall
= nullptr;
2366 int WorkshareLoopRuntimeCallCnt
= 0;
2367 for (auto Inst
= Preheader
->begin(); Inst
!= Preheader
->end(); ++Inst
) {
2368 CallInst
*Call
= dyn_cast
<CallInst
>(Inst
);
2371 if (!Call
->getCalledFunction())
2374 if (Call
->getCalledFunction()->getName() == "__kmpc_for_static_loop_4u") {
2375 WorkshareLoopRuntimeCall
= Call
;
2376 WorkshareLoopRuntimeCallCnt
++;
2379 EXPECT_NE(WorkshareLoopRuntimeCall
, nullptr);
2380 // Verify that there is only one call to workshare loop function
2381 EXPECT_EQ(WorkshareLoopRuntimeCallCnt
, 1);
2382 // Check that pointer to loop body function is passed as second argument
2383 Value
*LoopBodyFuncArg
= WorkshareLoopRuntimeCall
->getArgOperand(1);
2384 EXPECT_EQ(Builder
.getPtrTy(), LoopBodyFuncArg
->getType());
2385 Function
*ArgFunction
= dyn_cast
<Function
>(LoopBodyFuncArg
);
2386 EXPECT_NE(ArgFunction
, nullptr);
2387 EXPECT_EQ(ArgFunction
->arg_size(), 1u);
2388 EXPECT_EQ(ArgFunction
->getArg(0)->getType(), TripCount
->getType());
2389 // Check that no variables except for loop counter are used in loop body
2390 EXPECT_EQ(Constant::getNullValue(Builder
.getPtrTy()),
2391 WorkshareLoopRuntimeCall
->getArgOperand(2));
2392 // Check loop trip count argument
2393 EXPECT_EQ(TripCount
, WorkshareLoopRuntimeCall
->getArgOperand(3));
2396 TEST_F(OpenMPIRBuilderTest
, StaticWorkShareLoop
) {
2397 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2398 OpenMPIRBuilder
OMPBuilder(*M
);
2399 OMPBuilder
.Config
.IsTargetDevice
= false;
2400 OMPBuilder
.initialize();
2401 IRBuilder
<> Builder(BB
);
2402 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2404 Type
*LCTy
= Type::getInt32Ty(Ctx
);
2405 Value
*StartVal
= ConstantInt::get(LCTy
, 10);
2406 Value
*StopVal
= ConstantInt::get(LCTy
, 52);
2407 Value
*StepVal
= ConstantInt::get(LCTy
, 2);
2408 auto LoopBodyGen
= [&](InsertPointTy
, llvm::Value
*) {
2409 return Error::success();
2412 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
2413 Loc
, LoopBodyGen
, StartVal
, StopVal
, StepVal
,
2414 /*IsSigned=*/false, /*InclusiveStop=*/false);
2415 assert(LoopResult
&& "unexpected error");
2416 CanonicalLoopInfo
*CLI
= *LoopResult
;
2417 BasicBlock
*Preheader
= CLI
->getPreheader();
2418 BasicBlock
*Body
= CLI
->getBody();
2419 Value
*IV
= CLI
->getIndVar();
2420 BasicBlock
*ExitBlock
= CLI
->getExit();
2422 Builder
.SetInsertPoint(BB
, BB
->getFirstInsertionPt());
2423 InsertPointTy AllocaIP
= Builder
.saveIP();
2425 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.applyWorkshareLoop(
2426 DL
, CLI
, AllocaIP
, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static
);
2427 assert(AfterIP
&& "unexpected error");
2429 BasicBlock
*Cond
= Body
->getSinglePredecessor();
2430 Instruction
*Cmp
= &*Cond
->begin();
2431 Value
*TripCount
= Cmp
->getOperand(1);
2433 auto AllocaIter
= BB
->begin();
2434 ASSERT_GE(std::distance(BB
->begin(), BB
->end()), 4);
2435 AllocaInst
*PLastIter
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2436 AllocaInst
*PLowerBound
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2437 AllocaInst
*PUpperBound
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2438 AllocaInst
*PStride
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2439 EXPECT_NE(PLastIter
, nullptr);
2440 EXPECT_NE(PLowerBound
, nullptr);
2441 EXPECT_NE(PUpperBound
, nullptr);
2442 EXPECT_NE(PStride
, nullptr);
2444 auto PreheaderIter
= Preheader
->begin();
2445 ASSERT_GE(std::distance(Preheader
->begin(), Preheader
->end()), 7);
2446 StoreInst
*LowerBoundStore
= dyn_cast
<StoreInst
>(&*(PreheaderIter
++));
2447 StoreInst
*UpperBoundStore
= dyn_cast
<StoreInst
>(&*(PreheaderIter
++));
2448 StoreInst
*StrideStore
= dyn_cast
<StoreInst
>(&*(PreheaderIter
++));
2449 ASSERT_NE(LowerBoundStore
, nullptr);
2450 ASSERT_NE(UpperBoundStore
, nullptr);
2451 ASSERT_NE(StrideStore
, nullptr);
2453 auto *OrigLowerBound
=
2454 dyn_cast
<ConstantInt
>(LowerBoundStore
->getValueOperand());
2455 auto *OrigUpperBound
=
2456 dyn_cast
<ConstantInt
>(UpperBoundStore
->getValueOperand());
2457 auto *OrigStride
= dyn_cast
<ConstantInt
>(StrideStore
->getValueOperand());
2458 ASSERT_NE(OrigLowerBound
, nullptr);
2459 ASSERT_NE(OrigUpperBound
, nullptr);
2460 ASSERT_NE(OrigStride
, nullptr);
2461 EXPECT_EQ(OrigLowerBound
->getValue(), 0);
2462 EXPECT_EQ(OrigUpperBound
->getValue(), 20);
2463 EXPECT_EQ(OrigStride
->getValue(), 1);
2465 // Check that the loop IV is updated to account for the lower bound returned
2466 // by the OpenMP runtime call.
2467 BinaryOperator
*Add
= dyn_cast
<BinaryOperator
>(&Body
->front());
2468 EXPECT_EQ(Add
->getOperand(0), IV
);
2469 auto *LoadedLowerBound
= dyn_cast
<LoadInst
>(Add
->getOperand(1));
2470 ASSERT_NE(LoadedLowerBound
, nullptr);
2471 EXPECT_EQ(LoadedLowerBound
->getPointerOperand(), PLowerBound
);
2473 // Check that the trip count is updated to account for the lower and upper
2474 // bounds return by the OpenMP runtime call.
2475 auto *AddOne
= dyn_cast
<Instruction
>(TripCount
);
2476 ASSERT_NE(AddOne
, nullptr);
2477 ASSERT_TRUE(AddOne
->isBinaryOp());
2478 auto *One
= dyn_cast
<ConstantInt
>(AddOne
->getOperand(1));
2479 ASSERT_NE(One
, nullptr);
2480 EXPECT_EQ(One
->getValue(), 1);
2481 auto *Difference
= dyn_cast
<Instruction
>(AddOne
->getOperand(0));
2482 ASSERT_NE(Difference
, nullptr);
2483 ASSERT_TRUE(Difference
->isBinaryOp());
2484 EXPECT_EQ(Difference
->getOperand(1), LoadedLowerBound
);
2485 auto *LoadedUpperBound
= dyn_cast
<LoadInst
>(Difference
->getOperand(0));
2486 ASSERT_NE(LoadedUpperBound
, nullptr);
2487 EXPECT_EQ(LoadedUpperBound
->getPointerOperand(), PUpperBound
);
2489 // The original loop iterator should only be used in the condition, in the
2490 // increment and in the statement that adds the lower bound to it.
2491 EXPECT_EQ(std::distance(IV
->use_begin(), IV
->use_end()), 3);
2493 // The exit block should contain the "fini" call and the barrier call,
2494 // plus the call to obtain the thread ID.
2495 size_t NumCallsInExitBlock
=
2496 count_if(*ExitBlock
, [](Instruction
&I
) { return isa
<CallInst
>(I
); });
2497 EXPECT_EQ(NumCallsInExitBlock
, 3u);
2500 TEST_P(OpenMPIRBuilderTestWithIVBits
, StaticChunkedWorkshareLoop
) {
2501 unsigned IVBits
= GetParam();
2503 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2504 OpenMPIRBuilder
OMPBuilder(*M
);
2505 OMPBuilder
.Config
.IsTargetDevice
= false;
2509 CanonicalLoopInfo
*CLI
=
2510 buildSingleLoopFunction(DL
, OMPBuilder
, IVBits
, &Call
, &Body
);
2512 Instruction
*OrigIndVar
= CLI
->getIndVar();
2513 EXPECT_EQ(Call
->getOperand(1), OrigIndVar
);
2515 Type
*LCTy
= Type::getInt32Ty(Ctx
);
2516 Value
*ChunkSize
= ConstantInt::get(LCTy
, 5);
2517 InsertPointTy AllocaIP
{&F
->getEntryBlock(),
2518 F
->getEntryBlock().getFirstInsertionPt()};
2519 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.applyWorkshareLoop(
2520 DL
, CLI
, AllocaIP
, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static
, ChunkSize
);
2521 assert(AfterIP
&& "unexpected error");
2523 OMPBuilder
.finalize();
2524 EXPECT_FALSE(verifyModule(*M
, &errs()));
2526 BasicBlock
*Entry
= &F
->getEntryBlock();
2527 BasicBlock
*Preheader
= Entry
->getSingleSuccessor();
2529 BasicBlock
*DispatchPreheader
= Preheader
->getSingleSuccessor();
2530 BasicBlock
*DispatchHeader
= DispatchPreheader
->getSingleSuccessor();
2531 BasicBlock
*DispatchCond
= DispatchHeader
->getSingleSuccessor();
2532 BasicBlock
*DispatchBody
= succ_begin(DispatchCond
)[0];
2533 BasicBlock
*DispatchExit
= succ_begin(DispatchCond
)[1];
2534 BasicBlock
*DispatchAfter
= DispatchExit
->getSingleSuccessor();
2535 BasicBlock
*Return
= DispatchAfter
->getSingleSuccessor();
2537 BasicBlock
*ChunkPreheader
= DispatchBody
->getSingleSuccessor();
2538 BasicBlock
*ChunkHeader
= ChunkPreheader
->getSingleSuccessor();
2539 BasicBlock
*ChunkCond
= ChunkHeader
->getSingleSuccessor();
2540 BasicBlock
*ChunkBody
= succ_begin(ChunkCond
)[0];
2541 BasicBlock
*ChunkExit
= succ_begin(ChunkCond
)[1];
2542 BasicBlock
*ChunkInc
= ChunkBody
->getSingleSuccessor();
2543 BasicBlock
*ChunkAfter
= ChunkExit
->getSingleSuccessor();
2545 BasicBlock
*DispatchInc
= ChunkAfter
;
2547 EXPECT_EQ(ChunkBody
, Body
);
2548 EXPECT_EQ(ChunkInc
->getSingleSuccessor(), ChunkHeader
);
2549 EXPECT_EQ(DispatchInc
->getSingleSuccessor(), DispatchHeader
);
2551 EXPECT_TRUE(isa
<ReturnInst
>(Return
->front()));
2553 Value
*NewIV
= Call
->getOperand(1);
2554 EXPECT_EQ(NewIV
->getType()->getScalarSizeInBits(), IVBits
);
2556 CallInst
*InitCall
= findSingleCall(
2558 (IVBits
> 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u
2559 : omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u
,
2561 EXPECT_EQ(InitCall
->getParent(), Preheader
);
2562 EXPECT_EQ(cast
<ConstantInt
>(InitCall
->getArgOperand(2))->getSExtValue(), 33);
2563 EXPECT_EQ(cast
<ConstantInt
>(InitCall
->getArgOperand(7))->getSExtValue(), 1);
2564 EXPECT_EQ(cast
<ConstantInt
>(InitCall
->getArgOperand(8))->getSExtValue(), 5);
2566 CallInst
*FiniCall
= findSingleCall(
2567 F
, omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini
, OMPBuilder
);
2568 EXPECT_EQ(FiniCall
->getParent(), DispatchExit
);
2570 CallInst
*BarrierCall
= findSingleCall(
2571 F
, omp::RuntimeFunction::OMPRTL___kmpc_barrier
, OMPBuilder
);
2572 EXPECT_EQ(BarrierCall
->getParent(), DispatchExit
);
2575 INSTANTIATE_TEST_SUITE_P(IVBits
, OpenMPIRBuilderTestWithIVBits
,
2576 ::testing::Values(8, 16, 32, 64));
2578 TEST_P(OpenMPIRBuilderTestWithParams
, DynamicWorkShareLoop
) {
2579 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2580 OpenMPIRBuilder
OMPBuilder(*M
);
2581 OMPBuilder
.Config
.IsTargetDevice
= false;
2582 OMPBuilder
.initialize();
2583 IRBuilder
<> Builder(BB
);
2584 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2586 omp::OMPScheduleType SchedType
= GetParam();
2587 uint32_t ChunkSize
= 1;
2588 switch (SchedType
& ~OMPScheduleType::ModifierMask
) {
2589 case omp::OMPScheduleType::BaseDynamicChunked
:
2590 case omp::OMPScheduleType::BaseGuidedChunked
:
2593 case omp::OMPScheduleType::BaseAuto
:
2594 case omp::OMPScheduleType::BaseRuntime
:
2598 assert(0 && "unknown type for this test");
2602 Type
*LCTy
= Type::getInt32Ty(Ctx
);
2603 Value
*StartVal
= ConstantInt::get(LCTy
, 10);
2604 Value
*StopVal
= ConstantInt::get(LCTy
, 52);
2605 Value
*StepVal
= ConstantInt::get(LCTy
, 2);
2607 (ChunkSize
== 1) ? nullptr : ConstantInt::get(LCTy
, ChunkSize
);
2608 auto LoopBodyGen
= [&](InsertPointTy
, llvm::Value
*) {
2609 return Error::success();
2612 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
2613 Loc
, LoopBodyGen
, StartVal
, StopVal
, StepVal
,
2614 /*IsSigned=*/false, /*InclusiveStop=*/false);
2615 assert(LoopResult
&& "unexpected error");
2616 CanonicalLoopInfo
*CLI
= *LoopResult
;
2618 Builder
.SetInsertPoint(BB
, BB
->getFirstInsertionPt());
2619 InsertPointTy AllocaIP
= Builder
.saveIP();
2621 // Collect all the info from CLI, as it isn't usable after the call to
2622 // createDynamicWorkshareLoop.
2623 InsertPointTy AfterIP
= CLI
->getAfterIP();
2624 BasicBlock
*Preheader
= CLI
->getPreheader();
2625 BasicBlock
*ExitBlock
= CLI
->getExit();
2626 BasicBlock
*LatchBlock
= CLI
->getLatch();
2627 Value
*IV
= CLI
->getIndVar();
2629 OpenMPIRBuilder::InsertPointOrErrorTy EndIP
= OMPBuilder
.applyWorkshareLoop(
2630 DL
, CLI
, AllocaIP
, /*NeedsBarrier=*/true, getSchedKind(SchedType
),
2631 ChunkVal
, /*Simd=*/false,
2632 (SchedType
& omp::OMPScheduleType::ModifierMonotonic
) ==
2633 omp::OMPScheduleType::ModifierMonotonic
,
2634 (SchedType
& omp::OMPScheduleType::ModifierNonmonotonic
) ==
2635 omp::OMPScheduleType::ModifierNonmonotonic
,
2637 assert(EndIP
&& "unexpected error");
2639 // The returned value should be the "after" point.
2640 ASSERT_EQ(EndIP
->getBlock(), AfterIP
.getBlock());
2641 ASSERT_EQ(EndIP
->getPoint(), AfterIP
.getPoint());
2643 auto AllocaIter
= BB
->begin();
2644 ASSERT_GE(std::distance(BB
->begin(), BB
->end()), 4);
2645 AllocaInst
*PLastIter
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2646 AllocaInst
*PLowerBound
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2647 AllocaInst
*PUpperBound
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2648 AllocaInst
*PStride
= dyn_cast
<AllocaInst
>(&*(AllocaIter
++));
2649 EXPECT_NE(PLastIter
, nullptr);
2650 EXPECT_NE(PLowerBound
, nullptr);
2651 EXPECT_NE(PUpperBound
, nullptr);
2652 EXPECT_NE(PStride
, nullptr);
2654 auto PreheaderIter
= Preheader
->begin();
2655 ASSERT_GE(std::distance(Preheader
->begin(), Preheader
->end()), 6);
2656 StoreInst
*LowerBoundStore
= dyn_cast
<StoreInst
>(&*(PreheaderIter
++));
2657 StoreInst
*UpperBoundStore
= dyn_cast
<StoreInst
>(&*(PreheaderIter
++));
2658 StoreInst
*StrideStore
= dyn_cast
<StoreInst
>(&*(PreheaderIter
++));
2659 ASSERT_NE(LowerBoundStore
, nullptr);
2660 ASSERT_NE(UpperBoundStore
, nullptr);
2661 ASSERT_NE(StrideStore
, nullptr);
2663 CallInst
*ThreadIdCall
= dyn_cast
<CallInst
>(&*(PreheaderIter
++));
2664 ASSERT_NE(ThreadIdCall
, nullptr);
2665 EXPECT_EQ(ThreadIdCall
->getCalledFunction()->getName(),
2666 "__kmpc_global_thread_num");
2668 CallInst
*InitCall
= dyn_cast
<CallInst
>(&*PreheaderIter
);
2670 ASSERT_NE(InitCall
, nullptr);
2671 EXPECT_EQ(InitCall
->getCalledFunction()->getName(),
2672 "__kmpc_dispatch_init_4u");
2673 EXPECT_EQ(InitCall
->arg_size(), 7U);
2674 EXPECT_EQ(InitCall
->getArgOperand(6), ConstantInt::get(LCTy
, ChunkSize
));
2675 ConstantInt
*SchedVal
= cast
<ConstantInt
>(InitCall
->getArgOperand(2));
2676 if ((SchedType
& OMPScheduleType::MonotonicityMask
) ==
2677 OMPScheduleType::None
) {
2678 // Implementation is allowed to add default nonmonotonicity flag
2680 static_cast<OMPScheduleType
>(SchedVal
->getValue().getZExtValue()) |
2681 OMPScheduleType::ModifierNonmonotonic
,
2682 SchedType
| OMPScheduleType::ModifierNonmonotonic
);
2684 EXPECT_EQ(static_cast<OMPScheduleType
>(SchedVal
->getValue().getZExtValue()),
2688 ConstantInt
*OrigLowerBound
=
2689 dyn_cast
<ConstantInt
>(LowerBoundStore
->getValueOperand());
2690 ConstantInt
*OrigUpperBound
=
2691 dyn_cast
<ConstantInt
>(UpperBoundStore
->getValueOperand());
2692 ConstantInt
*OrigStride
=
2693 dyn_cast
<ConstantInt
>(StrideStore
->getValueOperand());
2694 ASSERT_NE(OrigLowerBound
, nullptr);
2695 ASSERT_NE(OrigUpperBound
, nullptr);
2696 ASSERT_NE(OrigStride
, nullptr);
2697 EXPECT_EQ(OrigLowerBound
->getValue(), 1);
2698 EXPECT_EQ(OrigUpperBound
->getValue(), 21);
2699 EXPECT_EQ(OrigStride
->getValue(), 1);
2701 CallInst
*FiniCall
= dyn_cast
<CallInst
>(
2702 &*(LatchBlock
->getTerminator()->getPrevNonDebugInstruction(true)));
2703 EXPECT_EQ(FiniCall
, nullptr);
2705 // The original loop iterator should only be used in the condition, in the
2706 // increment and in the statement that adds the lower bound to it.
2707 EXPECT_EQ(std::distance(IV
->use_begin(), IV
->use_end()), 3);
2709 // The exit block should contain the barrier call, plus the call to obtain
2711 size_t NumCallsInExitBlock
=
2712 count_if(*ExitBlock
, [](Instruction
&I
) { return isa
<CallInst
>(I
); });
2713 EXPECT_EQ(NumCallsInExitBlock
, 2u);
2715 // Add a termination to our block and check that it is internally consistent.
2716 Builder
.restoreIP(*EndIP
);
2717 Builder
.CreateRetVoid();
2718 OMPBuilder
.finalize();
2719 EXPECT_FALSE(verifyModule(*M
, &errs()));
2722 INSTANTIATE_TEST_SUITE_P(
2723 OpenMPWSLoopSchedulingTypes
, OpenMPIRBuilderTestWithParams
,
2724 ::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked
,
2725 omp::OMPScheduleType::UnorderedGuidedChunked
,
2726 omp::OMPScheduleType::UnorderedAuto
,
2727 omp::OMPScheduleType::UnorderedRuntime
,
2728 omp::OMPScheduleType::UnorderedDynamicChunked
|
2729 omp::OMPScheduleType::ModifierMonotonic
,
2730 omp::OMPScheduleType::UnorderedDynamicChunked
|
2731 omp::OMPScheduleType::ModifierNonmonotonic
,
2732 omp::OMPScheduleType::UnorderedGuidedChunked
|
2733 omp::OMPScheduleType::ModifierMonotonic
,
2734 omp::OMPScheduleType::UnorderedGuidedChunked
|
2735 omp::OMPScheduleType::ModifierNonmonotonic
,
2736 omp::OMPScheduleType::UnorderedAuto
|
2737 omp::OMPScheduleType::ModifierMonotonic
,
2738 omp::OMPScheduleType::UnorderedRuntime
|
2739 omp::OMPScheduleType::ModifierMonotonic
));
2741 TEST_F(OpenMPIRBuilderTest
, DynamicWorkShareLoopOrdered
) {
2742 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2743 OpenMPIRBuilder
OMPBuilder(*M
);
2744 OMPBuilder
.Config
.IsTargetDevice
= false;
2745 OMPBuilder
.initialize();
2746 IRBuilder
<> Builder(BB
);
2747 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2749 uint32_t ChunkSize
= 1;
2750 Type
*LCTy
= Type::getInt32Ty(Ctx
);
2751 Value
*StartVal
= ConstantInt::get(LCTy
, 10);
2752 Value
*StopVal
= ConstantInt::get(LCTy
, 52);
2753 Value
*StepVal
= ConstantInt::get(LCTy
, 2);
2754 Value
*ChunkVal
= ConstantInt::get(LCTy
, ChunkSize
);
2755 auto LoopBodyGen
= [&](InsertPointTy
, llvm::Value
*) {
2756 return llvm::Error::success();
2759 Expected
<CanonicalLoopInfo
*> LoopResult
= OMPBuilder
.createCanonicalLoop(
2760 Loc
, LoopBodyGen
, StartVal
, StopVal
, StepVal
,
2761 /*IsSigned=*/false, /*InclusiveStop=*/false);
2762 assert(LoopResult
&& "unexpected error");
2763 CanonicalLoopInfo
*CLI
= *LoopResult
;
2765 Builder
.SetInsertPoint(BB
, BB
->getFirstInsertionPt());
2766 InsertPointTy AllocaIP
= Builder
.saveIP();
2768 // Collect all the info from CLI, as it isn't usable after the call to
2769 // createDynamicWorkshareLoop.
2770 BasicBlock
*Preheader
= CLI
->getPreheader();
2771 BasicBlock
*ExitBlock
= CLI
->getExit();
2772 BasicBlock
*LatchBlock
= CLI
->getLatch();
2773 Value
*IV
= CLI
->getIndVar();
2775 OpenMPIRBuilder::InsertPointOrErrorTy EndIP
= OMPBuilder
.applyWorkshareLoop(
2776 DL
, CLI
, AllocaIP
, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static
, ChunkVal
,
2777 /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
2778 /*HasNonmonotonicModifier=*/false,
2779 /*HasOrderedClause=*/true);
2780 assert(EndIP
&& "unexpected error");
2782 // Add a termination to our block and check that it is internally consistent.
2783 Builder
.restoreIP(*EndIP
);
2784 Builder
.CreateRetVoid();
2785 OMPBuilder
.finalize();
2786 EXPECT_FALSE(verifyModule(*M
, &errs()));
2788 CallInst
*InitCall
= nullptr;
2789 for (Instruction
&EI
: *Preheader
) {
2790 Instruction
*Cur
= &EI
;
2791 if (isa
<CallInst
>(Cur
)) {
2792 InitCall
= cast
<CallInst
>(Cur
);
2793 if (InitCall
->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")
2798 EXPECT_NE(InitCall
, nullptr);
2799 EXPECT_EQ(InitCall
->arg_size(), 7U);
2800 ConstantInt
*SchedVal
= cast
<ConstantInt
>(InitCall
->getArgOperand(2));
2801 EXPECT_EQ(SchedVal
->getValue(),
2802 static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked
));
2804 CallInst
*FiniCall
= dyn_cast
<CallInst
>(
2805 &*(LatchBlock
->getTerminator()->getPrevNonDebugInstruction(true)));
2806 ASSERT_NE(FiniCall
, nullptr);
2807 EXPECT_EQ(FiniCall
->getCalledFunction()->getName(),
2808 "__kmpc_dispatch_fini_4u");
2809 EXPECT_EQ(FiniCall
->arg_size(), 2U);
2810 EXPECT_EQ(InitCall
->getArgOperand(0), FiniCall
->getArgOperand(0));
2811 EXPECT_EQ(InitCall
->getArgOperand(1), FiniCall
->getArgOperand(1));
2813 // The original loop iterator should only be used in the condition, in the
2814 // increment and in the statement that adds the lower bound to it.
2815 EXPECT_EQ(std::distance(IV
->use_begin(), IV
->use_end()), 3);
2817 // The exit block should contain the barrier call, plus the call to obtain
2819 size_t NumCallsInExitBlock
=
2820 count_if(*ExitBlock
, [](Instruction
&I
) { return isa
<CallInst
>(I
); });
2821 EXPECT_EQ(NumCallsInExitBlock
, 2u);
2824 TEST_F(OpenMPIRBuilderTest
, MasterDirective
) {
2825 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2826 OpenMPIRBuilder
OMPBuilder(*M
);
2827 OMPBuilder
.initialize();
2829 IRBuilder
<> Builder(BB
);
2831 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2833 AllocaInst
*PrivAI
= nullptr;
2835 BasicBlock
*EntryBB
= nullptr;
2836 BasicBlock
*ThenBB
= nullptr;
2838 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
2839 if (AllocaIP
.isSet())
2840 Builder
.restoreIP(AllocaIP
);
2842 Builder
.SetInsertPoint(&*(F
->getEntryBlock().getFirstInsertionPt()));
2843 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
2844 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
2846 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
2847 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
2848 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
2850 Builder
.restoreIP(CodeGenIP
);
2852 // collect some info for checks later
2853 ThenBB
= Builder
.GetInsertBlock();
2854 EntryBB
= ThenBB
->getUniquePredecessor();
2856 // simple instructions for body
2858 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
2859 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
2862 auto FiniCB
= [&](InsertPointTy IP
) {
2863 BasicBlock
*IPBB
= IP
.getBlock();
2864 EXPECT_NE(IPBB
->end(), IP
.getPoint());
2867 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createMaster(
2868 Builder
, BODYGENCB_WRAPPER(BodyGenCB
), FINICB_WRAPPER(FiniCB
));
2869 assert(AfterIP
&& "unexpected error");
2870 Builder
.restoreIP(*AfterIP
);
2871 Value
*EntryBBTI
= EntryBB
->getTerminator();
2872 EXPECT_NE(EntryBBTI
, nullptr);
2873 EXPECT_TRUE(isa
<BranchInst
>(EntryBBTI
));
2874 BranchInst
*EntryBr
= cast
<BranchInst
>(EntryBB
->getTerminator());
2875 EXPECT_TRUE(EntryBr
->isConditional());
2876 EXPECT_EQ(EntryBr
->getSuccessor(0), ThenBB
);
2877 BasicBlock
*ExitBB
= ThenBB
->getUniqueSuccessor();
2878 EXPECT_EQ(EntryBr
->getSuccessor(1), ExitBB
);
2880 CmpInst
*CondInst
= cast
<CmpInst
>(EntryBr
->getCondition());
2881 EXPECT_TRUE(isa
<CallInst
>(CondInst
->getOperand(0)));
2883 CallInst
*MasterEntryCI
= cast
<CallInst
>(CondInst
->getOperand(0));
2884 EXPECT_EQ(MasterEntryCI
->arg_size(), 2U);
2885 EXPECT_EQ(MasterEntryCI
->getCalledFunction()->getName(), "__kmpc_master");
2886 EXPECT_TRUE(isa
<GlobalVariable
>(MasterEntryCI
->getArgOperand(0)));
2888 CallInst
*MasterEndCI
= nullptr;
2889 for (auto &FI
: *ThenBB
) {
2890 Instruction
*cur
= &FI
;
2891 if (isa
<CallInst
>(cur
)) {
2892 MasterEndCI
= cast
<CallInst
>(cur
);
2893 if (MasterEndCI
->getCalledFunction()->getName() == "__kmpc_end_master")
2895 MasterEndCI
= nullptr;
2898 EXPECT_NE(MasterEndCI
, nullptr);
2899 EXPECT_EQ(MasterEndCI
->arg_size(), 2U);
2900 EXPECT_TRUE(isa
<GlobalVariable
>(MasterEndCI
->getArgOperand(0)));
2901 EXPECT_EQ(MasterEndCI
->getArgOperand(1), MasterEntryCI
->getArgOperand(1));
2904 TEST_F(OpenMPIRBuilderTest
, MaskedDirective
) {
2905 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2906 OpenMPIRBuilder
OMPBuilder(*M
);
2907 OMPBuilder
.initialize();
2909 IRBuilder
<> Builder(BB
);
2911 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2913 AllocaInst
*PrivAI
= nullptr;
2915 BasicBlock
*EntryBB
= nullptr;
2916 BasicBlock
*ThenBB
= nullptr;
2918 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
2919 if (AllocaIP
.isSet())
2920 Builder
.restoreIP(AllocaIP
);
2922 Builder
.SetInsertPoint(&*(F
->getEntryBlock().getFirstInsertionPt()));
2923 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
2924 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
2926 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
2927 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
2928 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
2930 Builder
.restoreIP(CodeGenIP
);
2932 // collect some info for checks later
2933 ThenBB
= Builder
.GetInsertBlock();
2934 EntryBB
= ThenBB
->getUniquePredecessor();
2936 // simple instructions for body
2938 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
2939 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
2942 auto FiniCB
= [&](InsertPointTy IP
) {
2943 BasicBlock
*IPBB
= IP
.getBlock();
2944 EXPECT_NE(IPBB
->end(), IP
.getPoint());
2947 Constant
*Filter
= ConstantInt::get(Type::getInt32Ty(M
->getContext()), 0);
2948 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createMasked(
2949 Builder
, BODYGENCB_WRAPPER(BodyGenCB
), FINICB_WRAPPER(FiniCB
), Filter
);
2950 assert(AfterIP
&& "unexpected error");
2951 Builder
.restoreIP(*AfterIP
);
2952 Value
*EntryBBTI
= EntryBB
->getTerminator();
2953 EXPECT_NE(EntryBBTI
, nullptr);
2954 EXPECT_TRUE(isa
<BranchInst
>(EntryBBTI
));
2955 BranchInst
*EntryBr
= cast
<BranchInst
>(EntryBB
->getTerminator());
2956 EXPECT_TRUE(EntryBr
->isConditional());
2957 EXPECT_EQ(EntryBr
->getSuccessor(0), ThenBB
);
2958 BasicBlock
*ExitBB
= ThenBB
->getUniqueSuccessor();
2959 EXPECT_EQ(EntryBr
->getSuccessor(1), ExitBB
);
2961 CmpInst
*CondInst
= cast
<CmpInst
>(EntryBr
->getCondition());
2962 EXPECT_TRUE(isa
<CallInst
>(CondInst
->getOperand(0)));
2964 CallInst
*MaskedEntryCI
= cast
<CallInst
>(CondInst
->getOperand(0));
2965 EXPECT_EQ(MaskedEntryCI
->arg_size(), 3U);
2966 EXPECT_EQ(MaskedEntryCI
->getCalledFunction()->getName(), "__kmpc_masked");
2967 EXPECT_TRUE(isa
<GlobalVariable
>(MaskedEntryCI
->getArgOperand(0)));
2969 CallInst
*MaskedEndCI
= nullptr;
2970 for (auto &FI
: *ThenBB
) {
2971 Instruction
*cur
= &FI
;
2972 if (isa
<CallInst
>(cur
)) {
2973 MaskedEndCI
= cast
<CallInst
>(cur
);
2974 if (MaskedEndCI
->getCalledFunction()->getName() == "__kmpc_end_masked")
2976 MaskedEndCI
= nullptr;
2979 EXPECT_NE(MaskedEndCI
, nullptr);
2980 EXPECT_EQ(MaskedEndCI
->arg_size(), 2U);
2981 EXPECT_TRUE(isa
<GlobalVariable
>(MaskedEndCI
->getArgOperand(0)));
2982 EXPECT_EQ(MaskedEndCI
->getArgOperand(1), MaskedEntryCI
->getArgOperand(1));
2985 TEST_F(OpenMPIRBuilderTest
, CriticalDirective
) {
2986 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
2987 OpenMPIRBuilder
OMPBuilder(*M
);
2988 OMPBuilder
.initialize();
2990 IRBuilder
<> Builder(BB
);
2992 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
2994 AllocaInst
*PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
2996 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
2997 // actual start for bodyCB
2998 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
2999 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
3000 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
3003 Builder
.restoreIP(CodeGenIP
);
3004 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
3006 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
3007 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
3010 auto FiniCB
= [&](InsertPointTy IP
) {
3011 BasicBlock
*IPBB
= IP
.getBlock();
3012 EXPECT_NE(IPBB
->end(), IP
.getPoint());
3014 BasicBlock
*EntryBB
= Builder
.GetInsertBlock();
3016 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
3017 OMPBuilder
.createCritical(Builder
, BODYGENCB_WRAPPER(BodyGenCB
),
3018 FINICB_WRAPPER(FiniCB
), "testCRT", nullptr);
3019 assert(AfterIP
&& "unexpected error");
3020 Builder
.restoreIP(*AfterIP
);
3022 CallInst
*CriticalEntryCI
= nullptr;
3023 for (auto &EI
: *EntryBB
) {
3024 Instruction
*cur
= &EI
;
3025 if (isa
<CallInst
>(cur
)) {
3026 CriticalEntryCI
= cast
<CallInst
>(cur
);
3027 if (CriticalEntryCI
->getCalledFunction()->getName() == "__kmpc_critical")
3029 CriticalEntryCI
= nullptr;
3032 EXPECT_NE(CriticalEntryCI
, nullptr);
3033 EXPECT_EQ(CriticalEntryCI
->arg_size(), 3U);
3034 EXPECT_EQ(CriticalEntryCI
->getCalledFunction()->getName(), "__kmpc_critical");
3035 EXPECT_TRUE(isa
<GlobalVariable
>(CriticalEntryCI
->getArgOperand(0)));
3037 CallInst
*CriticalEndCI
= nullptr;
3038 for (auto &FI
: *EntryBB
) {
3039 Instruction
*cur
= &FI
;
3040 if (isa
<CallInst
>(cur
)) {
3041 CriticalEndCI
= cast
<CallInst
>(cur
);
3042 if (CriticalEndCI
->getCalledFunction()->getName() ==
3043 "__kmpc_end_critical")
3045 CriticalEndCI
= nullptr;
3048 EXPECT_NE(CriticalEndCI
, nullptr);
3049 EXPECT_EQ(CriticalEndCI
->arg_size(), 3U);
3050 EXPECT_TRUE(isa
<GlobalVariable
>(CriticalEndCI
->getArgOperand(0)));
3051 EXPECT_EQ(CriticalEndCI
->getArgOperand(1), CriticalEntryCI
->getArgOperand(1));
3052 PointerType
*CriticalNamePtrTy
=
3053 PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx
), 8));
3054 EXPECT_EQ(CriticalEndCI
->getArgOperand(2), CriticalEntryCI
->getArgOperand(2));
3055 GlobalVariable
*GV
=
3056 dyn_cast
<GlobalVariable
>(CriticalEndCI
->getArgOperand(2));
3057 ASSERT_NE(GV
, nullptr);
3058 EXPECT_EQ(GV
->getType(), CriticalNamePtrTy
);
3059 const DataLayout
&DL
= M
->getDataLayout();
3060 const llvm::Align TypeAlign
= DL
.getABITypeAlign(CriticalNamePtrTy
);
3061 const llvm::Align PtrAlign
= DL
.getPointerABIAlignment(GV
->getAddressSpace());
3062 if (const llvm::MaybeAlign Alignment
= GV
->getAlign())
3063 EXPECT_EQ(*Alignment
, std::max(TypeAlign
, PtrAlign
));
3066 TEST_F(OpenMPIRBuilderTest
, OrderedDirectiveDependSource
) {
3067 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3068 OpenMPIRBuilder
OMPBuilder(*M
);
3069 OMPBuilder
.initialize();
3071 IRBuilder
<> Builder(BB
);
3072 LLVMContext
&Ctx
= M
->getContext();
3074 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3076 InsertPointTy
AllocaIP(&F
->getEntryBlock(),
3077 F
->getEntryBlock().getFirstInsertionPt());
3079 unsigned NumLoops
= 2;
3080 SmallVector
<Value
*, 2> StoreValues
;
3081 Type
*LCTy
= Type::getInt64Ty(Ctx
);
3082 StoreValues
.emplace_back(ConstantInt::get(LCTy
, 1));
3083 StoreValues
.emplace_back(ConstantInt::get(LCTy
, 2));
3085 // Test for "#omp ordered depend(source)"
3086 Builder
.restoreIP(OMPBuilder
.createOrderedDepend(Builder
, AllocaIP
, NumLoops
,
3087 StoreValues
, ".cnt.addr",
3088 /*IsDependSource=*/true));
3090 Builder
.CreateRetVoid();
3091 OMPBuilder
.finalize();
3092 EXPECT_FALSE(verifyModule(*M
, &errs()));
3094 AllocaInst
*AllocInst
= dyn_cast
<AllocaInst
>(&BB
->front());
3095 ASSERT_NE(AllocInst
, nullptr);
3096 ArrayType
*ArrType
= dyn_cast
<ArrayType
>(AllocInst
->getAllocatedType());
3097 EXPECT_EQ(ArrType
->getNumElements(), NumLoops
);
3099 AllocInst
->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
3101 Instruction
*IterInst
= dyn_cast
<Instruction
>(AllocInst
);
3102 for (unsigned Iter
= 0; Iter
< NumLoops
; Iter
++) {
3103 GetElementPtrInst
*DependAddrGEPIter
=
3104 dyn_cast
<GetElementPtrInst
>(IterInst
->getNextNode());
3105 ASSERT_NE(DependAddrGEPIter
, nullptr);
3106 EXPECT_EQ(DependAddrGEPIter
->getPointerOperand(), AllocInst
);
3107 EXPECT_EQ(DependAddrGEPIter
->getNumIndices(), (unsigned)2);
3108 auto *FirstIdx
= dyn_cast
<ConstantInt
>(DependAddrGEPIter
->getOperand(1));
3109 auto *SecondIdx
= dyn_cast
<ConstantInt
>(DependAddrGEPIter
->getOperand(2));
3110 ASSERT_NE(FirstIdx
, nullptr);
3111 ASSERT_NE(SecondIdx
, nullptr);
3112 EXPECT_EQ(FirstIdx
->getValue(), 0);
3113 EXPECT_EQ(SecondIdx
->getValue(), Iter
);
3114 StoreInst
*StoreValue
=
3115 dyn_cast
<StoreInst
>(DependAddrGEPIter
->getNextNode());
3116 ASSERT_NE(StoreValue
, nullptr);
3117 EXPECT_EQ(StoreValue
->getValueOperand(), StoreValues
[Iter
]);
3118 EXPECT_EQ(StoreValue
->getPointerOperand(), DependAddrGEPIter
);
3119 EXPECT_EQ(StoreValue
->getAlign(), Align(8));
3120 IterInst
= dyn_cast
<Instruction
>(StoreValue
);
3123 GetElementPtrInst
*DependBaseAddrGEP
=
3124 dyn_cast
<GetElementPtrInst
>(IterInst
->getNextNode());
3125 ASSERT_NE(DependBaseAddrGEP
, nullptr);
3126 EXPECT_EQ(DependBaseAddrGEP
->getPointerOperand(), AllocInst
);
3127 EXPECT_EQ(DependBaseAddrGEP
->getNumIndices(), (unsigned)2);
3128 auto *FirstIdx
= dyn_cast
<ConstantInt
>(DependBaseAddrGEP
->getOperand(1));
3129 auto *SecondIdx
= dyn_cast
<ConstantInt
>(DependBaseAddrGEP
->getOperand(2));
3130 ASSERT_NE(FirstIdx
, nullptr);
3131 ASSERT_NE(SecondIdx
, nullptr);
3132 EXPECT_EQ(FirstIdx
->getValue(), 0);
3133 EXPECT_EQ(SecondIdx
->getValue(), 0);
3135 CallInst
*GTID
= dyn_cast
<CallInst
>(DependBaseAddrGEP
->getNextNode());
3136 ASSERT_NE(GTID
, nullptr);
3137 EXPECT_EQ(GTID
->arg_size(), 1U);
3138 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
3139 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotAccessMemory());
3140 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotFreeMemory());
3142 CallInst
*Depend
= dyn_cast
<CallInst
>(GTID
->getNextNode());
3143 ASSERT_NE(Depend
, nullptr);
3144 EXPECT_EQ(Depend
->arg_size(), 3U);
3145 EXPECT_EQ(Depend
->getCalledFunction()->getName(), "__kmpc_doacross_post");
3146 EXPECT_TRUE(isa
<GlobalVariable
>(Depend
->getArgOperand(0)));
3147 EXPECT_EQ(Depend
->getArgOperand(1), GTID
);
3148 EXPECT_EQ(Depend
->getArgOperand(2), DependBaseAddrGEP
);
3151 TEST_F(OpenMPIRBuilderTest
, OrderedDirectiveDependSink
) {
3152 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3153 OpenMPIRBuilder
OMPBuilder(*M
);
3154 OMPBuilder
.initialize();
3156 IRBuilder
<> Builder(BB
);
3157 LLVMContext
&Ctx
= M
->getContext();
3159 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3161 InsertPointTy
AllocaIP(&F
->getEntryBlock(),
3162 F
->getEntryBlock().getFirstInsertionPt());
3164 unsigned NumLoops
= 2;
3165 SmallVector
<Value
*, 2> StoreValues
;
3166 Type
*LCTy
= Type::getInt64Ty(Ctx
);
3167 StoreValues
.emplace_back(ConstantInt::get(LCTy
, 1));
3168 StoreValues
.emplace_back(ConstantInt::get(LCTy
, 2));
3170 // Test for "#omp ordered depend(sink: vec)"
3171 Builder
.restoreIP(OMPBuilder
.createOrderedDepend(Builder
, AllocaIP
, NumLoops
,
3172 StoreValues
, ".cnt.addr",
3173 /*IsDependSource=*/false));
3175 Builder
.CreateRetVoid();
3176 OMPBuilder
.finalize();
3177 EXPECT_FALSE(verifyModule(*M
, &errs()));
3179 AllocaInst
*AllocInst
= dyn_cast
<AllocaInst
>(&BB
->front());
3180 ASSERT_NE(AllocInst
, nullptr);
3181 ArrayType
*ArrType
= dyn_cast
<ArrayType
>(AllocInst
->getAllocatedType());
3182 EXPECT_EQ(ArrType
->getNumElements(), NumLoops
);
3184 AllocInst
->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
3186 Instruction
*IterInst
= dyn_cast
<Instruction
>(AllocInst
);
3187 for (unsigned Iter
= 0; Iter
< NumLoops
; Iter
++) {
3188 GetElementPtrInst
*DependAddrGEPIter
=
3189 dyn_cast
<GetElementPtrInst
>(IterInst
->getNextNode());
3190 ASSERT_NE(DependAddrGEPIter
, nullptr);
3191 EXPECT_EQ(DependAddrGEPIter
->getPointerOperand(), AllocInst
);
3192 EXPECT_EQ(DependAddrGEPIter
->getNumIndices(), (unsigned)2);
3193 auto *FirstIdx
= dyn_cast
<ConstantInt
>(DependAddrGEPIter
->getOperand(1));
3194 auto *SecondIdx
= dyn_cast
<ConstantInt
>(DependAddrGEPIter
->getOperand(2));
3195 ASSERT_NE(FirstIdx
, nullptr);
3196 ASSERT_NE(SecondIdx
, nullptr);
3197 EXPECT_EQ(FirstIdx
->getValue(), 0);
3198 EXPECT_EQ(SecondIdx
->getValue(), Iter
);
3199 StoreInst
*StoreValue
=
3200 dyn_cast
<StoreInst
>(DependAddrGEPIter
->getNextNode());
3201 ASSERT_NE(StoreValue
, nullptr);
3202 EXPECT_EQ(StoreValue
->getValueOperand(), StoreValues
[Iter
]);
3203 EXPECT_EQ(StoreValue
->getPointerOperand(), DependAddrGEPIter
);
3204 EXPECT_EQ(StoreValue
->getAlign(), Align(8));
3205 IterInst
= dyn_cast
<Instruction
>(StoreValue
);
3208 GetElementPtrInst
*DependBaseAddrGEP
=
3209 dyn_cast
<GetElementPtrInst
>(IterInst
->getNextNode());
3210 ASSERT_NE(DependBaseAddrGEP
, nullptr);
3211 EXPECT_EQ(DependBaseAddrGEP
->getPointerOperand(), AllocInst
);
3212 EXPECT_EQ(DependBaseAddrGEP
->getNumIndices(), (unsigned)2);
3213 auto *FirstIdx
= dyn_cast
<ConstantInt
>(DependBaseAddrGEP
->getOperand(1));
3214 auto *SecondIdx
= dyn_cast
<ConstantInt
>(DependBaseAddrGEP
->getOperand(2));
3215 ASSERT_NE(FirstIdx
, nullptr);
3216 ASSERT_NE(SecondIdx
, nullptr);
3217 EXPECT_EQ(FirstIdx
->getValue(), 0);
3218 EXPECT_EQ(SecondIdx
->getValue(), 0);
3220 CallInst
*GTID
= dyn_cast
<CallInst
>(DependBaseAddrGEP
->getNextNode());
3221 ASSERT_NE(GTID
, nullptr);
3222 EXPECT_EQ(GTID
->arg_size(), 1U);
3223 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
3224 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotAccessMemory());
3225 EXPECT_FALSE(GTID
->getCalledFunction()->doesNotFreeMemory());
3227 CallInst
*Depend
= dyn_cast
<CallInst
>(GTID
->getNextNode());
3228 ASSERT_NE(Depend
, nullptr);
3229 EXPECT_EQ(Depend
->arg_size(), 3U);
3230 EXPECT_EQ(Depend
->getCalledFunction()->getName(), "__kmpc_doacross_wait");
3231 EXPECT_TRUE(isa
<GlobalVariable
>(Depend
->getArgOperand(0)));
3232 EXPECT_EQ(Depend
->getArgOperand(1), GTID
);
3233 EXPECT_EQ(Depend
->getArgOperand(2), DependBaseAddrGEP
);
3236 TEST_F(OpenMPIRBuilderTest
, OrderedDirectiveThreads
) {
3237 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3238 OpenMPIRBuilder
OMPBuilder(*M
);
3239 OMPBuilder
.initialize();
3241 IRBuilder
<> Builder(BB
);
3243 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3245 AllocaInst
*PrivAI
=
3246 Builder
.CreateAlloca(F
->arg_begin()->getType(), nullptr, "priv.inst");
3248 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
3249 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
3250 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
3251 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
3253 Builder
.restoreIP(CodeGenIP
);
3254 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
3256 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
3257 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
3260 auto FiniCB
= [&](InsertPointTy IP
) {
3261 BasicBlock
*IPBB
= IP
.getBlock();
3262 EXPECT_NE(IPBB
->end(), IP
.getPoint());
3265 // Test for "#omp ordered [threads]"
3266 BasicBlock
*EntryBB
= Builder
.GetInsertBlock();
3267 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
3268 OMPBuilder
.createOrderedThreadsSimd(Builder
, BODYGENCB_WRAPPER(BodyGenCB
),
3269 FINICB_WRAPPER(FiniCB
), true);
3270 assert(AfterIP
&& "unexpected error");
3271 Builder
.restoreIP(*AfterIP
);
3273 Builder
.CreateRetVoid();
3274 OMPBuilder
.finalize();
3275 EXPECT_FALSE(verifyModule(*M
, &errs()));
3277 EXPECT_NE(EntryBB
->getTerminator(), nullptr);
3279 CallInst
*OrderedEntryCI
= nullptr;
3280 for (auto &EI
: *EntryBB
) {
3281 Instruction
*Cur
= &EI
;
3282 if (isa
<CallInst
>(Cur
)) {
3283 OrderedEntryCI
= cast
<CallInst
>(Cur
);
3284 if (OrderedEntryCI
->getCalledFunction()->getName() == "__kmpc_ordered")
3286 OrderedEntryCI
= nullptr;
3289 EXPECT_NE(OrderedEntryCI
, nullptr);
3290 EXPECT_EQ(OrderedEntryCI
->arg_size(), 2U);
3291 EXPECT_EQ(OrderedEntryCI
->getCalledFunction()->getName(), "__kmpc_ordered");
3292 EXPECT_TRUE(isa
<GlobalVariable
>(OrderedEntryCI
->getArgOperand(0)));
3294 CallInst
*OrderedEndCI
= nullptr;
3295 for (auto &FI
: *EntryBB
) {
3296 Instruction
*Cur
= &FI
;
3297 if (isa
<CallInst
>(Cur
)) {
3298 OrderedEndCI
= cast
<CallInst
>(Cur
);
3299 if (OrderedEndCI
->getCalledFunction()->getName() == "__kmpc_end_ordered")
3301 OrderedEndCI
= nullptr;
3304 EXPECT_NE(OrderedEndCI
, nullptr);
3305 EXPECT_EQ(OrderedEndCI
->arg_size(), 2U);
3306 EXPECT_TRUE(isa
<GlobalVariable
>(OrderedEndCI
->getArgOperand(0)));
3307 EXPECT_EQ(OrderedEndCI
->getArgOperand(1), OrderedEntryCI
->getArgOperand(1));
3310 TEST_F(OpenMPIRBuilderTest
, OrderedDirectiveSimd
) {
3311 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3312 OpenMPIRBuilder
OMPBuilder(*M
);
3313 OMPBuilder
.initialize();
3315 IRBuilder
<> Builder(BB
);
3317 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3319 AllocaInst
*PrivAI
=
3320 Builder
.CreateAlloca(F
->arg_begin()->getType(), nullptr, "priv.inst");
3322 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
3323 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
3324 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
3325 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
3327 Builder
.restoreIP(CodeGenIP
);
3328 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
3330 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
3331 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
3334 auto FiniCB
= [&](InsertPointTy IP
) {
3335 BasicBlock
*IPBB
= IP
.getBlock();
3336 EXPECT_NE(IPBB
->end(), IP
.getPoint());
3339 // Test for "#omp ordered simd"
3340 BasicBlock
*EntryBB
= Builder
.GetInsertBlock();
3341 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
3342 OMPBuilder
.createOrderedThreadsSimd(Builder
, BODYGENCB_WRAPPER(BodyGenCB
),
3343 FINICB_WRAPPER(FiniCB
), false);
3344 assert(AfterIP
&& "unexpected error");
3345 Builder
.restoreIP(*AfterIP
);
3347 Builder
.CreateRetVoid();
3348 OMPBuilder
.finalize();
3349 EXPECT_FALSE(verifyModule(*M
, &errs()));
3351 EXPECT_NE(EntryBB
->getTerminator(), nullptr);
3353 CallInst
*OrderedEntryCI
= nullptr;
3354 for (auto &EI
: *EntryBB
) {
3355 Instruction
*Cur
= &EI
;
3356 if (isa
<CallInst
>(Cur
)) {
3357 OrderedEntryCI
= cast
<CallInst
>(Cur
);
3358 if (OrderedEntryCI
->getCalledFunction()->getName() == "__kmpc_ordered")
3360 OrderedEntryCI
= nullptr;
3363 EXPECT_EQ(OrderedEntryCI
, nullptr);
3365 CallInst
*OrderedEndCI
= nullptr;
3366 for (auto &FI
: *EntryBB
) {
3367 Instruction
*Cur
= &FI
;
3368 if (isa
<CallInst
>(Cur
)) {
3369 OrderedEndCI
= cast
<CallInst
>(Cur
);
3370 if (OrderedEndCI
->getCalledFunction()->getName() == "__kmpc_end_ordered")
3372 OrderedEndCI
= nullptr;
3375 EXPECT_EQ(OrderedEndCI
, nullptr);
3378 TEST_F(OpenMPIRBuilderTest
, CopyinBlocks
) {
3379 OpenMPIRBuilder
OMPBuilder(*M
);
3380 OMPBuilder
.initialize();
3382 IRBuilder
<> Builder(BB
);
3384 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3386 IntegerType
*Int32
= Type::getInt32Ty(M
->getContext());
3387 AllocaInst
*MasterAddress
= Builder
.CreateAlloca(Builder
.getPtrTy());
3388 AllocaInst
*PrivAddress
= Builder
.CreateAlloca(Builder
.getPtrTy());
3390 BasicBlock
*EntryBB
= BB
;
3392 OMPBuilder
.createCopyinClauseBlocks(Builder
.saveIP(), MasterAddress
,
3393 PrivAddress
, Int32
, /*BranchtoEnd*/ true);
3395 BranchInst
*EntryBr
= dyn_cast_or_null
<BranchInst
>(EntryBB
->getTerminator());
3397 EXPECT_NE(EntryBr
, nullptr);
3398 EXPECT_TRUE(EntryBr
->isConditional());
3400 BasicBlock
*NotMasterBB
= EntryBr
->getSuccessor(0);
3401 BasicBlock
*CopyinEnd
= EntryBr
->getSuccessor(1);
3402 CmpInst
*CMP
= dyn_cast_or_null
<CmpInst
>(EntryBr
->getCondition());
3404 EXPECT_NE(CMP
, nullptr);
3405 EXPECT_NE(NotMasterBB
, nullptr);
3406 EXPECT_NE(CopyinEnd
, nullptr);
3408 BranchInst
*NotMasterBr
=
3409 dyn_cast_or_null
<BranchInst
>(NotMasterBB
->getTerminator());
3410 EXPECT_NE(NotMasterBr
, nullptr);
3411 EXPECT_FALSE(NotMasterBr
->isConditional());
3412 EXPECT_EQ(CopyinEnd
, NotMasterBr
->getSuccessor(0));
3415 TEST_F(OpenMPIRBuilderTest
, SingleDirective
) {
3416 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3417 OpenMPIRBuilder
OMPBuilder(*M
);
3418 OMPBuilder
.initialize();
3420 IRBuilder
<> Builder(BB
);
3422 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3424 AllocaInst
*PrivAI
= nullptr;
3426 BasicBlock
*EntryBB
= nullptr;
3427 BasicBlock
*ThenBB
= nullptr;
3429 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
3430 if (AllocaIP
.isSet())
3431 Builder
.restoreIP(AllocaIP
);
3433 Builder
.SetInsertPoint(&*(F
->getEntryBlock().getFirstInsertionPt()));
3434 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
3435 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
3437 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
3438 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
3439 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
3441 Builder
.restoreIP(CodeGenIP
);
3443 // collect some info for checks later
3444 ThenBB
= Builder
.GetInsertBlock();
3445 EntryBB
= ThenBB
->getUniquePredecessor();
3447 // simple instructions for body
3449 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
3450 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
3453 auto FiniCB
= [&](InsertPointTy IP
) {
3454 BasicBlock
*IPBB
= IP
.getBlock();
3455 EXPECT_NE(IPBB
->end(), IP
.getPoint());
3458 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
3459 OMPBuilder
.createSingle(Builder
, BODYGENCB_WRAPPER(BodyGenCB
),
3460 FINICB_WRAPPER(FiniCB
), /*IsNowait*/ false);
3461 assert(AfterIP
&& "unexpected error");
3462 Builder
.restoreIP(*AfterIP
);
3463 Value
*EntryBBTI
= EntryBB
->getTerminator();
3464 EXPECT_NE(EntryBBTI
, nullptr);
3465 EXPECT_TRUE(isa
<BranchInst
>(EntryBBTI
));
3466 BranchInst
*EntryBr
= cast
<BranchInst
>(EntryBB
->getTerminator());
3467 EXPECT_TRUE(EntryBr
->isConditional());
3468 EXPECT_EQ(EntryBr
->getSuccessor(0), ThenBB
);
3469 BasicBlock
*ExitBB
= ThenBB
->getUniqueSuccessor();
3470 EXPECT_EQ(EntryBr
->getSuccessor(1), ExitBB
);
3472 CmpInst
*CondInst
= cast
<CmpInst
>(EntryBr
->getCondition());
3473 EXPECT_TRUE(isa
<CallInst
>(CondInst
->getOperand(0)));
3475 CallInst
*SingleEntryCI
= cast
<CallInst
>(CondInst
->getOperand(0));
3476 EXPECT_EQ(SingleEntryCI
->arg_size(), 2U);
3477 EXPECT_EQ(SingleEntryCI
->getCalledFunction()->getName(), "__kmpc_single");
3478 EXPECT_TRUE(isa
<GlobalVariable
>(SingleEntryCI
->getArgOperand(0)));
3480 CallInst
*SingleEndCI
= nullptr;
3481 for (auto &FI
: *ThenBB
) {
3482 Instruction
*cur
= &FI
;
3483 if (isa
<CallInst
>(cur
)) {
3484 SingleEndCI
= cast
<CallInst
>(cur
);
3485 if (SingleEndCI
->getCalledFunction()->getName() == "__kmpc_end_single")
3487 SingleEndCI
= nullptr;
3490 EXPECT_NE(SingleEndCI
, nullptr);
3491 EXPECT_EQ(SingleEndCI
->arg_size(), 2U);
3492 EXPECT_TRUE(isa
<GlobalVariable
>(SingleEndCI
->getArgOperand(0)));
3493 EXPECT_EQ(SingleEndCI
->getArgOperand(1), SingleEntryCI
->getArgOperand(1));
3495 bool FoundBarrier
= false;
3496 for (auto &FI
: *ExitBB
) {
3497 Instruction
*cur
= &FI
;
3498 if (auto CI
= dyn_cast
<CallInst
>(cur
)) {
3499 if (CI
->getCalledFunction()->getName() == "__kmpc_barrier") {
3500 FoundBarrier
= true;
3505 EXPECT_TRUE(FoundBarrier
);
3508 TEST_F(OpenMPIRBuilderTest
, SingleDirectiveNowait
) {
3509 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3510 OpenMPIRBuilder
OMPBuilder(*M
);
3511 OMPBuilder
.initialize();
3513 IRBuilder
<> Builder(BB
);
3515 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3517 AllocaInst
*PrivAI
= nullptr;
3519 BasicBlock
*EntryBB
= nullptr;
3520 BasicBlock
*ThenBB
= nullptr;
3522 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
3523 if (AllocaIP
.isSet())
3524 Builder
.restoreIP(AllocaIP
);
3526 Builder
.SetInsertPoint(&*(F
->getEntryBlock().getFirstInsertionPt()));
3527 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
3528 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
3530 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
3531 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
3532 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
3534 Builder
.restoreIP(CodeGenIP
);
3536 // collect some info for checks later
3537 ThenBB
= Builder
.GetInsertBlock();
3538 EntryBB
= ThenBB
->getUniquePredecessor();
3540 // simple instructions for body
3542 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
3543 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
3546 auto FiniCB
= [&](InsertPointTy IP
) {
3547 BasicBlock
*IPBB
= IP
.getBlock();
3548 EXPECT_NE(IPBB
->end(), IP
.getPoint());
3551 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
3552 OMPBuilder
.createSingle(Builder
, BODYGENCB_WRAPPER(BodyGenCB
),
3553 FINICB_WRAPPER(FiniCB
), /*IsNowait*/ true);
3554 assert(AfterIP
&& "unexpected error");
3555 Builder
.restoreIP(*AfterIP
);
3556 Value
*EntryBBTI
= EntryBB
->getTerminator();
3557 EXPECT_NE(EntryBBTI
, nullptr);
3558 EXPECT_TRUE(isa
<BranchInst
>(EntryBBTI
));
3559 BranchInst
*EntryBr
= cast
<BranchInst
>(EntryBB
->getTerminator());
3560 EXPECT_TRUE(EntryBr
->isConditional());
3561 EXPECT_EQ(EntryBr
->getSuccessor(0), ThenBB
);
3562 BasicBlock
*ExitBB
= ThenBB
->getUniqueSuccessor();
3563 EXPECT_EQ(EntryBr
->getSuccessor(1), ExitBB
);
3565 CmpInst
*CondInst
= cast
<CmpInst
>(EntryBr
->getCondition());
3566 EXPECT_TRUE(isa
<CallInst
>(CondInst
->getOperand(0)));
3568 CallInst
*SingleEntryCI
= cast
<CallInst
>(CondInst
->getOperand(0));
3569 EXPECT_EQ(SingleEntryCI
->arg_size(), 2U);
3570 EXPECT_EQ(SingleEntryCI
->getCalledFunction()->getName(), "__kmpc_single");
3571 EXPECT_TRUE(isa
<GlobalVariable
>(SingleEntryCI
->getArgOperand(0)));
3573 CallInst
*SingleEndCI
= nullptr;
3574 for (auto &FI
: *ThenBB
) {
3575 Instruction
*cur
= &FI
;
3576 if (isa
<CallInst
>(cur
)) {
3577 SingleEndCI
= cast
<CallInst
>(cur
);
3578 if (SingleEndCI
->getCalledFunction()->getName() == "__kmpc_end_single")
3580 SingleEndCI
= nullptr;
3583 EXPECT_NE(SingleEndCI
, nullptr);
3584 EXPECT_EQ(SingleEndCI
->arg_size(), 2U);
3585 EXPECT_TRUE(isa
<GlobalVariable
>(SingleEndCI
->getArgOperand(0)));
3586 EXPECT_EQ(SingleEndCI
->getArgOperand(1), SingleEntryCI
->getArgOperand(1));
3588 CallInst
*ExitBarrier
= nullptr;
3589 for (auto &FI
: *ExitBB
) {
3590 Instruction
*cur
= &FI
;
3591 if (auto CI
= dyn_cast
<CallInst
>(cur
)) {
3592 if (CI
->getCalledFunction()->getName() == "__kmpc_barrier") {
3598 EXPECT_EQ(ExitBarrier
, nullptr);
3601 // Helper class to check each instruction of a BB.
3604 BasicBlock::iterator BBI
;
3607 BBInstIter(BasicBlock
*BB
) : BB(BB
), BBI(BB
->begin()) {}
3609 bool hasNext() const { return BBI
!= BB
->end(); }
3611 template <typename InstTy
> InstTy
*next() {
3614 Instruction
*Cur
= &*BBI
++;
3615 if (!isa
<InstTy
>(Cur
))
3617 return cast
<InstTy
>(Cur
);
3621 TEST_F(OpenMPIRBuilderTest
, SingleDirectiveCopyPrivate
) {
3622 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
3623 OpenMPIRBuilder
OMPBuilder(*M
);
3624 OMPBuilder
.initialize();
3626 IRBuilder
<> Builder(BB
);
3628 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3630 AllocaInst
*PrivAI
= nullptr;
3632 BasicBlock
*EntryBB
= nullptr;
3633 BasicBlock
*ThenBB
= nullptr;
3635 Value
*CPVar
= Builder
.CreateAlloca(F
->arg_begin()->getType());
3636 Builder
.CreateStore(F
->arg_begin(), CPVar
);
3638 FunctionType
*CopyFuncTy
= FunctionType::get(
3639 Builder
.getVoidTy(), {Builder
.getPtrTy(), Builder
.getPtrTy()}, false);
3640 Function
*CopyFunc
=
3641 Function::Create(CopyFuncTy
, Function::PrivateLinkage
, "copy_var", *M
);
3643 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
3644 if (AllocaIP
.isSet())
3645 Builder
.restoreIP(AllocaIP
);
3647 Builder
.SetInsertPoint(&*(F
->getEntryBlock().getFirstInsertionPt()));
3648 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
3649 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
3651 llvm::BasicBlock
*CodeGenIPBB
= CodeGenIP
.getBlock();
3652 llvm::Instruction
*CodeGenIPInst
= &*CodeGenIP
.getPoint();
3653 EXPECT_EQ(CodeGenIPBB
->getTerminator(), CodeGenIPInst
);
3655 Builder
.restoreIP(CodeGenIP
);
3657 // collect some info for checks later
3658 ThenBB
= Builder
.GetInsertBlock();
3659 EntryBB
= ThenBB
->getUniquePredecessor();
3661 // simple instructions for body
3663 Builder
.CreateLoad(PrivAI
->getAllocatedType(), PrivAI
, "local.use");
3664 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
3667 auto FiniCB
= [&](InsertPointTy IP
) {
3668 BasicBlock
*IPBB
= IP
.getBlock();
3669 // IP must be before the unconditional branch to ExitBB
3670 EXPECT_NE(IPBB
->end(), IP
.getPoint());
3673 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createSingle(
3674 Builder
, BODYGENCB_WRAPPER(BodyGenCB
), FINICB_WRAPPER(FiniCB
),
3675 /*IsNowait*/ false, {CPVar
}, {CopyFunc
});
3676 assert(AfterIP
&& "unexpected error");
3677 Builder
.restoreIP(*AfterIP
);
3678 Value
*EntryBBTI
= EntryBB
->getTerminator();
3679 EXPECT_NE(EntryBBTI
, nullptr);
3680 EXPECT_TRUE(isa
<BranchInst
>(EntryBBTI
));
3681 BranchInst
*EntryBr
= cast
<BranchInst
>(EntryBB
->getTerminator());
3682 EXPECT_TRUE(EntryBr
->isConditional());
3683 EXPECT_EQ(EntryBr
->getSuccessor(0), ThenBB
);
3684 BasicBlock
*ExitBB
= ThenBB
->getUniqueSuccessor();
3685 EXPECT_EQ(EntryBr
->getSuccessor(1), ExitBB
);
3687 CmpInst
*CondInst
= cast
<CmpInst
>(EntryBr
->getCondition());
3688 EXPECT_TRUE(isa
<CallInst
>(CondInst
->getOperand(0)));
3690 CallInst
*SingleEntryCI
= cast
<CallInst
>(CondInst
->getOperand(0));
3691 EXPECT_EQ(SingleEntryCI
->arg_size(), 2U);
3692 EXPECT_EQ(SingleEntryCI
->getCalledFunction()->getName(), "__kmpc_single");
3693 EXPECT_TRUE(isa
<GlobalVariable
>(SingleEntryCI
->getArgOperand(0)));
3696 BBInstIter
ThenBBI(ThenBB
);
3698 auto *PrivLI
= ThenBBI
.next
<LoadInst
>();
3699 EXPECT_NE(PrivLI
, nullptr);
3700 EXPECT_EQ(PrivLI
->getPointerOperand(), PrivAI
);
3702 EXPECT_TRUE(ThenBBI
.next
<ICmpInst
>());
3704 auto *DidItSI
= ThenBBI
.next
<StoreInst
>();
3705 EXPECT_NE(DidItSI
, nullptr);
3706 EXPECT_EQ(DidItSI
->getValueOperand(),
3707 ConstantInt::get(Type::getInt32Ty(Ctx
), 1));
3708 Value
*DidIt
= DidItSI
->getPointerOperand();
3709 // call __kmpc_end_single
3710 auto *SingleEndCI
= ThenBBI
.next
<CallInst
>();
3711 EXPECT_NE(SingleEndCI
, nullptr);
3712 EXPECT_EQ(SingleEndCI
->getCalledFunction()->getName(), "__kmpc_end_single");
3713 EXPECT_EQ(SingleEndCI
->arg_size(), 2U);
3714 EXPECT_TRUE(isa
<GlobalVariable
>(SingleEndCI
->getArgOperand(0)));
3715 EXPECT_EQ(SingleEndCI
->getArgOperand(1), SingleEntryCI
->getArgOperand(1));
3717 auto *ExitBBBI
= ThenBBI
.next
<BranchInst
>();
3718 EXPECT_NE(ExitBBBI
, nullptr);
3719 EXPECT_TRUE(ExitBBBI
->isUnconditional());
3720 EXPECT_EQ(ExitBBBI
->getOperand(0), ExitBB
);
3721 EXPECT_FALSE(ThenBBI
.hasNext());
3724 BBInstIter
ExitBBI(ExitBB
);
3725 // call __kmpc_global_thread_num
3726 auto *ThreadNumCI
= ExitBBI
.next
<CallInst
>();
3727 EXPECT_NE(ThreadNumCI
, nullptr);
3728 EXPECT_EQ(ThreadNumCI
->getCalledFunction()->getName(),
3729 "__kmpc_global_thread_num");
3731 auto *DidItLI
= ExitBBI
.next
<LoadInst
>();
3732 EXPECT_NE(DidItLI
, nullptr);
3733 EXPECT_EQ(DidItLI
->getPointerOperand(), DidIt
);
3734 // call __kmpc_copyprivate
3735 auto *CopyPrivateCI
= ExitBBI
.next
<CallInst
>();
3736 EXPECT_NE(CopyPrivateCI
, nullptr);
3737 EXPECT_EQ(CopyPrivateCI
->arg_size(), 6U);
3738 EXPECT_TRUE(isa
<AllocaInst
>(CopyPrivateCI
->getArgOperand(3)));
3739 EXPECT_EQ(CopyPrivateCI
->getArgOperand(3), CPVar
);
3740 EXPECT_TRUE(isa
<Function
>(CopyPrivateCI
->getArgOperand(4)));
3741 EXPECT_EQ(CopyPrivateCI
->getArgOperand(4), CopyFunc
);
3742 EXPECT_TRUE(isa
<LoadInst
>(CopyPrivateCI
->getArgOperand(5)));
3743 DidItLI
= cast
<LoadInst
>(CopyPrivateCI
->getArgOperand(5));
3744 EXPECT_EQ(DidItLI
->getOperand(0), DidIt
);
3745 EXPECT_FALSE(ExitBBI
.hasNext());
3748 TEST_F(OpenMPIRBuilderTest
, OMPAtomicReadFlt
) {
3749 OpenMPIRBuilder
OMPBuilder(*M
);
3750 OMPBuilder
.initialize();
3752 IRBuilder
<> Builder(BB
);
3754 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3756 Type
*Float32
= Type::getFloatTy(M
->getContext());
3757 AllocaInst
*XVal
= Builder
.CreateAlloca(Float32
);
3758 XVal
->setName("AtomicVar");
3759 AllocaInst
*VVal
= Builder
.CreateAlloca(Float32
);
3760 VVal
->setName("AtomicRead");
3761 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
3762 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Float32
, false, false};
3763 OpenMPIRBuilder::AtomicOpValue V
= {VVal
, Float32
, false, false};
3765 Builder
.restoreIP(OMPBuilder
.createAtomicRead(Loc
, X
, V
, AO
));
3767 IntegerType
*IntCastTy
=
3768 IntegerType::get(M
->getContext(), Float32
->getScalarSizeInBits());
3770 LoadInst
*AtomicLoad
= cast
<LoadInst
>(VVal
->getNextNode());
3771 EXPECT_TRUE(AtomicLoad
->isAtomic());
3772 EXPECT_EQ(AtomicLoad
->getPointerOperand(), XVal
);
3774 BitCastInst
*CastToFlt
= cast
<BitCastInst
>(AtomicLoad
->getNextNode());
3775 EXPECT_EQ(CastToFlt
->getSrcTy(), IntCastTy
);
3776 EXPECT_EQ(CastToFlt
->getDestTy(), Float32
);
3777 EXPECT_EQ(CastToFlt
->getOperand(0), AtomicLoad
);
3779 StoreInst
*StoreofAtomic
= cast
<StoreInst
>(CastToFlt
->getNextNode());
3780 EXPECT_EQ(StoreofAtomic
->getValueOperand(), CastToFlt
);
3781 EXPECT_EQ(StoreofAtomic
->getPointerOperand(), VVal
);
3783 Builder
.CreateRetVoid();
3784 OMPBuilder
.finalize();
3785 EXPECT_FALSE(verifyModule(*M
, &errs()));
3788 TEST_F(OpenMPIRBuilderTest
, OMPAtomicReadInt
) {
3789 OpenMPIRBuilder
OMPBuilder(*M
);
3790 OMPBuilder
.initialize();
3792 IRBuilder
<> Builder(BB
);
3794 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3796 IntegerType
*Int32
= Type::getInt32Ty(M
->getContext());
3797 AllocaInst
*XVal
= Builder
.CreateAlloca(Int32
);
3798 XVal
->setName("AtomicVar");
3799 AllocaInst
*VVal
= Builder
.CreateAlloca(Int32
);
3800 VVal
->setName("AtomicRead");
3801 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
3802 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Int32
, false, false};
3803 OpenMPIRBuilder::AtomicOpValue V
= {VVal
, Int32
, false, false};
3805 BasicBlock
*EntryBB
= BB
;
3807 Builder
.restoreIP(OMPBuilder
.createAtomicRead(Loc
, X
, V
, AO
));
3808 LoadInst
*AtomicLoad
= nullptr;
3809 StoreInst
*StoreofAtomic
= nullptr;
3811 for (Instruction
&Cur
: *EntryBB
) {
3812 if (isa
<LoadInst
>(Cur
)) {
3813 AtomicLoad
= cast
<LoadInst
>(&Cur
);
3814 if (AtomicLoad
->getPointerOperand() == XVal
)
3816 AtomicLoad
= nullptr;
3817 } else if (isa
<StoreInst
>(Cur
)) {
3818 StoreofAtomic
= cast
<StoreInst
>(&Cur
);
3819 if (StoreofAtomic
->getPointerOperand() == VVal
)
3821 StoreofAtomic
= nullptr;
3825 EXPECT_NE(AtomicLoad
, nullptr);
3826 EXPECT_TRUE(AtomicLoad
->isAtomic());
3828 EXPECT_NE(StoreofAtomic
, nullptr);
3829 EXPECT_EQ(StoreofAtomic
->getValueOperand(), AtomicLoad
);
3831 Builder
.CreateRetVoid();
3832 OMPBuilder
.finalize();
3834 EXPECT_FALSE(verifyModule(*M
, &errs()));
3837 TEST_F(OpenMPIRBuilderTest
, OMPAtomicWriteFlt
) {
3838 OpenMPIRBuilder
OMPBuilder(*M
);
3839 OMPBuilder
.initialize();
3841 IRBuilder
<> Builder(BB
);
3843 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3845 LLVMContext
&Ctx
= M
->getContext();
3846 Type
*Float32
= Type::getFloatTy(Ctx
);
3847 AllocaInst
*XVal
= Builder
.CreateAlloca(Float32
);
3848 XVal
->setName("AtomicVar");
3849 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Float32
, false, false};
3850 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
3851 Constant
*ValToWrite
= ConstantFP::get(Float32
, 1.0);
3853 Builder
.restoreIP(OMPBuilder
.createAtomicWrite(Loc
, X
, ValToWrite
, AO
));
3855 IntegerType
*IntCastTy
=
3856 IntegerType::get(M
->getContext(), Float32
->getScalarSizeInBits());
3858 Value
*ExprCast
= Builder
.CreateBitCast(ValToWrite
, IntCastTy
);
3860 StoreInst
*StoreofAtomic
= cast
<StoreInst
>(XVal
->getNextNode());
3861 EXPECT_EQ(StoreofAtomic
->getValueOperand(), ExprCast
);
3862 EXPECT_EQ(StoreofAtomic
->getPointerOperand(), XVal
);
3863 EXPECT_TRUE(StoreofAtomic
->isAtomic());
3865 Builder
.CreateRetVoid();
3866 OMPBuilder
.finalize();
3867 EXPECT_FALSE(verifyModule(*M
, &errs()));
3870 TEST_F(OpenMPIRBuilderTest
, OMPAtomicWriteInt
) {
3871 OpenMPIRBuilder
OMPBuilder(*M
);
3872 OMPBuilder
.initialize();
3874 IRBuilder
<> Builder(BB
);
3876 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3878 LLVMContext
&Ctx
= M
->getContext();
3879 IntegerType
*Int32
= Type::getInt32Ty(Ctx
);
3880 AllocaInst
*XVal
= Builder
.CreateAlloca(Int32
);
3881 XVal
->setName("AtomicVar");
3882 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Int32
, false, false};
3883 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
3884 ConstantInt
*ValToWrite
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
3886 BasicBlock
*EntryBB
= BB
;
3888 Builder
.restoreIP(OMPBuilder
.createAtomicWrite(Loc
, X
, ValToWrite
, AO
));
3890 StoreInst
*StoreofAtomic
= nullptr;
3892 for (Instruction
&Cur
: *EntryBB
) {
3893 if (isa
<StoreInst
>(Cur
)) {
3894 StoreofAtomic
= cast
<StoreInst
>(&Cur
);
3895 if (StoreofAtomic
->getPointerOperand() == XVal
)
3897 StoreofAtomic
= nullptr;
3901 EXPECT_NE(StoreofAtomic
, nullptr);
3902 EXPECT_TRUE(StoreofAtomic
->isAtomic());
3903 EXPECT_EQ(StoreofAtomic
->getValueOperand(), ValToWrite
);
3905 Builder
.CreateRetVoid();
3906 OMPBuilder
.finalize();
3907 EXPECT_FALSE(verifyModule(*M
, &errs()));
3910 TEST_F(OpenMPIRBuilderTest
, OMPAtomicUpdate
) {
3911 OpenMPIRBuilder
OMPBuilder(*M
);
3912 OMPBuilder
.initialize();
3914 IRBuilder
<> Builder(BB
);
3916 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3918 IntegerType
*Int32
= Type::getInt32Ty(M
->getContext());
3919 AllocaInst
*XVal
= Builder
.CreateAlloca(Int32
);
3920 XVal
->setName("AtomicVar");
3921 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 0U), XVal
);
3922 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Int32
, false, false};
3923 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
3924 ConstantInt
*ConstVal
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
3925 Value
*Expr
= nullptr;
3926 AtomicRMWInst::BinOp RMWOp
= AtomicRMWInst::Sub
;
3927 bool IsXLHSInRHSPart
= false;
3929 BasicBlock
*EntryBB
= BB
;
3930 OpenMPIRBuilder::InsertPointTy
AllocaIP(EntryBB
,
3931 EntryBB
->getFirstInsertionPt());
3932 Value
*Sub
= nullptr;
3934 auto UpdateOp
= [&](Value
*Atomic
, IRBuilder
<> &IRB
) {
3935 Sub
= IRB
.CreateSub(ConstVal
, Atomic
);
3938 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createAtomicUpdate(
3939 Builder
, AllocaIP
, X
, Expr
, AO
, RMWOp
, UpdateOp
, IsXLHSInRHSPart
);
3940 assert(AfterIP
&& "unexpected error");
3941 Builder
.restoreIP(*AfterIP
);
3942 BasicBlock
*ContBB
= EntryBB
->getSingleSuccessor();
3943 BranchInst
*ContTI
= dyn_cast
<BranchInst
>(ContBB
->getTerminator());
3944 EXPECT_NE(ContTI
, nullptr);
3945 BasicBlock
*EndBB
= ContTI
->getSuccessor(0);
3946 EXPECT_TRUE(ContTI
->isConditional());
3947 EXPECT_EQ(ContTI
->getSuccessor(1), ContBB
);
3948 EXPECT_NE(EndBB
, nullptr);
3950 PHINode
*Phi
= dyn_cast
<PHINode
>(&ContBB
->front());
3951 EXPECT_NE(Phi
, nullptr);
3952 EXPECT_EQ(Phi
->getNumIncomingValues(), 2U);
3953 EXPECT_EQ(Phi
->getIncomingBlock(0), EntryBB
);
3954 EXPECT_EQ(Phi
->getIncomingBlock(1), ContBB
);
3956 EXPECT_EQ(Sub
->getNumUses(), 1U);
3957 StoreInst
*St
= dyn_cast
<StoreInst
>(Sub
->user_back());
3958 AllocaInst
*UpdateTemp
= dyn_cast
<AllocaInst
>(St
->getPointerOperand());
3960 ExtractValueInst
*ExVI1
=
3961 dyn_cast
<ExtractValueInst
>(Phi
->getIncomingValueForBlock(ContBB
));
3962 EXPECT_NE(ExVI1
, nullptr);
3963 AtomicCmpXchgInst
*CmpExchg
=
3964 dyn_cast
<AtomicCmpXchgInst
>(ExVI1
->getAggregateOperand());
3965 EXPECT_NE(CmpExchg
, nullptr);
3966 EXPECT_EQ(CmpExchg
->getPointerOperand(), XVal
);
3967 EXPECT_EQ(CmpExchg
->getCompareOperand(), Phi
);
3968 EXPECT_EQ(CmpExchg
->getSuccessOrdering(), AtomicOrdering::Monotonic
);
3970 LoadInst
*Ld
= dyn_cast
<LoadInst
>(CmpExchg
->getNewValOperand());
3971 EXPECT_NE(Ld
, nullptr);
3972 EXPECT_EQ(UpdateTemp
, Ld
->getPointerOperand());
3974 Builder
.CreateRetVoid();
3975 OMPBuilder
.finalize();
3976 EXPECT_FALSE(verifyModule(*M
, &errs()));
3979 TEST_F(OpenMPIRBuilderTest
, OMPAtomicUpdateFloat
) {
3980 OpenMPIRBuilder
OMPBuilder(*M
);
3981 OMPBuilder
.initialize();
3983 IRBuilder
<> Builder(BB
);
3985 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
3987 Type
*FloatTy
= Type::getFloatTy(M
->getContext());
3988 AllocaInst
*XVal
= Builder
.CreateAlloca(FloatTy
);
3989 XVal
->setName("AtomicVar");
3990 Builder
.CreateStore(ConstantFP::get(Type::getFloatTy(Ctx
), 0.0), XVal
);
3991 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, FloatTy
, false, false};
3992 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
3993 Constant
*ConstVal
= ConstantFP::get(Type::getFloatTy(Ctx
), 1.0);
3994 Value
*Expr
= nullptr;
3995 AtomicRMWInst::BinOp RMWOp
= AtomicRMWInst::FSub
;
3996 bool IsXLHSInRHSPart
= false;
3998 BasicBlock
*EntryBB
= BB
;
3999 OpenMPIRBuilder::InsertPointTy
AllocaIP(EntryBB
,
4000 EntryBB
->getFirstInsertionPt());
4001 Value
*Sub
= nullptr;
4003 auto UpdateOp
= [&](Value
*Atomic
, IRBuilder
<> &IRB
) {
4004 Sub
= IRB
.CreateFSub(ConstVal
, Atomic
);
4007 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createAtomicUpdate(
4008 Builder
, AllocaIP
, X
, Expr
, AO
, RMWOp
, UpdateOp
, IsXLHSInRHSPart
);
4009 assert(AfterIP
&& "unexpected error");
4010 Builder
.restoreIP(*AfterIP
);
4011 BasicBlock
*ContBB
= EntryBB
->getSingleSuccessor();
4012 BranchInst
*ContTI
= dyn_cast
<BranchInst
>(ContBB
->getTerminator());
4013 EXPECT_NE(ContTI
, nullptr);
4014 BasicBlock
*EndBB
= ContTI
->getSuccessor(0);
4015 EXPECT_TRUE(ContTI
->isConditional());
4016 EXPECT_EQ(ContTI
->getSuccessor(1), ContBB
);
4017 EXPECT_NE(EndBB
, nullptr);
4019 PHINode
*Phi
= dyn_cast
<PHINode
>(&ContBB
->front());
4020 EXPECT_NE(Phi
, nullptr);
4021 EXPECT_EQ(Phi
->getNumIncomingValues(), 2U);
4022 EXPECT_EQ(Phi
->getIncomingBlock(0), EntryBB
);
4023 EXPECT_EQ(Phi
->getIncomingBlock(1), ContBB
);
4025 EXPECT_EQ(Sub
->getNumUses(), 1U);
4026 StoreInst
*St
= dyn_cast
<StoreInst
>(Sub
->user_back());
4027 AllocaInst
*UpdateTemp
= dyn_cast
<AllocaInst
>(St
->getPointerOperand());
4029 ExtractValueInst
*ExVI1
=
4030 dyn_cast
<ExtractValueInst
>(Phi
->getIncomingValueForBlock(ContBB
));
4031 EXPECT_NE(ExVI1
, nullptr);
4032 AtomicCmpXchgInst
*CmpExchg
=
4033 dyn_cast
<AtomicCmpXchgInst
>(ExVI1
->getAggregateOperand());
4034 EXPECT_NE(CmpExchg
, nullptr);
4035 EXPECT_EQ(CmpExchg
->getPointerOperand(), XVal
);
4036 EXPECT_EQ(CmpExchg
->getCompareOperand(), Phi
);
4037 EXPECT_EQ(CmpExchg
->getSuccessOrdering(), AtomicOrdering::Monotonic
);
4039 LoadInst
*Ld
= dyn_cast
<LoadInst
>(CmpExchg
->getNewValOperand());
4040 EXPECT_NE(Ld
, nullptr);
4041 EXPECT_EQ(UpdateTemp
, Ld
->getPointerOperand());
4042 Builder
.CreateRetVoid();
4043 OMPBuilder
.finalize();
4044 EXPECT_FALSE(verifyModule(*M
, &errs()));
4047 TEST_F(OpenMPIRBuilderTest
, OMPAtomicUpdateIntr
) {
4048 OpenMPIRBuilder
OMPBuilder(*M
);
4049 OMPBuilder
.initialize();
4051 IRBuilder
<> Builder(BB
);
4053 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
4055 Type
*IntTy
= Type::getInt32Ty(M
->getContext());
4056 AllocaInst
*XVal
= Builder
.CreateAlloca(IntTy
);
4057 XVal
->setName("AtomicVar");
4058 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 0), XVal
);
4059 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, IntTy
, false, false};
4060 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
4061 Constant
*ConstVal
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1);
4062 Value
*Expr
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1);
4063 AtomicRMWInst::BinOp RMWOp
= AtomicRMWInst::UMax
;
4064 bool IsXLHSInRHSPart
= false;
4066 BasicBlock
*EntryBB
= BB
;
4067 OpenMPIRBuilder::InsertPointTy
AllocaIP(EntryBB
,
4068 EntryBB
->getFirstInsertionPt());
4069 Value
*Sub
= nullptr;
4071 auto UpdateOp
= [&](Value
*Atomic
, IRBuilder
<> &IRB
) {
4072 Sub
= IRB
.CreateSub(ConstVal
, Atomic
);
4075 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createAtomicUpdate(
4076 Builder
, AllocaIP
, X
, Expr
, AO
, RMWOp
, UpdateOp
, IsXLHSInRHSPart
);
4077 assert(AfterIP
&& "unexpected error");
4078 Builder
.restoreIP(*AfterIP
);
4079 BasicBlock
*ContBB
= EntryBB
->getSingleSuccessor();
4080 BranchInst
*ContTI
= dyn_cast
<BranchInst
>(ContBB
->getTerminator());
4081 EXPECT_NE(ContTI
, nullptr);
4082 BasicBlock
*EndBB
= ContTI
->getSuccessor(0);
4083 EXPECT_TRUE(ContTI
->isConditional());
4084 EXPECT_EQ(ContTI
->getSuccessor(1), ContBB
);
4085 EXPECT_NE(EndBB
, nullptr);
4087 PHINode
*Phi
= dyn_cast
<PHINode
>(&ContBB
->front());
4088 EXPECT_NE(Phi
, nullptr);
4089 EXPECT_EQ(Phi
->getNumIncomingValues(), 2U);
4090 EXPECT_EQ(Phi
->getIncomingBlock(0), EntryBB
);
4091 EXPECT_EQ(Phi
->getIncomingBlock(1), ContBB
);
4093 EXPECT_EQ(Sub
->getNumUses(), 1U);
4094 StoreInst
*St
= dyn_cast
<StoreInst
>(Sub
->user_back());
4095 AllocaInst
*UpdateTemp
= dyn_cast
<AllocaInst
>(St
->getPointerOperand());
4097 ExtractValueInst
*ExVI1
=
4098 dyn_cast
<ExtractValueInst
>(Phi
->getIncomingValueForBlock(ContBB
));
4099 EXPECT_NE(ExVI1
, nullptr);
4100 AtomicCmpXchgInst
*CmpExchg
=
4101 dyn_cast
<AtomicCmpXchgInst
>(ExVI1
->getAggregateOperand());
4102 EXPECT_NE(CmpExchg
, nullptr);
4103 EXPECT_EQ(CmpExchg
->getPointerOperand(), XVal
);
4104 EXPECT_EQ(CmpExchg
->getCompareOperand(), Phi
);
4105 EXPECT_EQ(CmpExchg
->getSuccessOrdering(), AtomicOrdering::Monotonic
);
4107 LoadInst
*Ld
= dyn_cast
<LoadInst
>(CmpExchg
->getNewValOperand());
4108 EXPECT_NE(Ld
, nullptr);
4109 EXPECT_EQ(UpdateTemp
, Ld
->getPointerOperand());
4111 Builder
.CreateRetVoid();
4112 OMPBuilder
.finalize();
4113 EXPECT_FALSE(verifyModule(*M
, &errs()));
4116 TEST_F(OpenMPIRBuilderTest
, OMPAtomicCapture
) {
4117 OpenMPIRBuilder
OMPBuilder(*M
);
4118 OMPBuilder
.initialize();
4120 IRBuilder
<> Builder(BB
);
4122 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
4124 LLVMContext
&Ctx
= M
->getContext();
4125 IntegerType
*Int32
= Type::getInt32Ty(Ctx
);
4126 AllocaInst
*XVal
= Builder
.CreateAlloca(Int32
);
4127 XVal
->setName("AtomicVar");
4128 AllocaInst
*VVal
= Builder
.CreateAlloca(Int32
);
4129 VVal
->setName("AtomicCapTar");
4131 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 0U), XVal
);
4133 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Int32
, false, false};
4134 OpenMPIRBuilder::AtomicOpValue V
= {VVal
, Int32
, false, false};
4135 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
4136 ConstantInt
*Expr
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
4137 AtomicRMWInst::BinOp RMWOp
= AtomicRMWInst::Add
;
4138 bool IsXLHSInRHSPart
= true;
4139 bool IsPostfixUpdate
= true;
4140 bool UpdateExpr
= true;
4142 BasicBlock
*EntryBB
= BB
;
4143 OpenMPIRBuilder::InsertPointTy
AllocaIP(EntryBB
,
4144 EntryBB
->getFirstInsertionPt());
4146 // integer update - not used
4147 auto UpdateOp
= [&](Value
*Atomic
, IRBuilder
<> &IRB
) { return nullptr; };
4149 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
4150 OMPBuilder
.createAtomicCapture(Builder
, AllocaIP
, X
, V
, Expr
, AO
, RMWOp
,
4151 UpdateOp
, UpdateExpr
, IsPostfixUpdate
,
4153 assert(AfterIP
&& "unexpected error");
4154 Builder
.restoreIP(*AfterIP
);
4155 EXPECT_EQ(EntryBB
->getParent()->size(), 1U);
4156 AtomicRMWInst
*ARWM
= dyn_cast
<AtomicRMWInst
>(Init
->getNextNode());
4157 EXPECT_NE(ARWM
, nullptr);
4158 EXPECT_EQ(ARWM
->getPointerOperand(), XVal
);
4159 EXPECT_EQ(ARWM
->getOperation(), RMWOp
);
4160 StoreInst
*St
= dyn_cast
<StoreInst
>(ARWM
->user_back());
4161 EXPECT_NE(St
, nullptr);
4162 EXPECT_EQ(St
->getPointerOperand(), VVal
);
4164 Builder
.CreateRetVoid();
4165 OMPBuilder
.finalize();
4166 EXPECT_FALSE(verifyModule(*M
, &errs()));
4169 TEST_F(OpenMPIRBuilderTest
, OMPAtomicCompare
) {
4170 OpenMPIRBuilder
OMPBuilder(*M
);
4171 OMPBuilder
.initialize();
4173 IRBuilder
<> Builder(BB
);
4175 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
4177 LLVMContext
&Ctx
= M
->getContext();
4178 IntegerType
*Int32
= Type::getInt32Ty(Ctx
);
4179 AllocaInst
*XVal
= Builder
.CreateAlloca(Int32
);
4182 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 0U), XVal
);
4184 OpenMPIRBuilder::AtomicOpValue XSigned
= {XVal
, Int32
, true, false};
4185 OpenMPIRBuilder::AtomicOpValue XUnsigned
= {XVal
, Int32
, false, false};
4186 // V and R are not used in atomic compare
4187 OpenMPIRBuilder::AtomicOpValue V
= {nullptr, nullptr, false, false};
4188 OpenMPIRBuilder::AtomicOpValue R
= {nullptr, nullptr, false, false};
4189 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
4190 ConstantInt
*Expr
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
4191 ConstantInt
*D
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
4192 OMPAtomicCompareOp OpMax
= OMPAtomicCompareOp::MAX
;
4193 OMPAtomicCompareOp OpEQ
= OMPAtomicCompareOp::EQ
;
4195 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4196 Builder
, XSigned
, V
, R
, Expr
, nullptr, AO
, OpMax
, true, false, false));
4197 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4198 Builder
, XUnsigned
, V
, R
, Expr
, nullptr, AO
, OpMax
, false, false, false));
4199 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4200 Builder
, XSigned
, V
, R
, Expr
, D
, AO
, OpEQ
, true, false, false));
4202 BasicBlock
*EntryBB
= BB
;
4203 EXPECT_EQ(EntryBB
->getParent()->size(), 1U);
4204 EXPECT_EQ(EntryBB
->size(), 5U);
4206 AtomicRMWInst
*ARWM1
= dyn_cast
<AtomicRMWInst
>(Init
->getNextNode());
4207 EXPECT_NE(ARWM1
, nullptr);
4208 EXPECT_EQ(ARWM1
->getPointerOperand(), XVal
);
4209 EXPECT_EQ(ARWM1
->getValOperand(), Expr
);
4210 EXPECT_EQ(ARWM1
->getOperation(), AtomicRMWInst::Min
);
4212 AtomicRMWInst
*ARWM2
= dyn_cast
<AtomicRMWInst
>(ARWM1
->getNextNode());
4213 EXPECT_NE(ARWM2
, nullptr);
4214 EXPECT_EQ(ARWM2
->getPointerOperand(), XVal
);
4215 EXPECT_EQ(ARWM2
->getValOperand(), Expr
);
4216 EXPECT_EQ(ARWM2
->getOperation(), AtomicRMWInst::UMax
);
4218 AtomicCmpXchgInst
*AXCHG
= dyn_cast
<AtomicCmpXchgInst
>(ARWM2
->getNextNode());
4219 EXPECT_NE(AXCHG
, nullptr);
4220 EXPECT_EQ(AXCHG
->getPointerOperand(), XVal
);
4221 EXPECT_EQ(AXCHG
->getCompareOperand(), Expr
);
4222 EXPECT_EQ(AXCHG
->getNewValOperand(), D
);
4224 Builder
.CreateRetVoid();
4225 OMPBuilder
.finalize();
4226 EXPECT_FALSE(verifyModule(*M
, &errs()));
4229 TEST_F(OpenMPIRBuilderTest
, OMPAtomicCompareCapture
) {
4230 OpenMPIRBuilder
OMPBuilder(*M
);
4231 OMPBuilder
.initialize();
4233 IRBuilder
<> Builder(BB
);
4235 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
4237 LLVMContext
&Ctx
= M
->getContext();
4238 IntegerType
*Int32
= Type::getInt32Ty(Ctx
);
4239 AllocaInst
*XVal
= Builder
.CreateAlloca(Int32
);
4241 AllocaInst
*VVal
= Builder
.CreateAlloca(Int32
);
4243 AllocaInst
*RVal
= Builder
.CreateAlloca(Int32
);
4247 Builder
.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx
), 0U), XVal
);
4249 OpenMPIRBuilder::AtomicOpValue X
= {XVal
, Int32
, true, false};
4250 OpenMPIRBuilder::AtomicOpValue V
= {VVal
, Int32
, false, false};
4251 OpenMPIRBuilder::AtomicOpValue NoV
= {nullptr, nullptr, false, false};
4252 OpenMPIRBuilder::AtomicOpValue R
= {RVal
, Int32
, false, false};
4253 OpenMPIRBuilder::AtomicOpValue NoR
= {nullptr, nullptr, false, false};
4255 AtomicOrdering AO
= AtomicOrdering::Monotonic
;
4256 ConstantInt
*Expr
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
4257 ConstantInt
*D
= ConstantInt::get(Type::getInt32Ty(Ctx
), 1U);
4258 OMPAtomicCompareOp OpMax
= OMPAtomicCompareOp::MAX
;
4259 OMPAtomicCompareOp OpEQ
= OMPAtomicCompareOp::EQ
;
4261 // { cond-update-stmt v = x; }
4262 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4263 Builder
, X
, V
, NoR
, Expr
, D
, AO
, OpEQ
, /* IsXBinopExpr */ true,
4264 /* IsPostfixUpdate */ false,
4265 /* IsFailOnly */ false));
4266 // { v = x; cond-update-stmt }
4267 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4268 Builder
, X
, V
, NoR
, Expr
, D
, AO
, OpEQ
, /* IsXBinopExpr */ true,
4269 /* IsPostfixUpdate */ true,
4270 /* IsFailOnly */ false));
4271 // if(x == e) { x = d; } else { v = x; }
4272 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4273 Builder
, X
, V
, NoR
, Expr
, D
, AO
, OpEQ
, /* IsXBinopExpr */ true,
4274 /* IsPostfixUpdate */ false,
4275 /* IsFailOnly */ true));
4276 // { r = x == e; if(r) { x = d; } }
4277 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4278 Builder
, X
, NoV
, R
, Expr
, D
, AO
, OpEQ
, /* IsXBinopExpr */ true,
4279 /* IsPostfixUpdate */ false,
4280 /* IsFailOnly */ false));
4281 // { r = x == e; if(r) { x = d; } else { v = x; } }
4282 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4283 Builder
, X
, V
, R
, Expr
, D
, AO
, OpEQ
, /* IsXBinopExpr */ true,
4284 /* IsPostfixUpdate */ false,
4285 /* IsFailOnly */ true));
4287 // { v = x; cond-update-stmt }
4288 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4289 Builder
, X
, V
, NoR
, Expr
, nullptr, AO
, OpMax
, /* IsXBinopExpr */ true,
4290 /* IsPostfixUpdate */ true,
4291 /* IsFailOnly */ false));
4292 // { cond-update-stmt v = x; }
4293 Builder
.restoreIP(OMPBuilder
.createAtomicCompare(
4294 Builder
, X
, V
, NoR
, Expr
, nullptr, AO
, OpMax
, /* IsXBinopExpr */ false,
4295 /* IsPostfixUpdate */ false,
4296 /* IsFailOnly */ false));
4298 BasicBlock
*EntryBB
= BB
;
4299 EXPECT_EQ(EntryBB
->getParent()->size(), 5U);
4300 BasicBlock
*Cont1
= dyn_cast
<BasicBlock
>(EntryBB
->getNextNode());
4301 EXPECT_NE(Cont1
, nullptr);
4302 BasicBlock
*Exit1
= dyn_cast
<BasicBlock
>(Cont1
->getNextNode());
4303 EXPECT_NE(Exit1
, nullptr);
4304 BasicBlock
*Cont2
= dyn_cast
<BasicBlock
>(Exit1
->getNextNode());
4305 EXPECT_NE(Cont2
, nullptr);
4306 BasicBlock
*Exit2
= dyn_cast
<BasicBlock
>(Cont2
->getNextNode());
4307 EXPECT_NE(Exit2
, nullptr);
4309 AtomicCmpXchgInst
*CmpXchg1
=
4310 dyn_cast
<AtomicCmpXchgInst
>(Init
->getNextNode());
4311 EXPECT_NE(CmpXchg1
, nullptr);
4312 EXPECT_EQ(CmpXchg1
->getPointerOperand(), XVal
);
4313 EXPECT_EQ(CmpXchg1
->getCompareOperand(), Expr
);
4314 EXPECT_EQ(CmpXchg1
->getNewValOperand(), D
);
4315 ExtractValueInst
*ExtVal1
=
4316 dyn_cast
<ExtractValueInst
>(CmpXchg1
->getNextNode());
4317 EXPECT_NE(ExtVal1
, nullptr);
4318 EXPECT_EQ(ExtVal1
->getAggregateOperand(), CmpXchg1
);
4319 EXPECT_EQ(ExtVal1
->getIndices(), ArrayRef
<unsigned int>(0U));
4320 ExtractValueInst
*ExtVal2
=
4321 dyn_cast
<ExtractValueInst
>(ExtVal1
->getNextNode());
4322 EXPECT_NE(ExtVal2
, nullptr);
4323 EXPECT_EQ(ExtVal2
->getAggregateOperand(), CmpXchg1
);
4324 EXPECT_EQ(ExtVal2
->getIndices(), ArrayRef
<unsigned int>(1U));
4325 SelectInst
*Sel1
= dyn_cast
<SelectInst
>(ExtVal2
->getNextNode());
4326 EXPECT_NE(Sel1
, nullptr);
4327 EXPECT_EQ(Sel1
->getCondition(), ExtVal2
);
4328 EXPECT_EQ(Sel1
->getTrueValue(), Expr
);
4329 EXPECT_EQ(Sel1
->getFalseValue(), ExtVal1
);
4330 StoreInst
*Store1
= dyn_cast
<StoreInst
>(Sel1
->getNextNode());
4331 EXPECT_NE(Store1
, nullptr);
4332 EXPECT_EQ(Store1
->getPointerOperand(), VVal
);
4333 EXPECT_EQ(Store1
->getValueOperand(), Sel1
);
4335 AtomicCmpXchgInst
*CmpXchg2
=
4336 dyn_cast
<AtomicCmpXchgInst
>(Store1
->getNextNode());
4337 EXPECT_NE(CmpXchg2
, nullptr);
4338 EXPECT_EQ(CmpXchg2
->getPointerOperand(), XVal
);
4339 EXPECT_EQ(CmpXchg2
->getCompareOperand(), Expr
);
4340 EXPECT_EQ(CmpXchg2
->getNewValOperand(), D
);
4341 ExtractValueInst
*ExtVal3
=
4342 dyn_cast
<ExtractValueInst
>(CmpXchg2
->getNextNode());
4343 EXPECT_NE(ExtVal3
, nullptr);
4344 EXPECT_EQ(ExtVal3
->getAggregateOperand(), CmpXchg2
);
4345 EXPECT_EQ(ExtVal3
->getIndices(), ArrayRef
<unsigned int>(0U));
4346 StoreInst
*Store2
= dyn_cast
<StoreInst
>(ExtVal3
->getNextNode());
4347 EXPECT_NE(Store2
, nullptr);
4348 EXPECT_EQ(Store2
->getPointerOperand(), VVal
);
4349 EXPECT_EQ(Store2
->getValueOperand(), ExtVal3
);
4351 AtomicCmpXchgInst
*CmpXchg3
=
4352 dyn_cast
<AtomicCmpXchgInst
>(Store2
->getNextNode());
4353 EXPECT_NE(CmpXchg3
, nullptr);
4354 EXPECT_EQ(CmpXchg3
->getPointerOperand(), XVal
);
4355 EXPECT_EQ(CmpXchg3
->getCompareOperand(), Expr
);
4356 EXPECT_EQ(CmpXchg3
->getNewValOperand(), D
);
4357 ExtractValueInst
*ExtVal4
=
4358 dyn_cast
<ExtractValueInst
>(CmpXchg3
->getNextNode());
4359 EXPECT_NE(ExtVal4
, nullptr);
4360 EXPECT_EQ(ExtVal4
->getAggregateOperand(), CmpXchg3
);
4361 EXPECT_EQ(ExtVal4
->getIndices(), ArrayRef
<unsigned int>(0U));
4362 ExtractValueInst
*ExtVal5
=
4363 dyn_cast
<ExtractValueInst
>(ExtVal4
->getNextNode());
4364 EXPECT_NE(ExtVal5
, nullptr);
4365 EXPECT_EQ(ExtVal5
->getAggregateOperand(), CmpXchg3
);
4366 EXPECT_EQ(ExtVal5
->getIndices(), ArrayRef
<unsigned int>(1U));
4367 BranchInst
*Br1
= dyn_cast
<BranchInst
>(ExtVal5
->getNextNode());
4368 EXPECT_NE(Br1
, nullptr);
4369 EXPECT_EQ(Br1
->isConditional(), true);
4370 EXPECT_EQ(Br1
->getCondition(), ExtVal5
);
4371 EXPECT_EQ(Br1
->getSuccessor(0), Exit1
);
4372 EXPECT_EQ(Br1
->getSuccessor(1), Cont1
);
4374 StoreInst
*Store3
= dyn_cast
<StoreInst
>(&Cont1
->front());
4375 EXPECT_NE(Store3
, nullptr);
4376 EXPECT_EQ(Store3
->getPointerOperand(), VVal
);
4377 EXPECT_EQ(Store3
->getValueOperand(), ExtVal4
);
4378 BranchInst
*Br2
= dyn_cast
<BranchInst
>(Store3
->getNextNode());
4379 EXPECT_NE(Br2
, nullptr);
4380 EXPECT_EQ(Br2
->isUnconditional(), true);
4381 EXPECT_EQ(Br2
->getSuccessor(0), Exit1
);
4383 AtomicCmpXchgInst
*CmpXchg4
= dyn_cast
<AtomicCmpXchgInst
>(&Exit1
->front());
4384 EXPECT_NE(CmpXchg4
, nullptr);
4385 EXPECT_EQ(CmpXchg4
->getPointerOperand(), XVal
);
4386 EXPECT_EQ(CmpXchg4
->getCompareOperand(), Expr
);
4387 EXPECT_EQ(CmpXchg4
->getNewValOperand(), D
);
4388 ExtractValueInst
*ExtVal6
=
4389 dyn_cast
<ExtractValueInst
>(CmpXchg4
->getNextNode());
4390 EXPECT_NE(ExtVal6
, nullptr);
4391 EXPECT_EQ(ExtVal6
->getAggregateOperand(), CmpXchg4
);
4392 EXPECT_EQ(ExtVal6
->getIndices(), ArrayRef
<unsigned int>(1U));
4393 ZExtInst
*ZExt1
= dyn_cast
<ZExtInst
>(ExtVal6
->getNextNode());
4394 EXPECT_NE(ZExt1
, nullptr);
4395 EXPECT_EQ(ZExt1
->getDestTy(), Int32
);
4396 StoreInst
*Store4
= dyn_cast
<StoreInst
>(ZExt1
->getNextNode());
4397 EXPECT_NE(Store4
, nullptr);
4398 EXPECT_EQ(Store4
->getPointerOperand(), RVal
);
4399 EXPECT_EQ(Store4
->getValueOperand(), ZExt1
);
4401 AtomicCmpXchgInst
*CmpXchg5
=
4402 dyn_cast
<AtomicCmpXchgInst
>(Store4
->getNextNode());
4403 EXPECT_NE(CmpXchg5
, nullptr);
4404 EXPECT_EQ(CmpXchg5
->getPointerOperand(), XVal
);
4405 EXPECT_EQ(CmpXchg5
->getCompareOperand(), Expr
);
4406 EXPECT_EQ(CmpXchg5
->getNewValOperand(), D
);
4407 ExtractValueInst
*ExtVal7
=
4408 dyn_cast
<ExtractValueInst
>(CmpXchg5
->getNextNode());
4409 EXPECT_NE(ExtVal7
, nullptr);
4410 EXPECT_EQ(ExtVal7
->getAggregateOperand(), CmpXchg5
);
4411 EXPECT_EQ(ExtVal7
->getIndices(), ArrayRef
<unsigned int>(0U));
4412 ExtractValueInst
*ExtVal8
=
4413 dyn_cast
<ExtractValueInst
>(ExtVal7
->getNextNode());
4414 EXPECT_NE(ExtVal8
, nullptr);
4415 EXPECT_EQ(ExtVal8
->getAggregateOperand(), CmpXchg5
);
4416 EXPECT_EQ(ExtVal8
->getIndices(), ArrayRef
<unsigned int>(1U));
4417 BranchInst
*Br3
= dyn_cast
<BranchInst
>(ExtVal8
->getNextNode());
4418 EXPECT_NE(Br3
, nullptr);
4419 EXPECT_EQ(Br3
->isConditional(), true);
4420 EXPECT_EQ(Br3
->getCondition(), ExtVal8
);
4421 EXPECT_EQ(Br3
->getSuccessor(0), Exit2
);
4422 EXPECT_EQ(Br3
->getSuccessor(1), Cont2
);
4424 StoreInst
*Store5
= dyn_cast
<StoreInst
>(&Cont2
->front());
4425 EXPECT_NE(Store5
, nullptr);
4426 EXPECT_EQ(Store5
->getPointerOperand(), VVal
);
4427 EXPECT_EQ(Store5
->getValueOperand(), ExtVal7
);
4428 BranchInst
*Br4
= dyn_cast
<BranchInst
>(Store5
->getNextNode());
4429 EXPECT_NE(Br4
, nullptr);
4430 EXPECT_EQ(Br4
->isUnconditional(), true);
4431 EXPECT_EQ(Br4
->getSuccessor(0), Exit2
);
4433 ExtractValueInst
*ExtVal9
= dyn_cast
<ExtractValueInst
>(&Exit2
->front());
4434 EXPECT_NE(ExtVal9
, nullptr);
4435 EXPECT_EQ(ExtVal9
->getAggregateOperand(), CmpXchg5
);
4436 EXPECT_EQ(ExtVal9
->getIndices(), ArrayRef
<unsigned int>(1U));
4437 ZExtInst
*ZExt2
= dyn_cast
<ZExtInst
>(ExtVal9
->getNextNode());
4438 EXPECT_NE(ZExt2
, nullptr);
4439 EXPECT_EQ(ZExt2
->getDestTy(), Int32
);
4440 StoreInst
*Store6
= dyn_cast
<StoreInst
>(ZExt2
->getNextNode());
4441 EXPECT_NE(Store6
, nullptr);
4442 EXPECT_EQ(Store6
->getPointerOperand(), RVal
);
4443 EXPECT_EQ(Store6
->getValueOperand(), ZExt2
);
4445 AtomicRMWInst
*ARWM1
= dyn_cast
<AtomicRMWInst
>(Store6
->getNextNode());
4446 EXPECT_NE(ARWM1
, nullptr);
4447 EXPECT_EQ(ARWM1
->getPointerOperand(), XVal
);
4448 EXPECT_EQ(ARWM1
->getValOperand(), Expr
);
4449 EXPECT_EQ(ARWM1
->getOperation(), AtomicRMWInst::Min
);
4450 StoreInst
*Store7
= dyn_cast
<StoreInst
>(ARWM1
->getNextNode());
4451 EXPECT_NE(Store7
, nullptr);
4452 EXPECT_EQ(Store7
->getPointerOperand(), VVal
);
4453 EXPECT_EQ(Store7
->getValueOperand(), ARWM1
);
4455 AtomicRMWInst
*ARWM2
= dyn_cast
<AtomicRMWInst
>(Store7
->getNextNode());
4456 EXPECT_NE(ARWM2
, nullptr);
4457 EXPECT_EQ(ARWM2
->getPointerOperand(), XVal
);
4458 EXPECT_EQ(ARWM2
->getValOperand(), Expr
);
4459 EXPECT_EQ(ARWM2
->getOperation(), AtomicRMWInst::Max
);
4460 CmpInst
*Cmp1
= dyn_cast
<CmpInst
>(ARWM2
->getNextNode());
4461 EXPECT_NE(Cmp1
, nullptr);
4462 EXPECT_EQ(Cmp1
->getPredicate(), CmpInst::ICMP_SGT
);
4463 EXPECT_EQ(Cmp1
->getOperand(0), ARWM2
);
4464 EXPECT_EQ(Cmp1
->getOperand(1), Expr
);
4465 SelectInst
*Sel2
= dyn_cast
<SelectInst
>(Cmp1
->getNextNode());
4466 EXPECT_NE(Sel2
, nullptr);
4467 EXPECT_EQ(Sel2
->getCondition(), Cmp1
);
4468 EXPECT_EQ(Sel2
->getTrueValue(), Expr
);
4469 EXPECT_EQ(Sel2
->getFalseValue(), ARWM2
);
4470 StoreInst
*Store8
= dyn_cast
<StoreInst
>(Sel2
->getNextNode());
4471 EXPECT_NE(Store8
, nullptr);
4472 EXPECT_EQ(Store8
->getPointerOperand(), VVal
);
4473 EXPECT_EQ(Store8
->getValueOperand(), Sel2
);
4475 Builder
.CreateRetVoid();
4476 OMPBuilder
.finalize();
4477 EXPECT_FALSE(verifyModule(*M
, &errs()));
4480 TEST_F(OpenMPIRBuilderTest
, CreateTeams
) {
4481 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4482 OpenMPIRBuilder
OMPBuilder(*M
);
4483 OMPBuilder
.Config
.IsTargetDevice
= false;
4484 OMPBuilder
.initialize();
4486 IRBuilder
<> Builder(BB
);
4488 AllocaInst
*ValPtr32
= Builder
.CreateAlloca(Builder
.getInt32Ty());
4489 AllocaInst
*ValPtr128
= Builder
.CreateAlloca(Builder
.getInt128Ty());
4490 Value
*Val128
= Builder
.CreateLoad(Builder
.getInt128Ty(), ValPtr128
, "load");
4492 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4493 Builder
.restoreIP(AllocaIP
);
4494 AllocaInst
*Local128
= Builder
.CreateAlloca(Builder
.getInt128Ty(), nullptr,
4495 "bodygen.alloca128");
4497 Builder
.restoreIP(CodeGenIP
);
4498 // Loading and storing captured pointer and values
4499 Builder
.CreateStore(Val128
, Local128
);
4500 Value
*Val32
= Builder
.CreateLoad(ValPtr32
->getAllocatedType(), ValPtr32
,
4503 LoadInst
*PrivLoad128
= Builder
.CreateLoad(
4504 Local128
->getAllocatedType(), Local128
, "bodygen.local.load128");
4505 Value
*Cmp
= Builder
.CreateICmpNE(
4506 Val32
, Builder
.CreateTrunc(PrivLoad128
, Val32
->getType()));
4507 Instruction
*ThenTerm
, *ElseTerm
;
4508 SplitBlockAndInsertIfThenElse(Cmp
, CodeGenIP
.getBlock()->getTerminator(),
4509 &ThenTerm
, &ElseTerm
);
4510 return Error::success();
4513 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
4514 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTeams(
4515 Builder
, BodyGenCB
, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
4516 /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr);
4517 assert(AfterIP
&& "unexpected error");
4518 Builder
.restoreIP(*AfterIP
);
4520 OMPBuilder
.finalize();
4521 Builder
.CreateRetVoid();
4523 EXPECT_FALSE(verifyModule(*M
, &errs()));
4525 CallInst
*TeamsForkCall
= dyn_cast
<CallInst
>(
4526 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams
)
4529 // Verify the Ident argument
4530 GlobalVariable
*Ident
= cast
<GlobalVariable
>(TeamsForkCall
->getArgOperand(0));
4531 ASSERT_NE(Ident
, nullptr);
4532 EXPECT_TRUE(Ident
->hasInitializer());
4533 Constant
*Initializer
= Ident
->getInitializer();
4534 GlobalVariable
*SrcStrGlob
=
4535 cast
<GlobalVariable
>(Initializer
->getOperand(4)->stripPointerCasts());
4536 ASSERT_NE(SrcStrGlob
, nullptr);
4537 ConstantDataArray
*SrcSrc
=
4538 dyn_cast
<ConstantDataArray
>(SrcStrGlob
->getInitializer());
4539 ASSERT_NE(SrcSrc
, nullptr);
4541 // Verify the outlined function signature.
4542 Function
*OutlinedFn
=
4543 dyn_cast
<Function
>(TeamsForkCall
->getArgOperand(2)->stripPointerCasts());
4544 ASSERT_NE(OutlinedFn
, nullptr);
4545 EXPECT_FALSE(OutlinedFn
->isDeclaration());
4546 EXPECT_TRUE(OutlinedFn
->arg_size() >= 3);
4547 EXPECT_EQ(OutlinedFn
->getArg(0)->getType(), Builder
.getPtrTy()); // global_tid
4548 EXPECT_EQ(OutlinedFn
->getArg(1)->getType(), Builder
.getPtrTy()); // bound_tid
4549 EXPECT_EQ(OutlinedFn
->getArg(2)->getType(),
4550 Builder
.getPtrTy()); // captured args
4552 // Check for TruncInst and ICmpInst in the outlined function.
4553 EXPECT_TRUE(any_of(instructions(OutlinedFn
),
4554 [](Instruction
&inst
) { return isa
<TruncInst
>(&inst
); }));
4555 EXPECT_TRUE(any_of(instructions(OutlinedFn
),
4556 [](Instruction
&inst
) { return isa
<ICmpInst
>(&inst
); }));
4559 TEST_F(OpenMPIRBuilderTest
, CreateTeamsWithThreadLimit
) {
4560 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4561 OpenMPIRBuilder
OMPBuilder(*M
);
4562 OMPBuilder
.Config
.IsTargetDevice
= false;
4563 OMPBuilder
.initialize();
4565 IRBuilder
<> &Builder
= OMPBuilder
.Builder
;
4566 Builder
.SetInsertPoint(BB
);
4568 Function
*FakeFunction
=
4569 Function::Create(FunctionType::get(Builder
.getVoidTy(), false),
4570 GlobalValue::ExternalLinkage
, "fakeFunction", M
.get());
4572 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4573 Builder
.restoreIP(CodeGenIP
);
4574 Builder
.CreateCall(FakeFunction
, {});
4575 return Error::success();
4578 // `F` has an argument - an integer, so we use that as the thread limit.
4579 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTeams(
4580 /*=*/Builder
, BodyGenCB
, /*NumTeamsLower=*/nullptr,
4581 /*NumTeamsUpper=*/nullptr, /*ThreadLimit=*/F
->arg_begin(),
4582 /*IfExpr=*/nullptr);
4583 assert(AfterIP
&& "unexpected error");
4584 Builder
.restoreIP(*AfterIP
);
4586 Builder
.CreateRetVoid();
4587 OMPBuilder
.finalize();
4589 ASSERT_FALSE(verifyModule(*M
));
4591 CallInst
*PushNumTeamsCallInst
=
4592 findSingleCall(F
, OMPRTL___kmpc_push_num_teams_51
, OMPBuilder
);
4593 ASSERT_NE(PushNumTeamsCallInst
, nullptr);
4595 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(2), Builder
.getInt32(0));
4596 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(3), Builder
.getInt32(0));
4597 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(4), &*F
->arg_begin());
4599 // Verifying that the next instruction to execute is kmpc_fork_teams
4600 BranchInst
*BrInst
=
4601 dyn_cast
<BranchInst
>(PushNumTeamsCallInst
->getNextNonDebugInstruction());
4602 ASSERT_NE(BrInst
, nullptr);
4603 ASSERT_EQ(BrInst
->getNumSuccessors(), 1U);
4604 Instruction
*NextInstruction
=
4605 BrInst
->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4606 CallInst
*ForkTeamsCI
= dyn_cast_if_present
<CallInst
>(NextInstruction
);
4607 ASSERT_NE(ForkTeamsCI
, nullptr);
4608 EXPECT_EQ(ForkTeamsCI
->getCalledFunction(),
4609 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams
));
4612 TEST_F(OpenMPIRBuilderTest
, CreateTeamsWithNumTeamsUpper
) {
4613 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4614 OpenMPIRBuilder
OMPBuilder(*M
);
4615 OMPBuilder
.Config
.IsTargetDevice
= false;
4616 OMPBuilder
.initialize();
4618 IRBuilder
<> &Builder
= OMPBuilder
.Builder
;
4619 Builder
.SetInsertPoint(BB
);
4621 Function
*FakeFunction
=
4622 Function::Create(FunctionType::get(Builder
.getVoidTy(), false),
4623 GlobalValue::ExternalLinkage
, "fakeFunction", M
.get());
4625 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4626 Builder
.restoreIP(CodeGenIP
);
4627 Builder
.CreateCall(FakeFunction
, {});
4628 return Error::success();
4631 // `F` already has an integer argument, so we use that as upper bound to
4633 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
4634 OMPBuilder
.createTeams(Builder
, BodyGenCB
,
4635 /*NumTeamsLower=*/nullptr,
4636 /*NumTeamsUpper=*/F
->arg_begin(),
4637 /*ThreadLimit=*/nullptr,
4638 /*IfExpr=*/nullptr);
4639 assert(AfterIP
&& "unexpected error");
4640 Builder
.restoreIP(*AfterIP
);
4642 Builder
.CreateRetVoid();
4643 OMPBuilder
.finalize();
4645 ASSERT_FALSE(verifyModule(*M
));
4647 CallInst
*PushNumTeamsCallInst
=
4648 findSingleCall(F
, OMPRTL___kmpc_push_num_teams_51
, OMPBuilder
);
4649 ASSERT_NE(PushNumTeamsCallInst
, nullptr);
4651 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(2), &*F
->arg_begin());
4652 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(3), &*F
->arg_begin());
4653 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(4), Builder
.getInt32(0));
4655 // Verifying that the next instruction to execute is kmpc_fork_teams
4656 BranchInst
*BrInst
=
4657 dyn_cast
<BranchInst
>(PushNumTeamsCallInst
->getNextNonDebugInstruction());
4658 ASSERT_NE(BrInst
, nullptr);
4659 ASSERT_EQ(BrInst
->getNumSuccessors(), 1U);
4660 Instruction
*NextInstruction
=
4661 BrInst
->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4662 CallInst
*ForkTeamsCI
= dyn_cast_if_present
<CallInst
>(NextInstruction
);
4663 ASSERT_NE(ForkTeamsCI
, nullptr);
4664 EXPECT_EQ(ForkTeamsCI
->getCalledFunction(),
4665 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams
));
4668 TEST_F(OpenMPIRBuilderTest
, CreateTeamsWithNumTeamsBoth
) {
4669 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4670 OpenMPIRBuilder
OMPBuilder(*M
);
4671 OMPBuilder
.Config
.IsTargetDevice
= false;
4672 OMPBuilder
.initialize();
4674 IRBuilder
<> &Builder
= OMPBuilder
.Builder
;
4675 Builder
.SetInsertPoint(BB
);
4677 Function
*FakeFunction
=
4678 Function::Create(FunctionType::get(Builder
.getVoidTy(), false),
4679 GlobalValue::ExternalLinkage
, "fakeFunction", M
.get());
4681 Value
*NumTeamsLower
=
4682 Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(5), "numTeamsLower");
4683 Value
*NumTeamsUpper
=
4684 Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(10), "numTeamsUpper");
4686 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4687 Builder
.restoreIP(CodeGenIP
);
4688 Builder
.CreateCall(FakeFunction
, {});
4689 return Error::success();
4692 // `F` already has an integer argument, so we use that as upper bound to
4694 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
4695 OMPBuilder
.createTeams(Builder
, BodyGenCB
, NumTeamsLower
, NumTeamsUpper
,
4696 /*ThreadLimit=*/nullptr, /*IfExpr=*/nullptr);
4697 assert(AfterIP
&& "unexpected error");
4698 Builder
.restoreIP(*AfterIP
);
4700 Builder
.CreateRetVoid();
4701 OMPBuilder
.finalize();
4703 ASSERT_FALSE(verifyModule(*M
));
4705 CallInst
*PushNumTeamsCallInst
=
4706 findSingleCall(F
, OMPRTL___kmpc_push_num_teams_51
, OMPBuilder
);
4707 ASSERT_NE(PushNumTeamsCallInst
, nullptr);
4709 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(2), NumTeamsLower
);
4710 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(3), NumTeamsUpper
);
4711 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(4), Builder
.getInt32(0));
4713 // Verifying that the next instruction to execute is kmpc_fork_teams
4714 BranchInst
*BrInst
=
4715 dyn_cast
<BranchInst
>(PushNumTeamsCallInst
->getNextNonDebugInstruction());
4716 ASSERT_NE(BrInst
, nullptr);
4717 ASSERT_EQ(BrInst
->getNumSuccessors(), 1U);
4718 Instruction
*NextInstruction
=
4719 BrInst
->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4720 CallInst
*ForkTeamsCI
= dyn_cast_if_present
<CallInst
>(NextInstruction
);
4721 ASSERT_NE(ForkTeamsCI
, nullptr);
4722 EXPECT_EQ(ForkTeamsCI
->getCalledFunction(),
4723 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams
));
4726 TEST_F(OpenMPIRBuilderTest
, CreateTeamsWithNumTeamsAndThreadLimit
) {
4727 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4728 OpenMPIRBuilder
OMPBuilder(*M
);
4729 OMPBuilder
.Config
.IsTargetDevice
= false;
4730 OMPBuilder
.initialize();
4732 IRBuilder
<> &Builder
= OMPBuilder
.Builder
;
4733 Builder
.SetInsertPoint(BB
);
4735 BasicBlock
*CodegenBB
= splitBB(Builder
, true);
4736 Builder
.SetInsertPoint(CodegenBB
);
4738 // Generate values for `num_teams` and `thread_limit` using the first argument
4739 // of the testing function.
4740 Value
*NumTeamsLower
=
4741 Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(5), "numTeamsLower");
4742 Value
*NumTeamsUpper
=
4743 Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(10), "numTeamsUpper");
4744 Value
*ThreadLimit
=
4745 Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(20), "threadLimit");
4747 Function
*FakeFunction
=
4748 Function::Create(FunctionType::get(Builder
.getVoidTy(), false),
4749 GlobalValue::ExternalLinkage
, "fakeFunction", M
.get());
4751 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4752 Builder
.restoreIP(CodeGenIP
);
4753 Builder
.CreateCall(FakeFunction
, {});
4754 return Error::success();
4757 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
4758 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTeams(
4759 Builder
, BodyGenCB
, NumTeamsLower
, NumTeamsUpper
, ThreadLimit
, nullptr);
4760 assert(AfterIP
&& "unexpected error");
4761 Builder
.restoreIP(*AfterIP
);
4763 Builder
.CreateRetVoid();
4764 OMPBuilder
.finalize();
4766 ASSERT_FALSE(verifyModule(*M
));
4768 CallInst
*PushNumTeamsCallInst
=
4769 findSingleCall(F
, OMPRTL___kmpc_push_num_teams_51
, OMPBuilder
);
4770 ASSERT_NE(PushNumTeamsCallInst
, nullptr);
4772 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(2), NumTeamsLower
);
4773 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(3), NumTeamsUpper
);
4774 EXPECT_EQ(PushNumTeamsCallInst
->getArgOperand(4), ThreadLimit
);
4776 // Verifying that the next instruction to execute is kmpc_fork_teams
4777 BranchInst
*BrInst
=
4778 dyn_cast
<BranchInst
>(PushNumTeamsCallInst
->getNextNonDebugInstruction());
4779 ASSERT_NE(BrInst
, nullptr);
4780 ASSERT_EQ(BrInst
->getNumSuccessors(), 1U);
4781 Instruction
*NextInstruction
=
4782 BrInst
->getSuccessor(0)->getFirstNonPHIOrDbgOrLifetime();
4783 CallInst
*ForkTeamsCI
= dyn_cast_if_present
<CallInst
>(NextInstruction
);
4784 ASSERT_NE(ForkTeamsCI
, nullptr);
4785 EXPECT_EQ(ForkTeamsCI
->getCalledFunction(),
4786 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_teams
));
4789 TEST_F(OpenMPIRBuilderTest
, CreateTeamsWithIfCondition
) {
4790 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4791 OpenMPIRBuilder
OMPBuilder(*M
);
4792 OMPBuilder
.Config
.IsTargetDevice
= false;
4793 OMPBuilder
.initialize();
4795 IRBuilder
<> &Builder
= OMPBuilder
.Builder
;
4796 Builder
.SetInsertPoint(BB
);
4798 Value
*IfExpr
= Builder
.CreateLoad(Builder
.getInt1Ty(),
4799 Builder
.CreateAlloca(Builder
.getInt1Ty()));
4801 Function
*FakeFunction
=
4802 Function::Create(FunctionType::get(Builder
.getVoidTy(), false),
4803 GlobalValue::ExternalLinkage
, "fakeFunction", M
.get());
4805 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4806 Builder
.restoreIP(CodeGenIP
);
4807 Builder
.CreateCall(FakeFunction
, {});
4808 return Error::success();
4811 // `F` already has an integer argument, so we use that as upper bound to
4813 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTeams(
4814 Builder
, BodyGenCB
, /*NumTeamsLower=*/nullptr, /*NumTeamsUpper=*/nullptr,
4815 /*ThreadLimit=*/nullptr, IfExpr
);
4816 assert(AfterIP
&& "unexpected error");
4817 Builder
.restoreIP(*AfterIP
);
4819 Builder
.CreateRetVoid();
4820 OMPBuilder
.finalize();
4822 ASSERT_FALSE(verifyModule(*M
));
4824 CallInst
*PushNumTeamsCallInst
=
4825 findSingleCall(F
, OMPRTL___kmpc_push_num_teams_51
, OMPBuilder
);
4826 ASSERT_NE(PushNumTeamsCallInst
, nullptr);
4827 Value
*NumTeamsLower
= PushNumTeamsCallInst
->getArgOperand(2);
4828 Value
*NumTeamsUpper
= PushNumTeamsCallInst
->getArgOperand(3);
4829 Value
*ThreadLimit
= PushNumTeamsCallInst
->getArgOperand(4);
4831 // Check the lower_bound
4832 ASSERT_NE(NumTeamsLower
, nullptr);
4833 SelectInst
*NumTeamsLowerSelectInst
= dyn_cast
<SelectInst
>(NumTeamsLower
);
4834 ASSERT_NE(NumTeamsLowerSelectInst
, nullptr);
4835 EXPECT_EQ(NumTeamsLowerSelectInst
->getCondition(), IfExpr
);
4836 EXPECT_EQ(NumTeamsLowerSelectInst
->getTrueValue(), Builder
.getInt32(0));
4837 EXPECT_EQ(NumTeamsLowerSelectInst
->getFalseValue(), Builder
.getInt32(1));
4839 // Check the upper_bound
4840 ASSERT_NE(NumTeamsUpper
, nullptr);
4841 SelectInst
*NumTeamsUpperSelectInst
= dyn_cast
<SelectInst
>(NumTeamsUpper
);
4842 ASSERT_NE(NumTeamsUpperSelectInst
, nullptr);
4843 EXPECT_EQ(NumTeamsUpperSelectInst
->getCondition(), IfExpr
);
4844 EXPECT_EQ(NumTeamsUpperSelectInst
->getTrueValue(), Builder
.getInt32(0));
4845 EXPECT_EQ(NumTeamsUpperSelectInst
->getFalseValue(), Builder
.getInt32(1));
4847 // Check thread_limit
4848 EXPECT_EQ(ThreadLimit
, Builder
.getInt32(0));
4851 TEST_F(OpenMPIRBuilderTest
, CreateTeamsWithIfConditionAndNumTeams
) {
4852 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
4853 OpenMPIRBuilder
OMPBuilder(*M
);
4854 OMPBuilder
.Config
.IsTargetDevice
= false;
4855 OMPBuilder
.initialize();
4857 IRBuilder
<> &Builder
= OMPBuilder
.Builder
;
4858 Builder
.SetInsertPoint(BB
);
4860 Value
*IfExpr
= Builder
.CreateLoad(
4861 Builder
.getInt32Ty(), Builder
.CreateAlloca(Builder
.getInt32Ty()));
4862 Value
*NumTeamsLower
= Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(5));
4863 Value
*NumTeamsUpper
=
4864 Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(10));
4865 Value
*ThreadLimit
= Builder
.CreateAdd(F
->arg_begin(), Builder
.getInt32(20));
4867 Function
*FakeFunction
=
4868 Function::Create(FunctionType::get(Builder
.getVoidTy(), false),
4869 GlobalValue::ExternalLinkage
, "fakeFunction", M
.get());
4871 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
4872 Builder
.restoreIP(CodeGenIP
);
4873 Builder
.CreateCall(FakeFunction
, {});
4874 return Error::success();
4877 // `F` already has an integer argument, so we use that as upper bound to
4879 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTeams(
4880 Builder
, BodyGenCB
, NumTeamsLower
, NumTeamsUpper
, ThreadLimit
, IfExpr
);
4881 assert(AfterIP
&& "unexpected error");
4882 Builder
.restoreIP(*AfterIP
);
4884 Builder
.CreateRetVoid();
4885 OMPBuilder
.finalize();
4887 ASSERT_FALSE(verifyModule(*M
));
4889 CallInst
*PushNumTeamsCallInst
=
4890 findSingleCall(F
, OMPRTL___kmpc_push_num_teams_51
, OMPBuilder
);
4891 ASSERT_NE(PushNumTeamsCallInst
, nullptr);
4892 Value
*NumTeamsLowerArg
= PushNumTeamsCallInst
->getArgOperand(2);
4893 Value
*NumTeamsUpperArg
= PushNumTeamsCallInst
->getArgOperand(3);
4894 Value
*ThreadLimitArg
= PushNumTeamsCallInst
->getArgOperand(4);
4896 // Get the boolean conversion of if expression
4897 ASSERT_EQ(IfExpr
->getNumUses(), 1U);
4898 User
*IfExprInst
= IfExpr
->user_back();
4899 ICmpInst
*IfExprCmpInst
= dyn_cast
<ICmpInst
>(IfExprInst
);
4900 ASSERT_NE(IfExprCmpInst
, nullptr);
4901 EXPECT_EQ(IfExprCmpInst
->getPredicate(), ICmpInst::Predicate::ICMP_NE
);
4902 EXPECT_EQ(IfExprCmpInst
->getOperand(0), IfExpr
);
4903 EXPECT_EQ(IfExprCmpInst
->getOperand(1), Builder
.getInt32(0));
4905 // Check the lower_bound
4906 ASSERT_NE(NumTeamsLowerArg
, nullptr);
4907 SelectInst
*NumTeamsLowerSelectInst
= dyn_cast
<SelectInst
>(NumTeamsLowerArg
);
4908 ASSERT_NE(NumTeamsLowerSelectInst
, nullptr);
4909 EXPECT_EQ(NumTeamsLowerSelectInst
->getCondition(), IfExprCmpInst
);
4910 EXPECT_EQ(NumTeamsLowerSelectInst
->getTrueValue(), NumTeamsLower
);
4911 EXPECT_EQ(NumTeamsLowerSelectInst
->getFalseValue(), Builder
.getInt32(1));
4913 // Check the upper_bound
4914 ASSERT_NE(NumTeamsUpperArg
, nullptr);
4915 SelectInst
*NumTeamsUpperSelectInst
= dyn_cast
<SelectInst
>(NumTeamsUpperArg
);
4916 ASSERT_NE(NumTeamsUpperSelectInst
, nullptr);
4917 EXPECT_EQ(NumTeamsUpperSelectInst
->getCondition(), IfExprCmpInst
);
4918 EXPECT_EQ(NumTeamsUpperSelectInst
->getTrueValue(), NumTeamsUpper
);
4919 EXPECT_EQ(NumTeamsUpperSelectInst
->getFalseValue(), Builder
.getInt32(1));
4921 // Check thread_limit
4922 EXPECT_EQ(ThreadLimitArg
, ThreadLimit
);
4925 /// Returns the single instruction of InstTy type in BB that uses the value V.
4926 /// If there is more than one such instruction, returns null.
4927 template <typename InstTy
>
4928 static InstTy
*findSingleUserInBlock(Value
*V
, BasicBlock
*BB
) {
4929 InstTy
*Result
= nullptr;
4930 for (User
*U
: V
->users()) {
4931 auto *Inst
= dyn_cast
<InstTy
>(U
);
4932 if (!Inst
|| Inst
->getParent() != BB
)
4935 if (auto *SI
= dyn_cast
<StoreInst
>(Inst
)) {
4936 if (V
== SI
->getValueOperand())
4947 /// Returns true if BB contains a simple binary reduction that loads a value
4948 /// from Accum, performs some binary operation with it, and stores it back to
4950 static bool isSimpleBinaryReduction(Value
*Accum
, BasicBlock
*BB
,
4951 Instruction::BinaryOps
*OpCode
= nullptr) {
4952 StoreInst
*Store
= findSingleUserInBlock
<StoreInst
>(Accum
, BB
);
4955 auto *Stored
= dyn_cast
<BinaryOperator
>(Store
->getOperand(0));
4958 if (OpCode
&& *OpCode
!= Stored
->getOpcode())
4960 auto *Load
= dyn_cast
<LoadInst
>(Stored
->getOperand(0));
4961 return Load
&& Load
->getOperand(0) == Accum
;
4964 /// Returns true if BB contains a binary reduction that reduces V using a binary
4965 /// operator into an accumulator that is a function argument.
4966 static bool isValueReducedToFuncArg(Value
*V
, BasicBlock
*BB
) {
4967 auto *ReductionOp
= findSingleUserInBlock
<BinaryOperator
>(V
, BB
);
4971 auto *GlobalLoad
= dyn_cast
<LoadInst
>(ReductionOp
->getOperand(0));
4975 auto *Store
= findSingleUserInBlock
<StoreInst
>(ReductionOp
, BB
);
4979 return Store
->getPointerOperand() == GlobalLoad
->getPointerOperand() &&
4980 isa
<Argument
>(findAggregateFromValue(GlobalLoad
->getPointerOperand()));
4983 /// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and
4984 /// [0, 1], respectively, and assigns results of these instructions to Zero and
4985 /// One. Returns true on success, false on failure or if such instructions are
4986 /// not unique among the users of Ptr.
4987 static bool findGEPZeroOne(Value
*Ptr
, Value
*&Zero
, Value
*&One
) {
4990 for (User
*U
: Ptr
->users()) {
4991 if (auto *GEP
= dyn_cast
<GetElementPtrInst
>(U
)) {
4992 if (GEP
->getNumIndices() != 2)
4994 auto *FirstIdx
= dyn_cast
<ConstantInt
>(GEP
->getOperand(1));
4995 auto *SecondIdx
= dyn_cast
<ConstantInt
>(GEP
->getOperand(2));
4996 EXPECT_NE(FirstIdx
, nullptr);
4997 EXPECT_NE(SecondIdx
, nullptr);
4999 EXPECT_TRUE(FirstIdx
->isZero());
5000 if (SecondIdx
->isZero()) {
5004 } else if (SecondIdx
->isOne()) {
5013 return Zero
!= nullptr && One
!= nullptr;
5016 static OpenMPIRBuilder::InsertPointTy
5017 sumReduction(OpenMPIRBuilder::InsertPointTy IP
, Value
*LHS
, Value
*RHS
,
5019 IRBuilder
<> Builder(IP
.getBlock(), IP
.getPoint());
5020 Result
= Builder
.CreateFAdd(LHS
, RHS
, "red.add");
5021 return Builder
.saveIP();
5024 static OpenMPIRBuilder::InsertPointTy
5025 sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP
, Type
*Ty
, Value
*LHS
,
5027 IRBuilder
<> Builder(IP
.getBlock(), IP
.getPoint());
5028 Value
*Partial
= Builder
.CreateLoad(Ty
, RHS
, "red.partial");
5029 Builder
.CreateAtomicRMW(AtomicRMWInst::FAdd
, LHS
, Partial
, std::nullopt
,
5030 AtomicOrdering::Monotonic
);
5031 return Builder
.saveIP();
5034 static OpenMPIRBuilder::InsertPointTy
5035 xorReduction(OpenMPIRBuilder::InsertPointTy IP
, Value
*LHS
, Value
*RHS
,
5037 IRBuilder
<> Builder(IP
.getBlock(), IP
.getPoint());
5038 Result
= Builder
.CreateXor(LHS
, RHS
, "red.xor");
5039 return Builder
.saveIP();
5042 static OpenMPIRBuilder::InsertPointTy
5043 xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP
, Type
*Ty
, Value
*LHS
,
5045 IRBuilder
<> Builder(IP
.getBlock(), IP
.getPoint());
5046 Value
*Partial
= Builder
.CreateLoad(Ty
, RHS
, "red.partial");
5047 Builder
.CreateAtomicRMW(AtomicRMWInst::Xor
, LHS
, Partial
, std::nullopt
,
5048 AtomicOrdering::Monotonic
);
5049 return Builder
.saveIP();
5052 TEST_F(OpenMPIRBuilderTest
, CreateReductions
) {
5053 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5054 OpenMPIRBuilder
OMPBuilder(*M
);
5055 OMPBuilder
.Config
.IsTargetDevice
= false;
5056 OMPBuilder
.initialize();
5058 IRBuilder
<> Builder(BB
);
5060 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
5061 Builder
.CreateBr(EnterBB
);
5062 Builder
.SetInsertPoint(EnterBB
);
5063 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5065 // Create variables to be reduced.
5066 InsertPointTy
OuterAllocaIP(&F
->getEntryBlock(),
5067 F
->getEntryBlock().getFirstInsertionPt());
5068 Type
*SumType
= Builder
.getFloatTy();
5069 Type
*XorType
= Builder
.getInt32Ty();
5073 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5074 Builder
.restoreIP(OuterAllocaIP
);
5075 SumReduced
= Builder
.CreateAlloca(SumType
);
5076 XorReduced
= Builder
.CreateAlloca(XorType
);
5079 // Store initial values of reductions into global variables.
5080 Builder
.CreateStore(ConstantFP::get(Builder
.getFloatTy(), 0.0), SumReduced
);
5081 Builder
.CreateStore(Builder
.getInt32(1), XorReduced
);
5083 // The loop body computes two reductions:
5084 // sum of (float) thread-id;
5085 // xor of thread-id;
5086 // and store the result in global variables.
5087 InsertPointTy BodyIP
, BodyAllocaIP
;
5088 auto BodyGenCB
= [&](InsertPointTy InnerAllocaIP
, InsertPointTy CodeGenIP
) {
5089 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5090 Builder
.restoreIP(CodeGenIP
);
5093 Constant
*SrcLocStr
= OMPBuilder
.getOrCreateSrcLocStr(Loc
, StrSize
);
5094 Value
*Ident
= OMPBuilder
.getOrCreateIdent(SrcLocStr
, StrSize
);
5095 Value
*TID
= OMPBuilder
.getOrCreateThreadID(Ident
);
5097 Builder
.CreateUIToFP(TID
, Builder
.getFloatTy(), "sum.local");
5098 Value
*SumPartial
= Builder
.CreateLoad(SumType
, SumReduced
, "sum.partial");
5099 Value
*XorPartial
= Builder
.CreateLoad(XorType
, XorReduced
, "xor.partial");
5100 Value
*Sum
= Builder
.CreateFAdd(SumPartial
, SumLocal
, "sum");
5101 Value
*Xor
= Builder
.CreateXor(XorPartial
, TID
, "xor");
5102 Builder
.CreateStore(Sum
, SumReduced
);
5103 Builder
.CreateStore(Xor
, XorReduced
);
5105 BodyIP
= Builder
.saveIP();
5106 BodyAllocaIP
= InnerAllocaIP
;
5107 return Error::success();
5110 // Privatization for reduction creates local copies of reduction variables and
5111 // initializes them to reduction-neutral values.
5112 Value
*SumPrivatized
;
5113 Value
*XorPrivatized
;
5114 auto PrivCB
= [&](InsertPointTy InnerAllocaIP
, InsertPointTy CodeGenIP
,
5115 Value
&Original
, Value
&Inner
, Value
*&ReplVal
) {
5116 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5117 Builder
.restoreIP(InnerAllocaIP
);
5118 if (&Original
== SumReduced
) {
5119 SumPrivatized
= Builder
.CreateAlloca(Builder
.getFloatTy());
5120 ReplVal
= SumPrivatized
;
5121 } else if (&Original
== XorReduced
) {
5122 XorPrivatized
= Builder
.CreateAlloca(Builder
.getInt32Ty());
5123 ReplVal
= XorPrivatized
;
5129 Builder
.restoreIP(CodeGenIP
);
5130 if (&Original
== SumReduced
)
5131 Builder
.CreateStore(ConstantFP::get(Builder
.getFloatTy(), 0.0),
5133 else if (&Original
== XorReduced
)
5134 Builder
.CreateStore(Builder
.getInt32(0), XorPrivatized
);
5136 return Builder
.saveIP();
5139 // Do nothing in finalization.
5140 auto FiniCB
= [&](InsertPointTy CodeGenIP
) { return Error::success(); };
5142 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
5143 OMPBuilder
.createParallel(Loc
, OuterAllocaIP
, BodyGenCB
, PrivCB
, FiniCB
,
5144 /* IfCondition */ nullptr,
5145 /* NumThreads */ nullptr, OMP_PROC_BIND_default
,
5146 /* IsCancellable */ false);
5147 assert(AfterIP
&& "unexpected error");
5148 Builder
.restoreIP(*AfterIP
);
5150 OpenMPIRBuilder::ReductionInfo ReductionInfos
[] = {
5151 {SumType
, SumReduced
, SumPrivatized
,
5152 /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar
, sumReduction
,
5153 /*ReductionGenClang=*/nullptr, sumAtomicReduction
},
5154 {XorType
, XorReduced
, XorPrivatized
,
5155 /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar
, xorReduction
,
5156 /*ReductionGenClang=*/nullptr, xorAtomicReduction
}};
5157 OMPBuilder
.Config
.setIsGPU(false);
5159 bool ReduceVariableByRef
[] = {false, false};
5161 OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP
=
5162 OMPBuilder
.createReductions(BodyIP
, BodyAllocaIP
, ReductionInfos
,
5163 ReduceVariableByRef
);
5164 assert(ReductionsIP
&& "unexpected error");
5166 Builder
.restoreIP(*AfterIP
);
5167 Builder
.CreateRetVoid();
5169 OMPBuilder
.finalize(F
);
5171 // The IR must be valid.
5172 EXPECT_FALSE(verifyModule(*M
));
5174 // Outlining must have happened.
5175 SmallVector
<CallInst
*> ForkCalls
;
5176 findCalls(F
, omp::RuntimeFunction::OMPRTL___kmpc_fork_call
, OMPBuilder
,
5178 ASSERT_EQ(ForkCalls
.size(), 1u);
5179 Value
*CalleeVal
= ForkCalls
[0]->getOperand(2);
5180 Function
*Outlined
= dyn_cast
<Function
>(CalleeVal
);
5181 EXPECT_NE(Outlined
, nullptr);
5183 // Check that the lock variable was created with the expected name.
5184 GlobalVariable
*LockVar
=
5185 M
->getGlobalVariable(".gomp_critical_user_.reduction.var");
5186 EXPECT_NE(LockVar
, nullptr);
5188 // Find the allocation of a local array that will be used to call the runtime
5189 // reduciton function.
5190 BasicBlock
&AllocBlock
= Outlined
->getEntryBlock();
5191 Value
*LocalArray
= nullptr;
5192 for (Instruction
&I
: AllocBlock
) {
5193 if (AllocaInst
*Alloc
= dyn_cast
<AllocaInst
>(&I
)) {
5194 if (!Alloc
->getAllocatedType()->isArrayTy() ||
5195 !Alloc
->getAllocatedType()->getArrayElementType()->isPointerTy())
5201 ASSERT_NE(LocalArray
, nullptr);
5203 // Find the call to the runtime reduction function.
5204 BasicBlock
*BB
= AllocBlock
.getUniqueSuccessor();
5205 Value
*LocalArrayPtr
= nullptr;
5206 Value
*ReductionFnVal
= nullptr;
5207 Value
*SwitchArg
= nullptr;
5208 for (Instruction
&I
: *BB
) {
5209 if (CallInst
*Call
= dyn_cast
<CallInst
>(&I
)) {
5210 if (Call
->getCalledFunction() !=
5211 OMPBuilder
.getOrCreateRuntimeFunctionPtr(
5212 RuntimeFunction::OMPRTL___kmpc_reduce
))
5214 LocalArrayPtr
= Call
->getOperand(4);
5215 ReductionFnVal
= Call
->getOperand(5);
5221 // Check that the local array is passed to the function.
5222 ASSERT_NE(LocalArrayPtr
, nullptr);
5223 EXPECT_EQ(LocalArrayPtr
, LocalArray
);
5225 // Find the GEP instructions preceding stores to the local array.
5226 Value
*FirstArrayElemPtr
= nullptr;
5227 Value
*SecondArrayElemPtr
= nullptr;
5228 EXPECT_EQ(LocalArray
->getNumUses(), 3u);
5230 findGEPZeroOne(LocalArray
, FirstArrayElemPtr
, SecondArrayElemPtr
));
5232 // Check that the values stored into the local array are privatized reduction
5234 auto *FirstPrivatized
= dyn_cast_or_null
<AllocaInst
>(
5235 findStoredValue
<GetElementPtrInst
>(FirstArrayElemPtr
));
5236 auto *SecondPrivatized
= dyn_cast_or_null
<AllocaInst
>(
5237 findStoredValue
<GetElementPtrInst
>(SecondArrayElemPtr
));
5238 ASSERT_NE(FirstPrivatized
, nullptr);
5239 ASSERT_NE(SecondPrivatized
, nullptr);
5240 ASSERT_TRUE(isa
<Instruction
>(FirstArrayElemPtr
));
5241 EXPECT_TRUE(isSimpleBinaryReduction(
5242 FirstPrivatized
, cast
<Instruction
>(FirstArrayElemPtr
)->getParent()));
5243 EXPECT_TRUE(isSimpleBinaryReduction(
5244 SecondPrivatized
, cast
<Instruction
>(FirstArrayElemPtr
)->getParent()));
5246 // Check that the result of the runtime reduction call is used for further
5248 ASSERT_EQ(SwitchArg
->getNumUses(), 1u);
5249 SwitchInst
*Switch
= dyn_cast
<SwitchInst
>(*SwitchArg
->user_begin());
5250 ASSERT_NE(Switch
, nullptr);
5251 EXPECT_EQ(Switch
->getNumSuccessors(), 3u);
5252 BasicBlock
*NonAtomicBB
= Switch
->case_begin()->getCaseSuccessor();
5253 BasicBlock
*AtomicBB
= std::next(Switch
->case_begin())->getCaseSuccessor();
5255 // Non-atomic block contains reductions to the global reduction variable,
5256 // which is passed into the outlined function as an argument.
5258 findSingleUserInBlock
<LoadInst
>(FirstPrivatized
, NonAtomicBB
);
5260 findSingleUserInBlock
<LoadInst
>(SecondPrivatized
, NonAtomicBB
);
5261 EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad
, NonAtomicBB
));
5262 EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad
, NonAtomicBB
));
5264 // Atomic block also constains reductions to the global reduction variable.
5265 FirstLoad
= findSingleUserInBlock
<LoadInst
>(FirstPrivatized
, AtomicBB
);
5266 SecondLoad
= findSingleUserInBlock
<LoadInst
>(SecondPrivatized
, AtomicBB
);
5267 auto *FirstAtomic
= findSingleUserInBlock
<AtomicRMWInst
>(FirstLoad
, AtomicBB
);
5268 auto *SecondAtomic
=
5269 findSingleUserInBlock
<AtomicRMWInst
>(SecondLoad
, AtomicBB
);
5270 ASSERT_NE(FirstAtomic
, nullptr);
5271 Value
*AtomicStorePointer
= FirstAtomic
->getPointerOperand();
5272 EXPECT_TRUE(isa
<Argument
>(findAggregateFromValue(AtomicStorePointer
)));
5273 ASSERT_NE(SecondAtomic
, nullptr);
5274 AtomicStorePointer
= SecondAtomic
->getPointerOperand();
5275 EXPECT_TRUE(isa
<Argument
>(findAggregateFromValue(AtomicStorePointer
)));
5277 // Check that the separate reduction function also performs (non-atomic)
5278 // reductions after extracting reduction variables from its arguments.
5279 Function
*ReductionFn
= cast
<Function
>(ReductionFnVal
);
5280 BasicBlock
*FnReductionBB
= &ReductionFn
->getEntryBlock();
5282 Value
*SecondLHSPtr
;
5284 findGEPZeroOne(ReductionFn
->getArg(0), FirstLHSPtr
, SecondLHSPtr
));
5285 Value
*Opaque
= findSingleUserInBlock
<LoadInst
>(FirstLHSPtr
, FnReductionBB
);
5286 ASSERT_NE(Opaque
, nullptr);
5287 EXPECT_TRUE(isSimpleBinaryReduction(Opaque
, FnReductionBB
));
5288 Opaque
= findSingleUserInBlock
<LoadInst
>(SecondLHSPtr
, FnReductionBB
);
5289 ASSERT_NE(Opaque
, nullptr);
5290 EXPECT_TRUE(isSimpleBinaryReduction(Opaque
, FnReductionBB
));
5294 EXPECT_TRUE(findGEPZeroOne(ReductionFn
->getArg(1), FirstRHS
, SecondRHS
));
5297 TEST_F(OpenMPIRBuilderTest
, CreateTwoReductions
) {
5298 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5299 OpenMPIRBuilder
OMPBuilder(*M
);
5300 OMPBuilder
.Config
.IsTargetDevice
= false;
5301 OMPBuilder
.initialize();
5303 IRBuilder
<> Builder(BB
);
5305 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "parallel.enter", F
);
5306 Builder
.CreateBr(EnterBB
);
5307 Builder
.SetInsertPoint(EnterBB
);
5308 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5310 // Create variables to be reduced.
5311 InsertPointTy
OuterAllocaIP(&F
->getEntryBlock(),
5312 F
->getEntryBlock().getFirstInsertionPt());
5313 Type
*SumType
= Builder
.getFloatTy();
5314 Type
*XorType
= Builder
.getInt32Ty();
5318 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5319 Builder
.restoreIP(OuterAllocaIP
);
5320 SumReduced
= Builder
.CreateAlloca(SumType
);
5321 XorReduced
= Builder
.CreateAlloca(XorType
);
5324 // Store initial values of reductions into global variables.
5325 Builder
.CreateStore(ConstantFP::get(Builder
.getFloatTy(), 0.0), SumReduced
);
5326 Builder
.CreateStore(Builder
.getInt32(1), XorReduced
);
5328 InsertPointTy FirstBodyIP
, FirstBodyAllocaIP
;
5329 auto FirstBodyGenCB
= [&](InsertPointTy InnerAllocaIP
,
5330 InsertPointTy CodeGenIP
) {
5331 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5332 Builder
.restoreIP(CodeGenIP
);
5335 Constant
*SrcLocStr
= OMPBuilder
.getOrCreateSrcLocStr(Loc
, StrSize
);
5336 Value
*Ident
= OMPBuilder
.getOrCreateIdent(SrcLocStr
, StrSize
);
5337 Value
*TID
= OMPBuilder
.getOrCreateThreadID(Ident
);
5339 Builder
.CreateUIToFP(TID
, Builder
.getFloatTy(), "sum.local");
5340 Value
*SumPartial
= Builder
.CreateLoad(SumType
, SumReduced
, "sum.partial");
5341 Value
*Sum
= Builder
.CreateFAdd(SumPartial
, SumLocal
, "sum");
5342 Builder
.CreateStore(Sum
, SumReduced
);
5344 FirstBodyIP
= Builder
.saveIP();
5345 FirstBodyAllocaIP
= InnerAllocaIP
;
5346 return Error::success();
5349 InsertPointTy SecondBodyIP
, SecondBodyAllocaIP
;
5350 auto SecondBodyGenCB
= [&](InsertPointTy InnerAllocaIP
,
5351 InsertPointTy CodeGenIP
) {
5352 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5353 Builder
.restoreIP(CodeGenIP
);
5356 Constant
*SrcLocStr
= OMPBuilder
.getOrCreateSrcLocStr(Loc
, StrSize
);
5357 Value
*Ident
= OMPBuilder
.getOrCreateIdent(SrcLocStr
, StrSize
);
5358 Value
*TID
= OMPBuilder
.getOrCreateThreadID(Ident
);
5359 Value
*XorPartial
= Builder
.CreateLoad(XorType
, XorReduced
, "xor.partial");
5360 Value
*Xor
= Builder
.CreateXor(XorPartial
, TID
, "xor");
5361 Builder
.CreateStore(Xor
, XorReduced
);
5363 SecondBodyIP
= Builder
.saveIP();
5364 SecondBodyAllocaIP
= InnerAllocaIP
;
5365 return Error::success();
5368 // Privatization for reduction creates local copies of reduction variables and
5369 // initializes them to reduction-neutral values. The same privatization
5370 // callback is used for both loops, with dispatch based on the value being
5372 Value
*SumPrivatized
;
5373 Value
*XorPrivatized
;
5374 auto PrivCB
= [&](InsertPointTy InnerAllocaIP
, InsertPointTy CodeGenIP
,
5375 Value
&Original
, Value
&Inner
, Value
*&ReplVal
) {
5376 IRBuilderBase::InsertPointGuard
Guard(Builder
);
5377 Builder
.restoreIP(InnerAllocaIP
);
5378 if (&Original
== SumReduced
) {
5379 SumPrivatized
= Builder
.CreateAlloca(Builder
.getFloatTy());
5380 ReplVal
= SumPrivatized
;
5381 } else if (&Original
== XorReduced
) {
5382 XorPrivatized
= Builder
.CreateAlloca(Builder
.getInt32Ty());
5383 ReplVal
= XorPrivatized
;
5389 Builder
.restoreIP(CodeGenIP
);
5390 if (&Original
== SumReduced
)
5391 Builder
.CreateStore(ConstantFP::get(Builder
.getFloatTy(), 0.0),
5393 else if (&Original
== XorReduced
)
5394 Builder
.CreateStore(Builder
.getInt32(0), XorPrivatized
);
5396 return Builder
.saveIP();
5399 // Do nothing in finalization.
5400 auto FiniCB
= [&](InsertPointTy CodeGenIP
) { return Error::success(); };
5402 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP1
=
5403 OMPBuilder
.createParallel(Loc
, OuterAllocaIP
, FirstBodyGenCB
, PrivCB
,
5404 FiniCB
, /* IfCondition */ nullptr,
5405 /* NumThreads */ nullptr, OMP_PROC_BIND_default
,
5406 /* IsCancellable */ false);
5407 assert(AfterIP1
&& "unexpected error");
5408 Builder
.restoreIP(*AfterIP1
);
5409 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP2
= OMPBuilder
.createParallel(
5410 {Builder
.saveIP(), DL
}, OuterAllocaIP
, SecondBodyGenCB
, PrivCB
, FiniCB
,
5411 /* IfCondition */ nullptr,
5412 /* NumThreads */ nullptr, OMP_PROC_BIND_default
,
5413 /* IsCancellable */ false);
5414 assert(AfterIP2
&& "unexpected error");
5415 Builder
.restoreIP(*AfterIP2
);
5417 OMPBuilder
.Config
.setIsGPU(false);
5418 bool ReduceVariableByRef
[] = {false};
5420 OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP1
=
5421 OMPBuilder
.createReductions(
5422 FirstBodyIP
, FirstBodyAllocaIP
,
5423 {{SumType
, SumReduced
, SumPrivatized
,
5424 /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar
, sumReduction
,
5425 /*ReductionGenClang=*/nullptr, sumAtomicReduction
}},
5426 ReduceVariableByRef
);
5427 assert(ReductionsIP1
&& "unexpected error");
5428 OpenMPIRBuilder::InsertPointOrErrorTy ReductionsIP2
=
5429 OMPBuilder
.createReductions(
5430 SecondBodyIP
, SecondBodyAllocaIP
,
5431 {{XorType
, XorReduced
, XorPrivatized
,
5432 /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar
, xorReduction
,
5433 /*ReductionGenClang=*/nullptr, xorAtomicReduction
}},
5434 ReduceVariableByRef
);
5435 assert(ReductionsIP2
&& "unexpected error");
5437 Builder
.restoreIP(*AfterIP2
);
5438 Builder
.CreateRetVoid();
5440 OMPBuilder
.finalize(F
);
5442 // The IR must be valid.
5443 EXPECT_FALSE(verifyModule(*M
));
5445 // Two different outlined functions must have been created.
5446 SmallVector
<CallInst
*> ForkCalls
;
5447 findCalls(F
, omp::RuntimeFunction::OMPRTL___kmpc_fork_call
, OMPBuilder
,
5449 ASSERT_EQ(ForkCalls
.size(), 2u);
5450 Value
*CalleeVal
= ForkCalls
[0]->getOperand(2);
5451 Function
*FirstCallee
= cast
<Function
>(CalleeVal
);
5452 CalleeVal
= ForkCalls
[1]->getOperand(2);
5453 Function
*SecondCallee
= cast
<Function
>(CalleeVal
);
5454 EXPECT_NE(FirstCallee
, SecondCallee
);
5456 // Two different reduction functions must have been created.
5457 SmallVector
<CallInst
*> ReduceCalls
;
5458 findCalls(FirstCallee
, omp::RuntimeFunction::OMPRTL___kmpc_reduce
, OMPBuilder
,
5460 ASSERT_EQ(ReduceCalls
.size(), 1u);
5461 auto *AddReduction
= cast
<Function
>(ReduceCalls
[0]->getOperand(5));
5462 ReduceCalls
.clear();
5463 findCalls(SecondCallee
, omp::RuntimeFunction::OMPRTL___kmpc_reduce
,
5464 OMPBuilder
, ReduceCalls
);
5465 auto *XorReduction
= cast
<Function
>(ReduceCalls
[0]->getOperand(5));
5466 EXPECT_NE(AddReduction
, XorReduction
);
5468 // Each reduction function does its own kind of reduction.
5469 BasicBlock
*FnReductionBB
= &AddReduction
->getEntryBlock();
5470 Value
*FirstLHSPtr
= findSingleUserInBlock
<GetElementPtrInst
>(
5471 AddReduction
->getArg(0), FnReductionBB
);
5472 ASSERT_NE(FirstLHSPtr
, nullptr);
5473 Value
*Opaque
= findSingleUserInBlock
<LoadInst
>(FirstLHSPtr
, FnReductionBB
);
5474 ASSERT_NE(Opaque
, nullptr);
5475 Instruction::BinaryOps Opcode
= Instruction::FAdd
;
5476 EXPECT_TRUE(isSimpleBinaryReduction(Opaque
, FnReductionBB
, &Opcode
));
5478 FnReductionBB
= &XorReduction
->getEntryBlock();
5479 Value
*SecondLHSPtr
= findSingleUserInBlock
<GetElementPtrInst
>(
5480 XorReduction
->getArg(0), FnReductionBB
);
5481 ASSERT_NE(FirstLHSPtr
, nullptr);
5482 Opaque
= findSingleUserInBlock
<LoadInst
>(SecondLHSPtr
, FnReductionBB
);
5483 ASSERT_NE(Opaque
, nullptr);
5484 Opcode
= Instruction::Xor
;
5485 EXPECT_TRUE(isSimpleBinaryReduction(Opaque
, FnReductionBB
, &Opcode
));
5488 TEST_F(OpenMPIRBuilderTest
, CreateSectionsSimple
) {
5489 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5490 using BodyGenCallbackTy
= llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy
;
5491 OpenMPIRBuilder
OMPBuilder(*M
);
5492 OMPBuilder
.initialize();
5494 IRBuilder
<> Builder(BB
);
5496 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "sections.enter", F
);
5497 Builder
.CreateBr(EnterBB
);
5498 Builder
.SetInsertPoint(EnterBB
);
5499 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5501 llvm::SmallVector
<BodyGenCallbackTy
, 4> SectionCBVector
;
5502 llvm::SmallVector
<BasicBlock
*, 4> CaseBBs
;
5504 auto FiniCB
= [&](InsertPointTy IP
) { return Error::success(); };
5505 auto SectionCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
5506 return Error::success();
5508 SectionCBVector
.push_back(SectionCB
);
5510 auto PrivCB
= [](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
5511 llvm::Value
&, llvm::Value
&Val
,
5512 llvm::Value
*&ReplVal
) { return CodeGenIP
; };
5513 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5514 F
->getEntryBlock().getFirstInsertionPt());
5515 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createSections(
5516 Loc
, AllocaIP
, SectionCBVector
, PrivCB
, FiniCB
, false, false);
5517 assert(AfterIP
&& "unexpected error");
5518 Builder
.restoreIP(*AfterIP
);
5519 Builder
.CreateRetVoid(); // Required at the end of the function
5520 EXPECT_NE(F
->getEntryBlock().getTerminator(), nullptr);
5521 EXPECT_FALSE(verifyModule(*M
, &errs()));
5524 TEST_F(OpenMPIRBuilderTest
, CreateSections
) {
5525 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5526 using BodyGenCallbackTy
= llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy
;
5527 OpenMPIRBuilder
OMPBuilder(*M
);
5528 OMPBuilder
.initialize();
5530 IRBuilder
<> Builder(BB
);
5532 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5533 llvm::SmallVector
<BodyGenCallbackTy
, 4> SectionCBVector
;
5534 llvm::SmallVector
<BasicBlock
*, 4> CaseBBs
;
5536 BasicBlock
*SwitchBB
= nullptr;
5537 AllocaInst
*PrivAI
= nullptr;
5538 SwitchInst
*Switch
= nullptr;
5540 unsigned NumBodiesGenerated
= 0;
5541 unsigned NumFiniCBCalls
= 0;
5542 PrivAI
= Builder
.CreateAlloca(F
->arg_begin()->getType());
5544 auto FiniCB
= [&](InsertPointTy IP
) {
5546 BasicBlock
*IPBB
= IP
.getBlock();
5547 EXPECT_NE(IPBB
->end(), IP
.getPoint());
5550 auto SectionCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
5551 ++NumBodiesGenerated
;
5552 CaseBBs
.push_back(CodeGenIP
.getBlock());
5553 SwitchBB
= CodeGenIP
.getBlock()->getSinglePredecessor();
5554 Builder
.restoreIP(CodeGenIP
);
5555 Builder
.CreateStore(F
->arg_begin(), PrivAI
);
5557 Builder
.CreateLoad(F
->arg_begin()->getType(), PrivAI
, "local.alloca");
5558 Builder
.CreateICmpNE(F
->arg_begin(), PrivLoad
);
5559 return Error::success();
5561 auto PrivCB
= [](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
5562 llvm::Value
&, llvm::Value
&Val
, llvm::Value
*&ReplVal
) {
5563 // TODO: Privatization not implemented yet
5567 SectionCBVector
.push_back(SectionCB
);
5568 SectionCBVector
.push_back(SectionCB
);
5570 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5571 F
->getEntryBlock().getFirstInsertionPt());
5572 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
5573 OMPBuilder
.createSections(Loc
, AllocaIP
, SectionCBVector
, PrivCB
,
5574 FINICB_WRAPPER(FiniCB
), false, false);
5575 assert(AfterIP
&& "unexpected error");
5576 Builder
.restoreIP(*AfterIP
);
5577 Builder
.CreateRetVoid(); // Required at the end of the function
5579 // Switch BB's predecessor is loop condition BB, whose successor at index 1 is
5581 BasicBlock
*ForExitBB
=
5582 SwitchBB
->getSinglePredecessor()->getTerminator()->getSuccessor(1);
5583 EXPECT_NE(ForExitBB
, nullptr);
5585 EXPECT_NE(PrivAI
, nullptr);
5586 Function
*OutlinedFn
= PrivAI
->getFunction();
5587 EXPECT_EQ(F
, OutlinedFn
);
5588 EXPECT_FALSE(verifyModule(*M
, &errs()));
5589 EXPECT_EQ(OutlinedFn
->arg_size(), 1U);
5591 BasicBlock
*LoopPreheaderBB
=
5592 OutlinedFn
->getEntryBlock().getSingleSuccessor();
5593 // loop variables are 5 - lower bound, upper bound, stride, islastiter, and
5595 bool FoundForInit
= false;
5596 for (Instruction
&Inst
: *LoopPreheaderBB
) {
5597 if (isa
<CallInst
>(Inst
)) {
5598 if (cast
<CallInst
>(&Inst
)->getCalledFunction()->getName() ==
5599 "__kmpc_for_static_init_4u") {
5600 FoundForInit
= true;
5604 EXPECT_EQ(FoundForInit
, true);
5606 bool FoundForExit
= false;
5607 bool FoundBarrier
= false;
5608 for (Instruction
&Inst
: *ForExitBB
) {
5609 if (isa
<CallInst
>(Inst
)) {
5610 if (cast
<CallInst
>(&Inst
)->getCalledFunction()->getName() ==
5611 "__kmpc_for_static_fini") {
5612 FoundForExit
= true;
5614 if (cast
<CallInst
>(&Inst
)->getCalledFunction()->getName() ==
5616 FoundBarrier
= true;
5618 if (FoundForExit
&& FoundBarrier
)
5622 EXPECT_EQ(FoundForExit
, true);
5623 EXPECT_EQ(FoundBarrier
, true);
5625 EXPECT_NE(SwitchBB
, nullptr);
5626 EXPECT_NE(SwitchBB
->getTerminator(), nullptr);
5627 EXPECT_EQ(isa
<SwitchInst
>(SwitchBB
->getTerminator()), true);
5628 Switch
= cast
<SwitchInst
>(SwitchBB
->getTerminator());
5629 EXPECT_EQ(Switch
->getNumCases(), 2U);
5631 EXPECT_EQ(CaseBBs
.size(), 2U);
5632 for (auto *&CaseBB
: CaseBBs
) {
5633 EXPECT_EQ(CaseBB
->getParent(), OutlinedFn
);
5636 ASSERT_EQ(NumBodiesGenerated
, 2U);
5637 ASSERT_EQ(NumFiniCBCalls
, 1U);
5638 EXPECT_FALSE(verifyModule(*M
, &errs()));
5641 TEST_F(OpenMPIRBuilderTest
, CreateSectionsNoWait
) {
5642 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5643 using BodyGenCallbackTy
= llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy
;
5644 OpenMPIRBuilder
OMPBuilder(*M
);
5645 OMPBuilder
.initialize();
5647 IRBuilder
<> Builder(BB
);
5649 BasicBlock
*EnterBB
= BasicBlock::Create(Ctx
, "sections.enter", F
);
5650 Builder
.CreateBr(EnterBB
);
5651 Builder
.SetInsertPoint(EnterBB
);
5652 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5654 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5655 F
->getEntryBlock().getFirstInsertionPt());
5656 llvm::SmallVector
<BodyGenCallbackTy
, 4> SectionCBVector
;
5657 auto PrivCB
= [](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
,
5658 llvm::Value
&, llvm::Value
&Val
,
5659 llvm::Value
*&ReplVal
) { return CodeGenIP
; };
5660 auto FiniCB
= [&](InsertPointTy IP
) { return Error::success(); };
5662 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createSections(
5663 Loc
, AllocaIP
, SectionCBVector
, PrivCB
, FiniCB
, false, true);
5664 assert(AfterIP
&& "unexpected error");
5665 Builder
.restoreIP(*AfterIP
);
5666 Builder
.CreateRetVoid(); // Required at the end of the function
5667 for (auto &Inst
: instructions(*F
)) {
5668 EXPECT_FALSE(isa
<CallInst
>(Inst
) &&
5669 cast
<CallInst
>(&Inst
)->getCalledFunction()->getName() ==
5671 "call to function __kmpc_barrier found with nowait");
5675 TEST_F(OpenMPIRBuilderTest
, CreateOffloadMaptypes
) {
5676 OpenMPIRBuilder
OMPBuilder(*M
);
5677 OMPBuilder
.initialize();
5679 IRBuilder
<> Builder(BB
);
5681 SmallVector
<uint64_t> Mappings
= {0, 1};
5682 GlobalVariable
*OffloadMaptypesGlobal
=
5683 OMPBuilder
.createOffloadMaptypes(Mappings
, "offload_maptypes");
5684 EXPECT_FALSE(M
->global_empty());
5685 EXPECT_EQ(OffloadMaptypesGlobal
->getName(), "offload_maptypes");
5686 EXPECT_TRUE(OffloadMaptypesGlobal
->isConstant());
5687 EXPECT_TRUE(OffloadMaptypesGlobal
->hasGlobalUnnamedAddr());
5688 EXPECT_TRUE(OffloadMaptypesGlobal
->hasPrivateLinkage());
5689 EXPECT_TRUE(OffloadMaptypesGlobal
->hasInitializer());
5690 Constant
*Initializer
= OffloadMaptypesGlobal
->getInitializer();
5691 EXPECT_TRUE(isa
<ConstantDataArray
>(Initializer
));
5692 ConstantDataArray
*MappingInit
= dyn_cast
<ConstantDataArray
>(Initializer
);
5693 EXPECT_EQ(MappingInit
->getNumElements(), Mappings
.size());
5694 EXPECT_TRUE(MappingInit
->getType()->getElementType()->isIntegerTy(64));
5695 Constant
*CA
= ConstantDataArray::get(Builder
.getContext(), Mappings
);
5696 EXPECT_EQ(MappingInit
, CA
);
5699 TEST_F(OpenMPIRBuilderTest
, CreateOffloadMapnames
) {
5700 OpenMPIRBuilder
OMPBuilder(*M
);
5701 OMPBuilder
.initialize();
5703 IRBuilder
<> Builder(BB
);
5707 OMPBuilder
.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize
);
5709 OMPBuilder
.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize
);
5710 SmallVector
<llvm::Constant
*> Names
= {Cst1
, Cst2
};
5712 GlobalVariable
*OffloadMaptypesGlobal
=
5713 OMPBuilder
.createOffloadMapnames(Names
, "offload_mapnames");
5714 EXPECT_FALSE(M
->global_empty());
5715 EXPECT_EQ(OffloadMaptypesGlobal
->getName(), "offload_mapnames");
5716 EXPECT_TRUE(OffloadMaptypesGlobal
->isConstant());
5717 EXPECT_FALSE(OffloadMaptypesGlobal
->hasGlobalUnnamedAddr());
5718 EXPECT_TRUE(OffloadMaptypesGlobal
->hasPrivateLinkage());
5719 EXPECT_TRUE(OffloadMaptypesGlobal
->hasInitializer());
5720 Constant
*Initializer
= OffloadMaptypesGlobal
->getInitializer();
5721 EXPECT_TRUE(isa
<Constant
>(Initializer
->getOperand(0)->stripPointerCasts()));
5722 EXPECT_TRUE(isa
<Constant
>(Initializer
->getOperand(1)->stripPointerCasts()));
5724 GlobalVariable
*Name1Gbl
=
5725 cast
<GlobalVariable
>(Initializer
->getOperand(0)->stripPointerCasts());
5726 EXPECT_TRUE(isa
<ConstantDataArray
>(Name1Gbl
->getInitializer()));
5727 ConstantDataArray
*Name1GblCA
=
5728 dyn_cast
<ConstantDataArray
>(Name1Gbl
->getInitializer());
5729 EXPECT_EQ(Name1GblCA
->getAsCString(), ";file1;array1;2;5;;");
5731 GlobalVariable
*Name2Gbl
=
5732 cast
<GlobalVariable
>(Initializer
->getOperand(1)->stripPointerCasts());
5733 EXPECT_TRUE(isa
<ConstantDataArray
>(Name2Gbl
->getInitializer()));
5734 ConstantDataArray
*Name2GblCA
=
5735 dyn_cast
<ConstantDataArray
>(Name2Gbl
->getInitializer());
5736 EXPECT_EQ(Name2GblCA
->getAsCString(), ";file1;array2;3;5;;");
5738 EXPECT_TRUE(Initializer
->getType()->getArrayElementType()->isPointerTy());
5739 EXPECT_EQ(Initializer
->getType()->getArrayNumElements(), Names
.size());
5742 TEST_F(OpenMPIRBuilderTest
, CreateMapperAllocas
) {
5743 OpenMPIRBuilder
OMPBuilder(*M
);
5744 OMPBuilder
.initialize();
5746 IRBuilder
<> Builder(BB
);
5748 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5750 unsigned TotalNbOperand
= 2;
5752 OpenMPIRBuilder::MapperAllocas MapperAllocas
;
5753 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5754 F
->getEntryBlock().getFirstInsertionPt());
5755 OMPBuilder
.createMapperAllocas(Loc
, AllocaIP
, TotalNbOperand
, MapperAllocas
);
5756 EXPECT_NE(MapperAllocas
.ArgsBase
, nullptr);
5757 EXPECT_NE(MapperAllocas
.Args
, nullptr);
5758 EXPECT_NE(MapperAllocas
.ArgSizes
, nullptr);
5759 EXPECT_TRUE(MapperAllocas
.ArgsBase
->getAllocatedType()->isArrayTy());
5760 ArrayType
*ArrType
=
5761 dyn_cast
<ArrayType
>(MapperAllocas
.ArgsBase
->getAllocatedType());
5762 EXPECT_EQ(ArrType
->getNumElements(), TotalNbOperand
);
5763 EXPECT_TRUE(MapperAllocas
.ArgsBase
->getAllocatedType()
5764 ->getArrayElementType()
5767 EXPECT_TRUE(MapperAllocas
.Args
->getAllocatedType()->isArrayTy());
5768 ArrType
= dyn_cast
<ArrayType
>(MapperAllocas
.Args
->getAllocatedType());
5769 EXPECT_EQ(ArrType
->getNumElements(), TotalNbOperand
);
5770 EXPECT_TRUE(MapperAllocas
.Args
->getAllocatedType()
5771 ->getArrayElementType()
5774 EXPECT_TRUE(MapperAllocas
.ArgSizes
->getAllocatedType()->isArrayTy());
5775 ArrType
= dyn_cast
<ArrayType
>(MapperAllocas
.ArgSizes
->getAllocatedType());
5776 EXPECT_EQ(ArrType
->getNumElements(), TotalNbOperand
);
5777 EXPECT_TRUE(MapperAllocas
.ArgSizes
->getAllocatedType()
5778 ->getArrayElementType()
5782 TEST_F(OpenMPIRBuilderTest
, EmitMapperCall
) {
5783 OpenMPIRBuilder
OMPBuilder(*M
);
5784 OMPBuilder
.initialize();
5786 IRBuilder
<> Builder(BB
);
5787 LLVMContext
&Ctx
= M
->getContext();
5789 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5791 unsigned TotalNbOperand
= 2;
5793 OpenMPIRBuilder::MapperAllocas MapperAllocas
;
5794 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5795 F
->getEntryBlock().getFirstInsertionPt());
5796 OMPBuilder
.createMapperAllocas(Loc
, AllocaIP
, TotalNbOperand
, MapperAllocas
);
5798 auto *BeginMapperFunc
= OMPBuilder
.getOrCreateRuntimeFunctionPtr(
5799 omp::OMPRTL___tgt_target_data_begin_mapper
);
5801 SmallVector
<uint64_t> Flags
= {0, 2};
5804 Constant
*SrcLocCst
=
5805 OMPBuilder
.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize
);
5806 Value
*SrcLocInfo
= OMPBuilder
.getOrCreateIdent(SrcLocCst
, StrSize
);
5809 OMPBuilder
.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize
);
5811 OMPBuilder
.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize
);
5812 SmallVector
<llvm::Constant
*> Names
= {Cst1
, Cst2
};
5814 GlobalVariable
*Maptypes
=
5815 OMPBuilder
.createOffloadMaptypes(Flags
, ".offload_maptypes");
5816 Value
*MaptypesArg
= Builder
.CreateConstInBoundsGEP2_32(
5817 ArrayType::get(Type::getInt64Ty(Ctx
), TotalNbOperand
), Maptypes
,
5818 /*Idx0=*/0, /*Idx1=*/0);
5820 GlobalVariable
*Mapnames
=
5821 OMPBuilder
.createOffloadMapnames(Names
, ".offload_mapnames");
5822 Value
*MapnamesArg
= Builder
.CreateConstInBoundsGEP2_32(
5823 ArrayType::get(PointerType::getUnqual(Ctx
), TotalNbOperand
), Mapnames
,
5824 /*Idx0=*/0, /*Idx1=*/0);
5826 OMPBuilder
.emitMapperCall(Builder
.saveIP(), BeginMapperFunc
, SrcLocInfo
,
5827 MaptypesArg
, MapnamesArg
, MapperAllocas
, -1,
5830 CallInst
*MapperCall
= dyn_cast
<CallInst
>(&BB
->back());
5831 EXPECT_NE(MapperCall
, nullptr);
5832 EXPECT_EQ(MapperCall
->arg_size(), 9U);
5833 EXPECT_EQ(MapperCall
->getCalledFunction()->getName(),
5834 "__tgt_target_data_begin_mapper");
5835 EXPECT_EQ(MapperCall
->getOperand(0), SrcLocInfo
);
5836 EXPECT_TRUE(MapperCall
->getOperand(1)->getType()->isIntegerTy(64));
5837 EXPECT_TRUE(MapperCall
->getOperand(2)->getType()->isIntegerTy(32));
5839 EXPECT_EQ(MapperCall
->getOperand(6), MaptypesArg
);
5840 EXPECT_EQ(MapperCall
->getOperand(7), MapnamesArg
);
5841 EXPECT_TRUE(MapperCall
->getOperand(8)->getType()->isPointerTy());
5844 TEST_F(OpenMPIRBuilderTest
, TargetEnterData
) {
5845 OpenMPIRBuilder
OMPBuilder(*M
);
5846 OMPBuilder
.initialize();
5848 IRBuilder
<> Builder(BB
);
5849 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5851 int64_t DeviceID
= 2;
5854 Builder
.CreateAlloca(Builder
.getInt32Ty(), Builder
.getInt64(1));
5855 ASSERT_NE(Val1
, nullptr);
5857 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5858 F
->getEntryBlock().getFirstInsertionPt());
5860 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo
;
5861 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5863 [&](InsertPointTy codeGenIP
) -> llvm::OpenMPIRBuilder::MapInfosTy
& {
5864 // Get map clause information.
5865 Builder
.restoreIP(codeGenIP
);
5867 CombinedInfo
.BasePointers
.emplace_back(Val1
);
5868 CombinedInfo
.Pointers
.emplace_back(Val1
);
5869 CombinedInfo
.DevicePointers
.emplace_back(
5870 llvm::OpenMPIRBuilder::DeviceInfoTy::None
);
5871 CombinedInfo
.Sizes
.emplace_back(Builder
.getInt64(4));
5872 CombinedInfo
.Types
.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(1));
5874 CombinedInfo
.Names
.emplace_back(
5875 OMPBuilder
.getOrCreateSrcLocStr("unknown", temp
));
5876 return CombinedInfo
;
5879 llvm::OpenMPIRBuilder::TargetDataInfo
Info(
5880 /*RequiresDevicePointerInfo=*/false,
5881 /*SeparateBeginEndCalls=*/true);
5883 OMPBuilder
.Config
.setIsGPU(true);
5885 llvm::omp::RuntimeFunction RTLFunc
= OMPRTL___tgt_target_data_begin_mapper
;
5886 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTargetData(
5887 Loc
, AllocaIP
, Builder
.saveIP(), Builder
.getInt64(DeviceID
),
5888 /* IfCond= */ nullptr, Info
, GenMapInfoCB
, &RTLFunc
);
5889 assert(AfterIP
&& "unexpected error");
5890 Builder
.restoreIP(*AfterIP
);
5892 CallInst
*TargetDataCall
= dyn_cast
<CallInst
>(&BB
->back());
5893 EXPECT_NE(TargetDataCall
, nullptr);
5894 EXPECT_EQ(TargetDataCall
->arg_size(), 9U);
5895 EXPECT_EQ(TargetDataCall
->getCalledFunction()->getName(),
5896 "__tgt_target_data_begin_mapper");
5897 EXPECT_TRUE(TargetDataCall
->getOperand(1)->getType()->isIntegerTy(64));
5898 EXPECT_TRUE(TargetDataCall
->getOperand(2)->getType()->isIntegerTy(32));
5899 EXPECT_TRUE(TargetDataCall
->getOperand(8)->getType()->isPointerTy());
5901 Builder
.CreateRetVoid();
5902 EXPECT_FALSE(verifyModule(*M
, &errs()));
5905 TEST_F(OpenMPIRBuilderTest
, TargetExitData
) {
5906 OpenMPIRBuilder
OMPBuilder(*M
);
5907 OMPBuilder
.initialize();
5909 IRBuilder
<> Builder(BB
);
5910 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5912 int64_t DeviceID
= 2;
5915 Builder
.CreateAlloca(Builder
.getInt32Ty(), Builder
.getInt64(1));
5916 ASSERT_NE(Val1
, nullptr);
5918 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5919 F
->getEntryBlock().getFirstInsertionPt());
5921 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo
;
5922 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5924 [&](InsertPointTy codeGenIP
) -> llvm::OpenMPIRBuilder::MapInfosTy
& {
5925 // Get map clause information.
5926 Builder
.restoreIP(codeGenIP
);
5928 CombinedInfo
.BasePointers
.emplace_back(Val1
);
5929 CombinedInfo
.Pointers
.emplace_back(Val1
);
5930 CombinedInfo
.DevicePointers
.emplace_back(
5931 llvm::OpenMPIRBuilder::DeviceInfoTy::None
);
5932 CombinedInfo
.Sizes
.emplace_back(Builder
.getInt64(4));
5933 CombinedInfo
.Types
.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(2));
5935 CombinedInfo
.Names
.emplace_back(
5936 OMPBuilder
.getOrCreateSrcLocStr("unknown", temp
));
5937 return CombinedInfo
;
5940 llvm::OpenMPIRBuilder::TargetDataInfo
Info(
5941 /*RequiresDevicePointerInfo=*/false,
5942 /*SeparateBeginEndCalls=*/true);
5944 OMPBuilder
.Config
.setIsGPU(true);
5946 llvm::omp::RuntimeFunction RTLFunc
= OMPRTL___tgt_target_data_end_mapper
;
5947 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTargetData(
5948 Loc
, AllocaIP
, Builder
.saveIP(), Builder
.getInt64(DeviceID
),
5949 /* IfCond= */ nullptr, Info
, GenMapInfoCB
, &RTLFunc
);
5950 assert(AfterIP
&& "unexpected error");
5951 Builder
.restoreIP(*AfterIP
);
5953 CallInst
*TargetDataCall
= dyn_cast
<CallInst
>(&BB
->back());
5954 EXPECT_NE(TargetDataCall
, nullptr);
5955 EXPECT_EQ(TargetDataCall
->arg_size(), 9U);
5956 EXPECT_EQ(TargetDataCall
->getCalledFunction()->getName(),
5957 "__tgt_target_data_end_mapper");
5958 EXPECT_TRUE(TargetDataCall
->getOperand(1)->getType()->isIntegerTy(64));
5959 EXPECT_TRUE(TargetDataCall
->getOperand(2)->getType()->isIntegerTy(32));
5960 EXPECT_TRUE(TargetDataCall
->getOperand(8)->getType()->isPointerTy());
5962 Builder
.CreateRetVoid();
5963 EXPECT_FALSE(verifyModule(*M
, &errs()));
5966 TEST_F(OpenMPIRBuilderTest
, TargetDataRegion
) {
5967 OpenMPIRBuilder
OMPBuilder(*M
);
5968 OMPBuilder
.initialize();
5970 IRBuilder
<> Builder(BB
);
5971 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
5973 int64_t DeviceID
= 2;
5976 Builder
.CreateAlloca(Builder
.getInt32Ty(), Builder
.getInt64(1));
5977 ASSERT_NE(Val1
, nullptr);
5979 AllocaInst
*Val2
= Builder
.CreateAlloca(Builder
.getPtrTy());
5980 ASSERT_NE(Val2
, nullptr);
5982 AllocaInst
*Val3
= Builder
.CreateAlloca(Builder
.getPtrTy());
5983 ASSERT_NE(Val3
, nullptr);
5985 IRBuilder
<>::InsertPoint
AllocaIP(&F
->getEntryBlock(),
5986 F
->getEntryBlock().getFirstInsertionPt());
5988 using DeviceInfoTy
= llvm::OpenMPIRBuilder::DeviceInfoTy
;
5989 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo
;
5990 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
5992 [&](InsertPointTy codeGenIP
) -> llvm::OpenMPIRBuilder::MapInfosTy
& {
5993 // Get map clause information.
5994 Builder
.restoreIP(codeGenIP
);
5997 CombinedInfo
.BasePointers
.emplace_back(Val1
);
5998 CombinedInfo
.Pointers
.emplace_back(Val1
);
5999 CombinedInfo
.DevicePointers
.emplace_back(DeviceInfoTy::None
);
6000 CombinedInfo
.Sizes
.emplace_back(Builder
.getInt64(4));
6001 CombinedInfo
.Types
.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(3));
6002 CombinedInfo
.Names
.emplace_back(
6003 OMPBuilder
.getOrCreateSrcLocStr("unknown", temp
));
6005 CombinedInfo
.BasePointers
.emplace_back(Val2
);
6006 CombinedInfo
.Pointers
.emplace_back(Val2
);
6007 CombinedInfo
.DevicePointers
.emplace_back(DeviceInfoTy::Pointer
);
6008 CombinedInfo
.Sizes
.emplace_back(Builder
.getInt64(8));
6009 CombinedInfo
.Types
.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(67));
6010 CombinedInfo
.Names
.emplace_back(
6011 OMPBuilder
.getOrCreateSrcLocStr("unknown", temp
));
6013 CombinedInfo
.BasePointers
.emplace_back(Val3
);
6014 CombinedInfo
.Pointers
.emplace_back(Val3
);
6015 CombinedInfo
.DevicePointers
.emplace_back(DeviceInfoTy::Address
);
6016 CombinedInfo
.Sizes
.emplace_back(Builder
.getInt64(8));
6017 CombinedInfo
.Types
.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(67));
6018 CombinedInfo
.Names
.emplace_back(
6019 OMPBuilder
.getOrCreateSrcLocStr("unknown", temp
));
6020 return CombinedInfo
;
6023 llvm::OpenMPIRBuilder::TargetDataInfo
Info(
6024 /*RequiresDevicePointerInfo=*/true,
6025 /*SeparateBeginEndCalls=*/true);
6027 OMPBuilder
.Config
.setIsGPU(true);
6029 using BodyGenTy
= llvm::OpenMPIRBuilder::BodyGenTy
;
6030 auto BodyCB
= [&](InsertPointTy CodeGenIP
, BodyGenTy BodyGenType
) {
6031 if (BodyGenType
== BodyGenTy::Priv
) {
6032 EXPECT_EQ(Info
.DevicePtrInfoMap
.size(), 2u);
6033 Builder
.restoreIP(CodeGenIP
);
6034 CallInst
*TargetDataCall
=
6035 dyn_cast
<CallInst
>(BB
->back().getPrevNode()->getPrevNode());
6036 EXPECT_NE(TargetDataCall
, nullptr);
6037 EXPECT_EQ(TargetDataCall
->arg_size(), 9U);
6038 EXPECT_EQ(TargetDataCall
->getCalledFunction()->getName(),
6039 "__tgt_target_data_begin_mapper");
6040 EXPECT_TRUE(TargetDataCall
->getOperand(1)->getType()->isIntegerTy(64));
6041 EXPECT_TRUE(TargetDataCall
->getOperand(2)->getType()->isIntegerTy(32));
6042 EXPECT_TRUE(TargetDataCall
->getOperand(8)->getType()->isPointerTy());
6044 LoadInst
*LI
= dyn_cast
<LoadInst
>(BB
->back().getPrevNode());
6045 EXPECT_NE(LI
, nullptr);
6046 StoreInst
*SI
= dyn_cast
<StoreInst
>(&BB
->back());
6047 EXPECT_NE(SI
, nullptr);
6048 EXPECT_EQ(SI
->getValueOperand(), LI
);
6049 EXPECT_EQ(SI
->getPointerOperand(), Info
.DevicePtrInfoMap
[Val2
].second
);
6050 EXPECT_TRUE(isa
<AllocaInst
>(Info
.DevicePtrInfoMap
[Val2
].second
));
6051 EXPECT_TRUE(isa
<GetElementPtrInst
>(Info
.DevicePtrInfoMap
[Val3
].second
));
6052 Builder
.CreateStore(Builder
.getInt32(99), Val1
);
6054 return Builder
.saveIP();
6057 OpenMPIRBuilder::InsertPointOrErrorTy TargetDataIP1
=
6058 OMPBuilder
.createTargetData(
6059 Loc
, AllocaIP
, Builder
.saveIP(), Builder
.getInt64(DeviceID
),
6060 /* IfCond= */ nullptr, Info
, GenMapInfoCB
, nullptr, BodyCB
);
6061 assert(TargetDataIP1
&& "unexpected error");
6062 Builder
.restoreIP(*TargetDataIP1
);
6064 CallInst
*TargetDataCall
= dyn_cast
<CallInst
>(&BB
->back());
6065 EXPECT_NE(TargetDataCall
, nullptr);
6066 EXPECT_EQ(TargetDataCall
->arg_size(), 9U);
6067 EXPECT_EQ(TargetDataCall
->getCalledFunction()->getName(),
6068 "__tgt_target_data_end_mapper");
6069 EXPECT_TRUE(TargetDataCall
->getOperand(1)->getType()->isIntegerTy(64));
6070 EXPECT_TRUE(TargetDataCall
->getOperand(2)->getType()->isIntegerTy(32));
6071 EXPECT_TRUE(TargetDataCall
->getOperand(8)->getType()->isPointerTy());
6073 // Check that BodyGenCB is still made when IsTargetDevice is set to true.
6074 OMPBuilder
.Config
.setIsTargetDevice(true);
6075 bool CheckDevicePassBodyGen
= false;
6076 auto BodyTargetCB
= [&](InsertPointTy CodeGenIP
, BodyGenTy BodyGenType
) {
6077 CheckDevicePassBodyGen
= true;
6078 Builder
.restoreIP(CodeGenIP
);
6079 CallInst
*TargetDataCall
=
6080 dyn_cast
<CallInst
>(BB
->back().getPrevNode()->getPrevNode());
6081 // Make sure no begin_mapper call is present for device pass.
6082 EXPECT_EQ(TargetDataCall
, nullptr);
6083 return Builder
.saveIP();
6085 OpenMPIRBuilder::InsertPointOrErrorTy TargetDataIP2
=
6086 OMPBuilder
.createTargetData(
6087 Loc
, AllocaIP
, Builder
.saveIP(), Builder
.getInt64(DeviceID
),
6088 /* IfCond= */ nullptr, Info
, GenMapInfoCB
, nullptr, BodyTargetCB
);
6089 assert(TargetDataIP2
&& "unexpected error");
6090 Builder
.restoreIP(*TargetDataIP2
);
6091 EXPECT_TRUE(CheckDevicePassBodyGen
);
6093 Builder
.CreateRetVoid();
6094 EXPECT_FALSE(verifyModule(*M
, &errs()));
6098 // Some basic handling of argument mapping for the moment
6099 void CreateDefaultMapInfos(llvm::OpenMPIRBuilder
&OmpBuilder
,
6100 llvm::SmallVectorImpl
<llvm::Value
*> &Args
,
6101 llvm::OpenMPIRBuilder::MapInfosTy
&CombinedInfo
) {
6102 for (auto Arg
: Args
) {
6103 CombinedInfo
.BasePointers
.emplace_back(Arg
);
6104 CombinedInfo
.Pointers
.emplace_back(Arg
);
6105 uint32_t SrcLocStrSize
;
6106 CombinedInfo
.Names
.emplace_back(OmpBuilder
.getOrCreateSrcLocStr(
6107 "Unknown loc - stub implementation", SrcLocStrSize
));
6108 CombinedInfo
.Types
.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(
6109 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO
|
6110 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM
|
6111 llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM
));
6112 CombinedInfo
.Sizes
.emplace_back(OmpBuilder
.Builder
.getInt64(
6113 OmpBuilder
.M
.getDataLayout().getTypeAllocSize(Arg
->getType())));
6118 TEST_F(OpenMPIRBuilderTest
, TargetRegion
) {
6119 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6120 OpenMPIRBuilder
OMPBuilder(*M
);
6121 OMPBuilder
.initialize();
6122 OpenMPIRBuilderConfig
Config(false, false, false, false, false, false, false);
6123 OMPBuilder
.setConfig(Config
);
6125 IRBuilder
<> Builder(BB
);
6126 auto Int32Ty
= Builder
.getInt32Ty();
6128 AllocaInst
*APtr
= Builder
.CreateAlloca(Int32Ty
, nullptr, "a_ptr");
6129 AllocaInst
*BPtr
= Builder
.CreateAlloca(Int32Ty
, nullptr, "b_ptr");
6130 AllocaInst
*CPtr
= Builder
.CreateAlloca(Int32Ty
, nullptr, "c_ptr");
6132 Builder
.CreateStore(Builder
.getInt32(10), APtr
);
6133 Builder
.CreateStore(Builder
.getInt32(20), BPtr
);
6134 auto BodyGenCB
= [&](InsertPointTy AllocaIP
,
6135 InsertPointTy CodeGenIP
) -> InsertPointTy
{
6136 Builder
.restoreIP(CodeGenIP
);
6137 LoadInst
*AVal
= Builder
.CreateLoad(Int32Ty
, APtr
);
6138 LoadInst
*BVal
= Builder
.CreateLoad(Int32Ty
, BPtr
);
6139 Value
*Sum
= Builder
.CreateAdd(AVal
, BVal
);
6140 Builder
.CreateStore(Sum
, CPtr
);
6141 return Builder
.saveIP();
6144 llvm::SmallVector
<llvm::Value
*> Inputs
;
6145 Inputs
.push_back(APtr
);
6146 Inputs
.push_back(BPtr
);
6147 Inputs
.push_back(CPtr
);
6149 auto SimpleArgAccessorCB
=
6150 [&](llvm::Argument
&Arg
, llvm::Value
*Input
, llvm::Value
*&RetVal
,
6151 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP
,
6152 llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP
) {
6153 if (!OMPBuilder
.Config
.isTargetDevice()) {
6154 RetVal
= cast
<llvm::Value
>(&Arg
);
6158 Builder
.restoreIP(AllocaIP
);
6160 llvm::Value
*Addr
= Builder
.CreateAlloca(
6161 Arg
.getType()->isPointerTy()
6163 : Type::getInt64Ty(Builder
.getContext()),
6164 OMPBuilder
.M
.getDataLayout().getAllocaAddrSpace());
6165 llvm::Value
*AddrAscast
=
6166 Builder
.CreatePointerBitCastOrAddrSpaceCast(Addr
, Input
->getType());
6167 Builder
.CreateStore(&Arg
, AddrAscast
);
6169 Builder
.restoreIP(CodeGenIP
);
6171 RetVal
= Builder
.CreateLoad(Arg
.getType(), AddrAscast
);
6173 return Builder
.saveIP();
6176 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos
;
6177 auto GenMapInfoCB
= [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP
)
6178 -> llvm::OpenMPIRBuilder::MapInfosTy
& {
6179 CreateDefaultMapInfos(OMPBuilder
, Inputs
, CombinedInfos
);
6180 return CombinedInfos
;
6183 TargetRegionEntryInfo
EntryInfo("func", 42, 4711, 17);
6184 OpenMPIRBuilder::LocationDescription
OmpLoc({Builder
.saveIP(), DL
});
6185 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTarget(
6186 OmpLoc
, /*IsOffloadEntry=*/true, Builder
.saveIP(), Builder
.saveIP(),
6187 EntryInfo
, -1, 0, Inputs
, GenMapInfoCB
, BodyGenCB
, SimpleArgAccessorCB
);
6188 assert(AfterIP
&& "unexpected error");
6189 Builder
.restoreIP(*AfterIP
);
6190 OMPBuilder
.finalize();
6191 Builder
.CreateRetVoid();
6193 // Check the kernel launch sequence
6194 auto Iter
= F
->getEntryBlock().rbegin();
6195 EXPECT_TRUE(isa
<BranchInst
>(&*(Iter
)));
6196 BranchInst
*Branch
= dyn_cast
<BranchInst
>(&*(Iter
));
6197 EXPECT_TRUE(isa
<CmpInst
>(&*(++Iter
)));
6198 EXPECT_TRUE(isa
<CallInst
>(&*(++Iter
)));
6199 CallInst
*Call
= dyn_cast
<CallInst
>(&*(Iter
));
6201 // Check that the kernel launch function is called
6202 Function
*KernelLaunchFunc
= Call
->getCalledFunction();
6203 EXPECT_NE(KernelLaunchFunc
, nullptr);
6204 StringRef FunctionName
= KernelLaunchFunc
->getName();
6205 EXPECT_TRUE(FunctionName
.starts_with("__tgt_target_kernel"));
6207 // Check the fallback call
6208 BasicBlock
*FallbackBlock
= Branch
->getSuccessor(0);
6209 Iter
= FallbackBlock
->rbegin();
6210 CallInst
*FCall
= dyn_cast
<CallInst
>(&*(++Iter
));
6211 // 'F' has a dummy DISubprogram which causes OutlinedFunc to also
6212 // have a DISubprogram. In this case, the call to OutlinedFunc needs
6213 // to have a debug loc, otherwise verifier will complain.
6214 FCall
->setDebugLoc(DL
);
6215 EXPECT_NE(FCall
, nullptr);
6217 // Check that the correct aguments are passed in
6218 for (auto ArgInput
: zip(FCall
->args(), Inputs
)) {
6219 EXPECT_EQ(std::get
<0>(ArgInput
), std::get
<1>(ArgInput
));
6222 // Check that the outlined function exists with the expected prefix
6223 Function
*OutlinedFunc
= FCall
->getCalledFunction();
6224 EXPECT_NE(OutlinedFunc
, nullptr);
6225 StringRef FunctionName2
= OutlinedFunc
->getName();
6226 EXPECT_TRUE(FunctionName2
.starts_with("__omp_offloading"));
6228 EXPECT_FALSE(verifyModule(*M
, &errs()));
6231 TEST_F(OpenMPIRBuilderTest
, TargetRegionDevice
) {
6232 OpenMPIRBuilder
OMPBuilder(*M
);
6233 OMPBuilder
.setConfig(
6234 OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
6235 OMPBuilder
.initialize();
6238 IRBuilder
<> Builder(BB
);
6239 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
6241 LoadInst
*Value
= nullptr;
6242 StoreInst
*TargetStore
= nullptr;
6243 llvm::SmallVector
<llvm::Value
*, 2> CapturedArgs
= {
6244 Constant::getNullValue(PointerType::get(Ctx
, 0)),
6245 Constant::getNullValue(PointerType::get(Ctx
, 0))};
6247 auto SimpleArgAccessorCB
=
6248 [&](llvm::Argument
&Arg
, llvm::Value
*Input
, llvm::Value
*&RetVal
,
6249 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP
,
6250 llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP
) {
6251 if (!OMPBuilder
.Config
.isTargetDevice()) {
6252 RetVal
= cast
<llvm::Value
>(&Arg
);
6256 Builder
.restoreIP(AllocaIP
);
6258 llvm::Value
*Addr
= Builder
.CreateAlloca(
6259 Arg
.getType()->isPointerTy()
6261 : Type::getInt64Ty(Builder
.getContext()),
6262 OMPBuilder
.M
.getDataLayout().getAllocaAddrSpace());
6263 llvm::Value
*AddrAscast
=
6264 Builder
.CreatePointerBitCastOrAddrSpaceCast(Addr
, Input
->getType());
6265 Builder
.CreateStore(&Arg
, AddrAscast
);
6267 Builder
.restoreIP(CodeGenIP
);
6269 RetVal
= Builder
.CreateLoad(Arg
.getType(), AddrAscast
);
6271 return Builder
.saveIP();
6274 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos
;
6275 auto GenMapInfoCB
= [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP
)
6276 -> llvm::OpenMPIRBuilder::MapInfosTy
& {
6277 CreateDefaultMapInfos(OMPBuilder
, CapturedArgs
, CombinedInfos
);
6278 return CombinedInfos
;
6281 auto BodyGenCB
= [&](OpenMPIRBuilder::InsertPointTy AllocaIP
,
6282 OpenMPIRBuilder::InsertPointTy CodeGenIP
)
6283 -> OpenMPIRBuilder::InsertPointTy
{
6284 Builder
.restoreIP(CodeGenIP
);
6285 Value
= Builder
.CreateLoad(Type::getInt32Ty(Ctx
), CapturedArgs
[0]);
6286 TargetStore
= Builder
.CreateStore(Value
, CapturedArgs
[1]);
6287 return Builder
.saveIP();
6290 IRBuilder
<>::InsertPoint
EntryIP(&F
->getEntryBlock(),
6291 F
->getEntryBlock().getFirstInsertionPt());
6292 TargetRegionEntryInfo
EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
6293 /*Line=*/3, /*Count=*/0);
6295 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
6296 OMPBuilder
.createTarget(Loc
, /*IsOffloadEntry=*/true, EntryIP
, EntryIP
,
6297 EntryInfo
, /*NumTeams=*/-1,
6298 /*NumThreads=*/0, CapturedArgs
, GenMapInfoCB
,
6299 BodyGenCB
, SimpleArgAccessorCB
);
6300 assert(AfterIP
&& "unexpected error");
6301 Builder
.restoreIP(*AfterIP
);
6303 Builder
.CreateRetVoid();
6304 OMPBuilder
.finalize();
6306 // Check outlined function
6307 EXPECT_FALSE(verifyModule(*M
, &errs()));
6308 EXPECT_NE(TargetStore
, nullptr);
6309 Function
*OutlinedFn
= TargetStore
->getFunction();
6310 EXPECT_NE(F
, OutlinedFn
);
6312 EXPECT_TRUE(OutlinedFn
->hasWeakODRLinkage());
6313 // Account for the "implicit" first argument.
6314 EXPECT_EQ(OutlinedFn
->getName(), "__omp_offloading_1_2_parent_l3");
6315 EXPECT_EQ(OutlinedFn
->arg_size(), 3U);
6316 EXPECT_TRUE(OutlinedFn
->getArg(1)->getType()->isPointerTy());
6317 EXPECT_TRUE(OutlinedFn
->getArg(2)->getType()->isPointerTy());
6319 // Check entry block
6320 auto &EntryBlock
= OutlinedFn
->getEntryBlock();
6321 Instruction
*Alloca1
= EntryBlock
.getFirstNonPHI();
6322 EXPECT_NE(Alloca1
, nullptr);
6324 EXPECT_TRUE(isa
<AllocaInst
>(Alloca1
));
6325 auto *Store1
= Alloca1
->getNextNode();
6326 EXPECT_TRUE(isa
<StoreInst
>(Store1
));
6327 auto *Alloca2
= Store1
->getNextNode();
6328 EXPECT_TRUE(isa
<AllocaInst
>(Alloca2
));
6329 auto *Store2
= Alloca2
->getNextNode();
6330 EXPECT_TRUE(isa
<StoreInst
>(Store2
));
6332 auto *InitCall
= dyn_cast
<CallInst
>(Store2
->getNextNode());
6333 EXPECT_NE(InitCall
, nullptr);
6334 EXPECT_EQ(InitCall
->getCalledFunction()->getName(), "__kmpc_target_init");
6335 EXPECT_EQ(InitCall
->arg_size(), 2U);
6336 EXPECT_TRUE(isa
<GlobalVariable
>(InitCall
->getArgOperand(0)));
6337 auto *KernelEnvGV
= cast
<GlobalVariable
>(InitCall
->getArgOperand(0));
6338 EXPECT_TRUE(isa
<ConstantStruct
>(KernelEnvGV
->getInitializer()));
6339 auto *KernelEnvC
= cast
<ConstantStruct
>(KernelEnvGV
->getInitializer());
6340 EXPECT_TRUE(isa
<ConstantStruct
>(KernelEnvC
->getAggregateElement(0U)));
6341 auto ConfigC
= cast
<ConstantStruct
>(KernelEnvC
->getAggregateElement(0U));
6342 EXPECT_EQ(ConfigC
->getAggregateElement(0U),
6343 ConstantInt::get(Type::getInt8Ty(Ctx
), true));
6344 EXPECT_EQ(ConfigC
->getAggregateElement(1U),
6345 ConstantInt::get(Type::getInt8Ty(Ctx
), true));
6346 EXPECT_EQ(ConfigC
->getAggregateElement(2U),
6347 ConstantInt::get(Type::getInt8Ty(Ctx
), OMP_TGT_EXEC_MODE_GENERIC
));
6349 auto *EntryBlockBranch
= EntryBlock
.getTerminator();
6350 EXPECT_NE(EntryBlockBranch
, nullptr);
6351 EXPECT_EQ(EntryBlockBranch
->getNumSuccessors(), 2U);
6353 // Check user code block
6354 auto *UserCodeBlock
= EntryBlockBranch
->getSuccessor(0);
6355 EXPECT_EQ(UserCodeBlock
->getName(), "user_code.entry");
6356 auto *Load1
= UserCodeBlock
->getFirstNonPHI();
6357 EXPECT_TRUE(isa
<LoadInst
>(Load1
));
6358 auto *Load2
= Load1
->getNextNode();
6359 EXPECT_TRUE(isa
<LoadInst
>(Load2
));
6361 auto *Value1
= Load2
->getNextNode();
6362 EXPECT_EQ(Value1
, Value
);
6363 EXPECT_EQ(Value1
->getNextNode(), TargetStore
);
6364 auto *Deinit
= TargetStore
->getNextNode();
6365 EXPECT_NE(Deinit
, nullptr);
6367 auto *DeinitCall
= dyn_cast
<CallInst
>(Deinit
);
6368 EXPECT_NE(DeinitCall
, nullptr);
6369 EXPECT_EQ(DeinitCall
->getCalledFunction()->getName(), "__kmpc_target_deinit");
6370 EXPECT_EQ(DeinitCall
->arg_size(), 0U);
6372 EXPECT_TRUE(isa
<ReturnInst
>(DeinitCall
->getNextNode()));
6375 auto *ExitBlock
= EntryBlockBranch
->getSuccessor(1);
6376 EXPECT_EQ(ExitBlock
->getName(), "worker.exit");
6377 EXPECT_TRUE(isa
<ReturnInst
>(ExitBlock
->getFirstNonPHI()));
6380 TEST_F(OpenMPIRBuilderTest
, ConstantAllocaRaise
) {
6381 OpenMPIRBuilder
OMPBuilder(*M
);
6382 OMPBuilder
.setConfig(
6383 OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
6384 OMPBuilder
.initialize();
6387 IRBuilder
<> Builder(BB
);
6388 OpenMPIRBuilder::LocationDescription
Loc({Builder
.saveIP(), DL
});
6390 LoadInst
*Value
= nullptr;
6391 StoreInst
*TargetStore
= nullptr;
6392 llvm::SmallVector
<llvm::Value
*, 1> CapturedArgs
= {
6393 Constant::getNullValue(PointerType::get(Ctx
, 0))};
6395 auto SimpleArgAccessorCB
=
6396 [&](llvm::Argument
&Arg
, llvm::Value
*Input
, llvm::Value
*&RetVal
,
6397 llvm::OpenMPIRBuilder::InsertPointTy AllocaIP
,
6398 llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP
) {
6399 if (!OMPBuilder
.Config
.isTargetDevice()) {
6400 RetVal
= cast
<llvm::Value
>(&Arg
);
6404 Builder
.restoreIP(AllocaIP
);
6406 llvm::Value
*Addr
= Builder
.CreateAlloca(
6407 Arg
.getType()->isPointerTy()
6409 : Type::getInt64Ty(Builder
.getContext()),
6410 OMPBuilder
.M
.getDataLayout().getAllocaAddrSpace());
6411 llvm::Value
*AddrAscast
=
6412 Builder
.CreatePointerBitCastOrAddrSpaceCast(Addr
, Input
->getType());
6413 Builder
.CreateStore(&Arg
, AddrAscast
);
6415 Builder
.restoreIP(CodeGenIP
);
6417 RetVal
= Builder
.CreateLoad(Arg
.getType(), AddrAscast
);
6419 return Builder
.saveIP();
6422 llvm::OpenMPIRBuilder::MapInfosTy CombinedInfos
;
6423 auto GenMapInfoCB
= [&](llvm::OpenMPIRBuilder::InsertPointTy codeGenIP
)
6424 -> llvm::OpenMPIRBuilder::MapInfosTy
& {
6425 CreateDefaultMapInfos(OMPBuilder
, CapturedArgs
, CombinedInfos
);
6426 return CombinedInfos
;
6429 llvm::Value
*RaiseAlloca
= nullptr;
6431 auto BodyGenCB
= [&](OpenMPIRBuilder::InsertPointTy AllocaIP
,
6432 OpenMPIRBuilder::InsertPointTy CodeGenIP
)
6433 -> OpenMPIRBuilder::InsertPointTy
{
6434 Builder
.restoreIP(CodeGenIP
);
6435 RaiseAlloca
= Builder
.CreateAlloca(Builder
.getInt32Ty());
6436 Value
= Builder
.CreateLoad(Type::getInt32Ty(Ctx
), CapturedArgs
[0]);
6437 TargetStore
= Builder
.CreateStore(Value
, RaiseAlloca
);
6438 return Builder
.saveIP();
6441 IRBuilder
<>::InsertPoint
EntryIP(&F
->getEntryBlock(),
6442 F
->getEntryBlock().getFirstInsertionPt());
6443 TargetRegionEntryInfo
EntryInfo("parent", /*DeviceID=*/1, /*FileID=*/2,
6444 /*Line=*/3, /*Count=*/0);
6446 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
6447 OMPBuilder
.createTarget(Loc
, /*IsOffloadEntry=*/true, EntryIP
, EntryIP
,
6448 EntryInfo
, /*NumTeams=*/-1,
6449 /*NumThreads=*/0, CapturedArgs
, GenMapInfoCB
,
6450 BodyGenCB
, SimpleArgAccessorCB
);
6451 assert(AfterIP
&& "unexpected error");
6452 Builder
.restoreIP(*AfterIP
);
6454 Builder
.CreateRetVoid();
6455 OMPBuilder
.finalize();
6457 // Check outlined function
6458 EXPECT_FALSE(verifyModule(*M
, &errs()));
6459 EXPECT_NE(TargetStore
, nullptr);
6460 Function
*OutlinedFn
= TargetStore
->getFunction();
6461 EXPECT_NE(F
, OutlinedFn
);
6463 EXPECT_TRUE(OutlinedFn
->hasWeakODRLinkage());
6464 // Account for the "implicit" first argument.
6465 EXPECT_EQ(OutlinedFn
->getName(), "__omp_offloading_1_2_parent_l3");
6466 EXPECT_EQ(OutlinedFn
->arg_size(), 2U);
6467 EXPECT_TRUE(OutlinedFn
->getArg(1)->getType()->isPointerTy());
6469 // Check entry block, to see if we have raised our alloca
6470 // from the body to the entry block.
6471 auto &EntryBlock
= OutlinedFn
->getEntryBlock();
6473 // Check that we have moved our alloca created in the
6474 // BodyGenCB function, to the top of the function.
6475 Instruction
*Alloca1
= EntryBlock
.getFirstNonPHI();
6476 EXPECT_NE(Alloca1
, nullptr);
6477 EXPECT_TRUE(isa
<AllocaInst
>(Alloca1
));
6478 EXPECT_EQ(Alloca1
, RaiseAlloca
);
6480 // Verify we have not altered the rest of the function
6481 // inappropriately with our alloca movement.
6482 auto *Alloca2
= Alloca1
->getNextNode();
6483 EXPECT_TRUE(isa
<AllocaInst
>(Alloca2
));
6484 auto *Store2
= Alloca2
->getNextNode();
6485 EXPECT_TRUE(isa
<StoreInst
>(Store2
));
6487 auto *InitCall
= dyn_cast
<CallInst
>(Store2
->getNextNode());
6488 EXPECT_NE(InitCall
, nullptr);
6489 EXPECT_EQ(InitCall
->getCalledFunction()->getName(), "__kmpc_target_init");
6490 EXPECT_EQ(InitCall
->arg_size(), 2U);
6491 EXPECT_TRUE(isa
<GlobalVariable
>(InitCall
->getArgOperand(0)));
6492 auto *KernelEnvGV
= cast
<GlobalVariable
>(InitCall
->getArgOperand(0));
6493 EXPECT_TRUE(isa
<ConstantStruct
>(KernelEnvGV
->getInitializer()));
6494 auto *KernelEnvC
= cast
<ConstantStruct
>(KernelEnvGV
->getInitializer());
6495 EXPECT_TRUE(isa
<ConstantStruct
>(KernelEnvC
->getAggregateElement(0U)));
6496 auto *ConfigC
= cast
<ConstantStruct
>(KernelEnvC
->getAggregateElement(0U));
6497 EXPECT_EQ(ConfigC
->getAggregateElement(0U),
6498 ConstantInt::get(Type::getInt8Ty(Ctx
), true));
6499 EXPECT_EQ(ConfigC
->getAggregateElement(1U),
6500 ConstantInt::get(Type::getInt8Ty(Ctx
), true));
6501 EXPECT_EQ(ConfigC
->getAggregateElement(2U),
6502 ConstantInt::get(Type::getInt8Ty(Ctx
), OMP_TGT_EXEC_MODE_GENERIC
));
6504 auto *EntryBlockBranch
= EntryBlock
.getTerminator();
6505 EXPECT_NE(EntryBlockBranch
, nullptr);
6506 EXPECT_EQ(EntryBlockBranch
->getNumSuccessors(), 2U);
6508 // Check user code block
6509 auto *UserCodeBlock
= EntryBlockBranch
->getSuccessor(0);
6510 EXPECT_EQ(UserCodeBlock
->getName(), "user_code.entry");
6511 auto *Load1
= UserCodeBlock
->getFirstNonPHI();
6512 EXPECT_TRUE(isa
<LoadInst
>(Load1
));
6513 auto *Load2
= Load1
->getNextNode();
6514 EXPECT_TRUE(isa
<LoadInst
>(Load2
));
6515 EXPECT_EQ(Load2
, Value
);
6516 EXPECT_EQ(Load2
->getNextNode(), TargetStore
);
6517 auto *Deinit
= TargetStore
->getNextNode();
6518 EXPECT_NE(Deinit
, nullptr);
6520 auto *DeinitCall
= dyn_cast
<CallInst
>(Deinit
);
6521 EXPECT_NE(DeinitCall
, nullptr);
6522 EXPECT_EQ(DeinitCall
->getCalledFunction()->getName(), "__kmpc_target_deinit");
6523 EXPECT_EQ(DeinitCall
->arg_size(), 0U);
6525 EXPECT_TRUE(isa
<ReturnInst
>(DeinitCall
->getNextNode()));
6528 auto *ExitBlock
= EntryBlockBranch
->getSuccessor(1);
6529 EXPECT_EQ(ExitBlock
->getName(), "worker.exit");
6530 EXPECT_TRUE(isa
<ReturnInst
>(ExitBlock
->getFirstNonPHI()));
6533 TEST_F(OpenMPIRBuilderTest
, CreateTask
) {
6534 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6535 OpenMPIRBuilder
OMPBuilder(*M
);
6536 OMPBuilder
.Config
.IsTargetDevice
= false;
6537 OMPBuilder
.initialize();
6539 IRBuilder
<> Builder(BB
);
6541 AllocaInst
*ValPtr32
= Builder
.CreateAlloca(Builder
.getInt32Ty());
6542 AllocaInst
*ValPtr128
= Builder
.CreateAlloca(Builder
.getInt128Ty());
6544 Builder
.CreateLoad(Builder
.getInt128Ty(), ValPtr128
, "bodygen.load");
6546 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6547 Builder
.restoreIP(AllocaIP
);
6548 AllocaInst
*Local128
= Builder
.CreateAlloca(Builder
.getInt128Ty(), nullptr,
6549 "bodygen.alloca128");
6551 Builder
.restoreIP(CodeGenIP
);
6552 // Loading and storing captured pointer and values
6553 Builder
.CreateStore(Val128
, Local128
);
6554 Value
*Val32
= Builder
.CreateLoad(ValPtr32
->getAllocatedType(), ValPtr32
,
6557 LoadInst
*PrivLoad128
= Builder
.CreateLoad(
6558 Local128
->getAllocatedType(), Local128
, "bodygen.local.load128");
6559 Value
*Cmp
= Builder
.CreateICmpNE(
6560 Val32
, Builder
.CreateTrunc(PrivLoad128
, Val32
->getType()));
6561 Instruction
*ThenTerm
, *ElseTerm
;
6562 SplitBlockAndInsertIfThenElse(Cmp
, CodeGenIP
.getBlock()->getTerminator(),
6563 &ThenTerm
, &ElseTerm
);
6564 return Error::success();
6567 BasicBlock
*AllocaBB
= Builder
.GetInsertBlock();
6568 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6569 OpenMPIRBuilder::LocationDescription
Loc(
6570 InsertPointTy(BodyBB
, BodyBB
->getFirstInsertionPt()), DL
);
6571 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTask(
6572 Loc
, InsertPointTy(AllocaBB
, AllocaBB
->getFirstInsertionPt()), BodyGenCB
);
6573 assert(AfterIP
&& "unexpected error");
6574 Builder
.restoreIP(*AfterIP
);
6575 OMPBuilder
.finalize();
6576 Builder
.CreateRetVoid();
6578 EXPECT_FALSE(verifyModule(*M
, &errs()));
6580 CallInst
*TaskAllocCall
= dyn_cast
<CallInst
>(
6581 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc
)
6584 // Verify the Ident argument
6585 GlobalVariable
*Ident
= cast
<GlobalVariable
>(TaskAllocCall
->getArgOperand(0));
6586 ASSERT_NE(Ident
, nullptr);
6587 EXPECT_TRUE(Ident
->hasInitializer());
6588 Constant
*Initializer
= Ident
->getInitializer();
6589 GlobalVariable
*SrcStrGlob
=
6590 cast
<GlobalVariable
>(Initializer
->getOperand(4)->stripPointerCasts());
6591 ASSERT_NE(SrcStrGlob
, nullptr);
6592 ConstantDataArray
*SrcSrc
=
6593 dyn_cast
<ConstantDataArray
>(SrcStrGlob
->getInitializer());
6594 ASSERT_NE(SrcSrc
, nullptr);
6596 // Verify the num_threads argument.
6597 CallInst
*GTID
= dyn_cast
<CallInst
>(TaskAllocCall
->getArgOperand(1));
6598 ASSERT_NE(GTID
, nullptr);
6599 EXPECT_EQ(GTID
->arg_size(), 1U);
6600 EXPECT_EQ(GTID
->getCalledFunction()->getName(), "__kmpc_global_thread_num");
6603 // TODO: Check for others flags. Currently testing only for tiedness.
6604 ConstantInt
*Flags
= dyn_cast
<ConstantInt
>(TaskAllocCall
->getArgOperand(2));
6605 ASSERT_NE(Flags
, nullptr);
6606 EXPECT_EQ(Flags
->getSExtValue(), 1);
6608 // Verify the data size
6609 ConstantInt
*DataSize
=
6610 dyn_cast
<ConstantInt
>(TaskAllocCall
->getArgOperand(3));
6611 ASSERT_NE(DataSize
, nullptr);
6612 EXPECT_EQ(DataSize
->getSExtValue(), 40);
6614 ConstantInt
*SharedsSize
=
6615 dyn_cast
<ConstantInt
>(TaskAllocCall
->getOperand(4));
6616 EXPECT_EQ(SharedsSize
->getSExtValue(),
6617 24); // 64-bit pointer + 128-bit integer
6619 // Verify Wrapper function
6620 Function
*OutlinedFn
=
6621 dyn_cast
<Function
>(TaskAllocCall
->getArgOperand(5)->stripPointerCasts());
6622 ASSERT_NE(OutlinedFn
, nullptr);
6624 LoadInst
*SharedsLoad
= dyn_cast
<LoadInst
>(OutlinedFn
->begin()->begin());
6625 ASSERT_NE(SharedsLoad
, nullptr);
6626 EXPECT_EQ(SharedsLoad
->getPointerOperand(), OutlinedFn
->getArg(1));
6628 EXPECT_FALSE(OutlinedFn
->isDeclaration());
6629 EXPECT_EQ(OutlinedFn
->getArg(0)->getType(), Builder
.getInt32Ty());
6631 // Verify that the data argument is used only once, and that too in the load
6632 // instruction that is then used for accessing shared data.
6633 Value
*DataPtr
= OutlinedFn
->getArg(1);
6634 EXPECT_EQ(DataPtr
->getNumUses(), 1U);
6635 EXPECT_TRUE(isa
<LoadInst
>(DataPtr
->uses().begin()->getUser()));
6636 Value
*Data
= DataPtr
->uses().begin()->getUser();
6637 EXPECT_TRUE(all_of(Data
->uses(), [](Use
&U
) {
6638 return isa
<GetElementPtrInst
>(U
.getUser());
6641 // Verify the presence of `trunc` and `icmp` instructions in Outlined function
6642 EXPECT_TRUE(any_of(instructions(OutlinedFn
),
6643 [](Instruction
&inst
) { return isa
<TruncInst
>(&inst
); }));
6644 EXPECT_TRUE(any_of(instructions(OutlinedFn
),
6645 [](Instruction
&inst
) { return isa
<ICmpInst
>(&inst
); }));
6647 // Verify the execution of the task
6648 CallInst
*TaskCall
= dyn_cast
<CallInst
>(
6649 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task
)
6651 ASSERT_NE(TaskCall
, nullptr);
6652 EXPECT_EQ(TaskCall
->getArgOperand(0), Ident
);
6653 EXPECT_EQ(TaskCall
->getArgOperand(1), GTID
);
6654 EXPECT_EQ(TaskCall
->getArgOperand(2), TaskAllocCall
);
6656 // Verify that the argument data has been copied
6657 for (User
*in
: TaskAllocCall
->users()) {
6658 if (MemCpyInst
*memCpyInst
= dyn_cast
<MemCpyInst
>(in
)) {
6659 EXPECT_EQ(memCpyInst
->getDest(), TaskAllocCall
);
6664 TEST_F(OpenMPIRBuilderTest
, CreateTaskNoArgs
) {
6665 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6666 OpenMPIRBuilder
OMPBuilder(*M
);
6667 OMPBuilder
.Config
.IsTargetDevice
= false;
6668 OMPBuilder
.initialize();
6670 IRBuilder
<> Builder(BB
);
6672 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6673 return Error::success();
6676 BasicBlock
*AllocaBB
= Builder
.GetInsertBlock();
6677 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6678 OpenMPIRBuilder::LocationDescription
Loc(
6679 InsertPointTy(BodyBB
, BodyBB
->getFirstInsertionPt()), DL
);
6680 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTask(
6681 Loc
, InsertPointTy(AllocaBB
, AllocaBB
->getFirstInsertionPt()), BodyGenCB
);
6682 assert(AfterIP
&& "unexpected error");
6683 Builder
.restoreIP(*AfterIP
);
6684 OMPBuilder
.finalize();
6685 Builder
.CreateRetVoid();
6687 EXPECT_FALSE(verifyModule(*M
, &errs()));
6689 // Check that the outlined function has only one argument.
6690 CallInst
*TaskAllocCall
= dyn_cast
<CallInst
>(
6691 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc
)
6693 Function
*OutlinedFn
= dyn_cast
<Function
>(TaskAllocCall
->getArgOperand(5));
6694 ASSERT_NE(OutlinedFn
, nullptr);
6695 ASSERT_EQ(OutlinedFn
->arg_size(), 1U);
6698 TEST_F(OpenMPIRBuilderTest
, CreateTaskUntied
) {
6699 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6700 OpenMPIRBuilder
OMPBuilder(*M
);
6701 OMPBuilder
.Config
.IsTargetDevice
= false;
6702 OMPBuilder
.initialize();
6704 IRBuilder
<> Builder(BB
);
6705 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6706 return Error::success();
6708 BasicBlock
*AllocaBB
= Builder
.GetInsertBlock();
6709 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6710 OpenMPIRBuilder::LocationDescription
Loc(
6711 InsertPointTy(BodyBB
, BodyBB
->getFirstInsertionPt()), DL
);
6712 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTask(
6713 Loc
, InsertPointTy(AllocaBB
, AllocaBB
->getFirstInsertionPt()), BodyGenCB
,
6715 assert(AfterIP
&& "unexpected error");
6716 Builder
.restoreIP(*AfterIP
);
6717 OMPBuilder
.finalize();
6718 Builder
.CreateRetVoid();
6720 // Check for the `Tied` argument
6721 CallInst
*TaskAllocCall
= dyn_cast
<CallInst
>(
6722 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc
)
6724 ASSERT_NE(TaskAllocCall
, nullptr);
6725 ConstantInt
*Flags
= dyn_cast
<ConstantInt
>(TaskAllocCall
->getArgOperand(2));
6726 ASSERT_NE(Flags
, nullptr);
6727 EXPECT_EQ(Flags
->getZExtValue() & 1U, 0U);
6729 EXPECT_FALSE(verifyModule(*M
, &errs()));
6732 TEST_F(OpenMPIRBuilderTest
, CreateTaskDepend
) {
6733 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6734 OpenMPIRBuilder
OMPBuilder(*M
);
6735 OMPBuilder
.Config
.IsTargetDevice
= false;
6736 OMPBuilder
.initialize();
6738 IRBuilder
<> Builder(BB
);
6739 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6740 return Error::success();
6742 BasicBlock
*AllocaBB
= Builder
.GetInsertBlock();
6743 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6744 OpenMPIRBuilder::LocationDescription
Loc(
6745 InsertPointTy(BodyBB
, BodyBB
->getFirstInsertionPt()), DL
);
6746 AllocaInst
*InDep
= Builder
.CreateAlloca(Type::getInt32Ty(M
->getContext()));
6747 SmallVector
<OpenMPIRBuilder::DependData
> DDS
;
6749 OpenMPIRBuilder::DependData
DDIn(RTLDependenceKindTy::DepIn
,
6750 Type::getInt32Ty(M
->getContext()), InDep
);
6751 DDS
.push_back(DDIn
);
6753 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTask(
6754 Loc
, InsertPointTy(AllocaBB
, AllocaBB
->getFirstInsertionPt()), BodyGenCB
,
6755 /*Tied=*/false, /*Final*/ nullptr, /*IfCondition*/ nullptr, DDS
);
6756 assert(AfterIP
&& "unexpected error");
6757 Builder
.restoreIP(*AfterIP
);
6758 OMPBuilder
.finalize();
6759 Builder
.CreateRetVoid();
6761 // Check for the `NumDeps` argument
6762 CallInst
*TaskAllocCall
= dyn_cast
<CallInst
>(
6764 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_with_deps
)
6766 ASSERT_NE(TaskAllocCall
, nullptr);
6767 ConstantInt
*NumDeps
= dyn_cast
<ConstantInt
>(TaskAllocCall
->getArgOperand(3));
6768 ASSERT_NE(NumDeps
, nullptr);
6769 EXPECT_EQ(NumDeps
->getZExtValue(), 1U);
6771 // Check for the `DepInfo` array argument
6772 AllocaInst
*DepArray
= dyn_cast
<AllocaInst
>(TaskAllocCall
->getOperand(4));
6773 ASSERT_NE(DepArray
, nullptr);
6774 Value::user_iterator DepArrayI
= DepArray
->user_begin();
6776 Value::user_iterator DepInfoI
= DepArrayI
->user_begin();
6777 // Check for the `DependKind` flag in the `DepInfo` array
6778 Value
*Flag
= findStoredValue
<GetElementPtrInst
>(*DepInfoI
);
6779 ASSERT_NE(Flag
, nullptr);
6780 ConstantInt
*FlagInt
= dyn_cast
<ConstantInt
>(Flag
);
6781 ASSERT_NE(FlagInt
, nullptr);
6782 EXPECT_EQ(FlagInt
->getZExtValue(),
6783 static_cast<unsigned int>(RTLDependenceKindTy::DepIn
));
6785 // Check for the size in the `DepInfo` array
6786 Value
*Size
= findStoredValue
<GetElementPtrInst
>(*DepInfoI
);
6787 ASSERT_NE(Size
, nullptr);
6788 ConstantInt
*SizeInt
= dyn_cast
<ConstantInt
>(Size
);
6789 ASSERT_NE(SizeInt
, nullptr);
6790 EXPECT_EQ(SizeInt
->getZExtValue(), 4U);
6792 // Check for the variable address in the `DepInfo` array
6793 Value
*AddrStored
= findStoredValue
<GetElementPtrInst
>(*DepInfoI
);
6794 ASSERT_NE(AddrStored
, nullptr);
6795 PtrToIntInst
*AddrInt
= dyn_cast
<PtrToIntInst
>(AddrStored
);
6796 ASSERT_NE(AddrInt
, nullptr);
6797 Value
*Addr
= AddrInt
->getPointerOperand();
6798 EXPECT_EQ(Addr
, InDep
);
6800 ConstantInt
*NumDepsNoAlias
=
6801 dyn_cast
<ConstantInt
>(TaskAllocCall
->getArgOperand(5));
6802 ASSERT_NE(NumDepsNoAlias
, nullptr);
6803 EXPECT_EQ(NumDepsNoAlias
->getZExtValue(), 0U);
6804 EXPECT_EQ(TaskAllocCall
->getOperand(6),
6805 ConstantPointerNull::get(PointerType::getUnqual(M
->getContext())));
6807 EXPECT_FALSE(verifyModule(*M
, &errs()));
6810 TEST_F(OpenMPIRBuilderTest
, CreateTaskFinal
) {
6811 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6812 OpenMPIRBuilder
OMPBuilder(*M
);
6813 OMPBuilder
.Config
.IsTargetDevice
= false;
6814 OMPBuilder
.initialize();
6816 IRBuilder
<> Builder(BB
);
6817 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6818 return Error::success();
6820 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6821 IRBuilderBase::InsertPoint AllocaIP
= Builder
.saveIP();
6822 Builder
.SetInsertPoint(BodyBB
);
6823 Value
*Final
= Builder
.CreateICmp(
6824 CmpInst::Predicate::ICMP_EQ
, F
->getArg(0),
6825 ConstantInt::get(Type::getInt32Ty(M
->getContext()), 0U));
6826 OpenMPIRBuilder::LocationDescription
Loc(Builder
.saveIP(), DL
);
6827 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
6828 OMPBuilder
.createTask(Loc
, AllocaIP
, BodyGenCB
,
6829 /*Tied=*/false, Final
);
6830 assert(AfterIP
&& "unexpected error");
6831 Builder
.restoreIP(*AfterIP
);
6832 OMPBuilder
.finalize();
6833 Builder
.CreateRetVoid();
6835 // Check for the `Tied` argument
6836 CallInst
*TaskAllocCall
= dyn_cast
<CallInst
>(
6837 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc
)
6839 ASSERT_NE(TaskAllocCall
, nullptr);
6840 BinaryOperator
*OrInst
=
6841 dyn_cast
<BinaryOperator
>(TaskAllocCall
->getArgOperand(2));
6842 ASSERT_NE(OrInst
, nullptr);
6843 EXPECT_EQ(OrInst
->getOpcode(), BinaryOperator::BinaryOps::Or
);
6845 // One of the arguments to `or` instruction is the tied flag, which is equal
6847 EXPECT_TRUE(any_of(OrInst
->operands(), [](Value
*op
) {
6848 if (ConstantInt
*TiedValue
= dyn_cast
<ConstantInt
>(op
))
6849 return TiedValue
->getSExtValue() == 0;
6853 // One of the arguments to `or` instruction is the final condition.
6854 EXPECT_TRUE(any_of(OrInst
->operands(), [Final
](Value
*op
) {
6855 if (SelectInst
*Select
= dyn_cast
<SelectInst
>(op
)) {
6856 ConstantInt
*TrueValue
= dyn_cast
<ConstantInt
>(Select
->getTrueValue());
6857 ConstantInt
*FalseValue
= dyn_cast
<ConstantInt
>(Select
->getFalseValue());
6858 if (!TrueValue
|| !FalseValue
)
6860 return Select
->getCondition() == Final
&&
6861 TrueValue
->getSExtValue() == 2 && FalseValue
->getSExtValue() == 0;
6866 EXPECT_FALSE(verifyModule(*M
, &errs()));
6869 TEST_F(OpenMPIRBuilderTest
, CreateTaskIfCondition
) {
6870 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6871 OpenMPIRBuilder
OMPBuilder(*M
);
6872 OMPBuilder
.Config
.IsTargetDevice
= false;
6873 OMPBuilder
.initialize();
6875 IRBuilder
<> Builder(BB
);
6876 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6877 return Error::success();
6879 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6880 IRBuilderBase::InsertPoint AllocaIP
= Builder
.saveIP();
6881 Builder
.SetInsertPoint(BodyBB
);
6882 Value
*IfCondition
= Builder
.CreateICmp(
6883 CmpInst::Predicate::ICMP_EQ
, F
->getArg(0),
6884 ConstantInt::get(Type::getInt32Ty(M
->getContext()), 0U));
6885 OpenMPIRBuilder::LocationDescription
Loc(Builder
.saveIP(), DL
);
6886 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
=
6887 OMPBuilder
.createTask(Loc
, AllocaIP
, BodyGenCB
,
6888 /*Tied=*/false, /*Final=*/nullptr, IfCondition
);
6889 assert(AfterIP
&& "unexpected error");
6890 Builder
.restoreIP(*AfterIP
);
6891 OMPBuilder
.finalize();
6892 Builder
.CreateRetVoid();
6894 EXPECT_FALSE(verifyModule(*M
, &errs()));
6896 CallInst
*TaskAllocCall
= dyn_cast
<CallInst
>(
6897 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc
)
6899 ASSERT_NE(TaskAllocCall
, nullptr);
6901 // Check the branching is based on the if condition argument.
6902 BranchInst
*IfConditionBranchInst
=
6903 dyn_cast
<BranchInst
>(TaskAllocCall
->getParent()->getTerminator());
6904 ASSERT_NE(IfConditionBranchInst
, nullptr);
6905 ASSERT_TRUE(IfConditionBranchInst
->isConditional());
6906 EXPECT_EQ(IfConditionBranchInst
->getCondition(), IfCondition
);
6908 // Check that the `__kmpc_omp_task` executes only in the then branch.
6909 CallInst
*TaskCall
= dyn_cast
<CallInst
>(
6910 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task
)
6912 ASSERT_NE(TaskCall
, nullptr);
6913 EXPECT_EQ(TaskCall
->getParent(), IfConditionBranchInst
->getSuccessor(0));
6915 // Check that the OpenMP Runtime Functions specific to `if` clause execute
6916 // only in the else branch. Also check that the function call is between the
6917 // `__kmpc_omp_task_begin_if0` and `__kmpc_omp_task_complete_if0` calls.
6918 CallInst
*TaskBeginIfCall
= dyn_cast
<CallInst
>(
6920 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_begin_if0
)
6922 CallInst
*TaskCompleteCall
= dyn_cast
<CallInst
>(
6924 .getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_complete_if0
)
6926 ASSERT_NE(TaskBeginIfCall
, nullptr);
6927 ASSERT_NE(TaskCompleteCall
, nullptr);
6928 Function
*OulinedFn
=
6929 dyn_cast
<Function
>(TaskAllocCall
->getArgOperand(5)->stripPointerCasts());
6930 ASSERT_NE(OulinedFn
, nullptr);
6931 CallInst
*OulinedFnCall
= dyn_cast
<CallInst
>(OulinedFn
->user_back());
6932 ASSERT_NE(OulinedFnCall
, nullptr);
6933 EXPECT_EQ(TaskBeginIfCall
->getParent(),
6934 IfConditionBranchInst
->getSuccessor(1));
6936 EXPECT_EQ(TaskBeginIfCall
->getNextNonDebugInstruction(), OulinedFnCall
);
6937 EXPECT_EQ(OulinedFnCall
->getNextNonDebugInstruction(), TaskCompleteCall
);
6940 TEST_F(OpenMPIRBuilderTest
, CreateTaskgroup
) {
6941 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
6942 OpenMPIRBuilder
OMPBuilder(*M
);
6943 OMPBuilder
.initialize();
6945 IRBuilder
<> Builder(BB
);
6947 AllocaInst
*ValPtr32
= Builder
.CreateAlloca(Builder
.getInt32Ty());
6948 AllocaInst
*ValPtr128
= Builder
.CreateAlloca(Builder
.getInt128Ty());
6950 Builder
.CreateLoad(Builder
.getInt128Ty(), ValPtr128
, "bodygen.load");
6951 Instruction
*ThenTerm
, *ElseTerm
;
6953 Value
*InternalStoreInst
, *InternalLoad32
, *InternalLoad128
, *InternalIfCmp
;
6955 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
6956 Builder
.restoreIP(AllocaIP
);
6957 AllocaInst
*Local128
= Builder
.CreateAlloca(Builder
.getInt128Ty(), nullptr,
6958 "bodygen.alloca128");
6960 Builder
.restoreIP(CodeGenIP
);
6961 // Loading and storing captured pointer and values
6962 InternalStoreInst
= Builder
.CreateStore(Val128
, Local128
);
6963 InternalLoad32
= Builder
.CreateLoad(ValPtr32
->getAllocatedType(), ValPtr32
,
6966 InternalLoad128
= Builder
.CreateLoad(Local128
->getAllocatedType(), Local128
,
6967 "bodygen.local.load128");
6968 InternalIfCmp
= Builder
.CreateICmpNE(
6970 Builder
.CreateTrunc(InternalLoad128
, InternalLoad32
->getType()));
6971 SplitBlockAndInsertIfThenElse(InternalIfCmp
,
6972 CodeGenIP
.getBlock()->getTerminator(),
6973 &ThenTerm
, &ElseTerm
);
6974 return Error::success();
6977 BasicBlock
*AllocaBB
= Builder
.GetInsertBlock();
6978 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
6979 OpenMPIRBuilder::LocationDescription
Loc(
6980 InsertPointTy(BodyBB
, BodyBB
->getFirstInsertionPt()), DL
);
6981 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTaskgroup(
6982 Loc
, InsertPointTy(AllocaBB
, AllocaBB
->getFirstInsertionPt()), BodyGenCB
);
6983 assert(AfterIP
&& "unexpected error");
6984 Builder
.restoreIP(*AfterIP
);
6985 OMPBuilder
.finalize();
6986 Builder
.CreateRetVoid();
6988 EXPECT_FALSE(verifyModule(*M
, &errs()));
6990 CallInst
*TaskgroupCall
= dyn_cast
<CallInst
>(
6991 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup
)
6993 ASSERT_NE(TaskgroupCall
, nullptr);
6994 CallInst
*EndTaskgroupCall
= dyn_cast
<CallInst
>(
6995 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup
)
6997 ASSERT_NE(EndTaskgroupCall
, nullptr);
6999 // Verify the Ident argument
7000 GlobalVariable
*Ident
= cast
<GlobalVariable
>(TaskgroupCall
->getArgOperand(0));
7001 ASSERT_NE(Ident
, nullptr);
7002 EXPECT_TRUE(Ident
->hasInitializer());
7003 Constant
*Initializer
= Ident
->getInitializer();
7004 GlobalVariable
*SrcStrGlob
=
7005 cast
<GlobalVariable
>(Initializer
->getOperand(4)->stripPointerCasts());
7006 ASSERT_NE(SrcStrGlob
, nullptr);
7007 ConstantDataArray
*SrcSrc
=
7008 dyn_cast
<ConstantDataArray
>(SrcStrGlob
->getInitializer());
7009 ASSERT_NE(SrcSrc
, nullptr);
7011 // Verify the num_threads argument.
7012 CallInst
*GTID
= dyn_cast
<CallInst
>(TaskgroupCall
->getArgOperand(1));
7013 ASSERT_NE(GTID
, nullptr);
7014 EXPECT_EQ(GTID
->arg_size(), 1U);
7015 EXPECT_EQ(GTID
->getCalledFunction(), OMPBuilder
.getOrCreateRuntimeFunctionPtr(
7016 OMPRTL___kmpc_global_thread_num
));
7018 // Checking the general structure of the IR generated is same as expected.
7019 Instruction
*GeneratedStoreInst
= TaskgroupCall
->getNextNonDebugInstruction();
7020 EXPECT_EQ(GeneratedStoreInst
, InternalStoreInst
);
7021 Instruction
*GeneratedLoad32
=
7022 GeneratedStoreInst
->getNextNonDebugInstruction();
7023 EXPECT_EQ(GeneratedLoad32
, InternalLoad32
);
7024 Instruction
*GeneratedLoad128
= GeneratedLoad32
->getNextNonDebugInstruction();
7025 EXPECT_EQ(GeneratedLoad128
, InternalLoad128
);
7027 // Checking the ordering because of the if statements and that
7028 // `__kmp_end_taskgroup` call is after the if branching.
7029 BasicBlock
*RefOrder
[] = {TaskgroupCall
->getParent(), ThenTerm
->getParent(),
7030 ThenTerm
->getSuccessor(0),
7031 EndTaskgroupCall
->getParent(),
7032 ElseTerm
->getParent()};
7033 verifyDFSOrder(F
, RefOrder
);
7036 TEST_F(OpenMPIRBuilderTest
, CreateTaskgroupWithTasks
) {
7037 using InsertPointTy
= OpenMPIRBuilder::InsertPointTy
;
7038 OpenMPIRBuilder
OMPBuilder(*M
);
7039 OMPBuilder
.Config
.IsTargetDevice
= false;
7040 OMPBuilder
.initialize();
7042 IRBuilder
<> Builder(BB
);
7044 auto BodyGenCB
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
7045 Builder
.restoreIP(AllocaIP
);
7046 AllocaInst
*Alloca32
=
7047 Builder
.CreateAlloca(Builder
.getInt32Ty(), nullptr, "bodygen.alloca32");
7048 AllocaInst
*Alloca64
=
7049 Builder
.CreateAlloca(Builder
.getInt64Ty(), nullptr, "bodygen.alloca64");
7050 Builder
.restoreIP(CodeGenIP
);
7051 auto TaskBodyGenCB1
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
7052 Builder
.restoreIP(CodeGenIP
);
7053 LoadInst
*LoadValue
=
7054 Builder
.CreateLoad(Alloca64
->getAllocatedType(), Alloca64
);
7055 Value
*AddInst
= Builder
.CreateAdd(LoadValue
, Builder
.getInt64(64));
7056 Builder
.CreateStore(AddInst
, Alloca64
);
7057 return Error::success();
7059 OpenMPIRBuilder::LocationDescription
Loc(Builder
.saveIP(), DL
);
7060 OpenMPIRBuilder::InsertPointOrErrorTy TaskIP1
=
7061 OMPBuilder
.createTask(Loc
, AllocaIP
, TaskBodyGenCB1
);
7062 assert(TaskIP1
&& "unexpected error");
7063 Builder
.restoreIP(*TaskIP1
);
7065 auto TaskBodyGenCB2
= [&](InsertPointTy AllocaIP
, InsertPointTy CodeGenIP
) {
7066 Builder
.restoreIP(CodeGenIP
);
7067 LoadInst
*LoadValue
=
7068 Builder
.CreateLoad(Alloca32
->getAllocatedType(), Alloca32
);
7069 Value
*AddInst
= Builder
.CreateAdd(LoadValue
, Builder
.getInt32(32));
7070 Builder
.CreateStore(AddInst
, Alloca32
);
7071 return Error::success();
7073 OpenMPIRBuilder::LocationDescription
Loc2(Builder
.saveIP(), DL
);
7074 OpenMPIRBuilder::InsertPointOrErrorTy TaskIP2
=
7075 OMPBuilder
.createTask(Loc2
, AllocaIP
, TaskBodyGenCB2
);
7076 assert(TaskIP2
&& "unexpected error");
7077 Builder
.restoreIP(*TaskIP2
);
7078 return Error::success();
7081 BasicBlock
*AllocaBB
= Builder
.GetInsertBlock();
7082 BasicBlock
*BodyBB
= splitBB(Builder
, /*CreateBranch=*/true, "alloca.split");
7083 OpenMPIRBuilder::LocationDescription
Loc(
7084 InsertPointTy(BodyBB
, BodyBB
->getFirstInsertionPt()), DL
);
7085 OpenMPIRBuilder::InsertPointOrErrorTy AfterIP
= OMPBuilder
.createTaskgroup(
7086 Loc
, InsertPointTy(AllocaBB
, AllocaBB
->getFirstInsertionPt()), BodyGenCB
);
7087 assert(AfterIP
&& "unexpected error");
7088 Builder
.restoreIP(*AfterIP
);
7089 OMPBuilder
.finalize();
7090 Builder
.CreateRetVoid();
7092 EXPECT_FALSE(verifyModule(*M
, &errs()));
7094 CallInst
*TaskgroupCall
= dyn_cast
<CallInst
>(
7095 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup
)
7097 ASSERT_NE(TaskgroupCall
, nullptr);
7098 CallInst
*EndTaskgroupCall
= dyn_cast
<CallInst
>(
7099 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup
)
7101 ASSERT_NE(EndTaskgroupCall
, nullptr);
7103 Function
*TaskAllocFn
=
7104 OMPBuilder
.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc
);
7105 ASSERT_EQ(TaskAllocFn
->getNumUses(), 2u);
7107 CallInst
*FirstTaskAllocCall
=
7108 dyn_cast_or_null
<CallInst
>(*TaskAllocFn
->users().begin());
7109 CallInst
*SecondTaskAllocCall
=
7110 dyn_cast_or_null
<CallInst
>(*TaskAllocFn
->users().begin()++);
7111 ASSERT_NE(FirstTaskAllocCall
, nullptr);
7112 ASSERT_NE(SecondTaskAllocCall
, nullptr);
7114 // Verify that the tasks have been generated in order and inside taskgroup
7116 BasicBlock
*RefOrder
[] = {
7117 TaskgroupCall
->getParent(), FirstTaskAllocCall
->getParent(),
7118 SecondTaskAllocCall
->getParent(), EndTaskgroupCall
->getParent()};
7119 verifyDFSOrder(F
, RefOrder
);
7122 TEST_F(OpenMPIRBuilderTest
, EmitOffloadingArraysArguments
) {
7123 OpenMPIRBuilder
OMPBuilder(*M
);
7124 OMPBuilder
.initialize();
7126 IRBuilder
<> Builder(BB
);
7128 OpenMPIRBuilder::TargetDataRTArgs RTArgs
;
7129 OpenMPIRBuilder::TargetDataInfo
Info(true, false);
7131 auto VoidPtrPtrTy
= PointerType::getUnqual(Builder
.getContext());
7132 auto Int64PtrTy
= PointerType::getUnqual(Builder
.getContext());
7134 Info
.RTArgs
.BasePointersArray
= ConstantPointerNull::get(Builder
.getPtrTy(0));
7135 Info
.RTArgs
.PointersArray
= ConstantPointerNull::get(Builder
.getPtrTy(0));
7136 Info
.RTArgs
.SizesArray
= ConstantPointerNull::get(Builder
.getPtrTy(0));
7137 Info
.RTArgs
.MapTypesArray
= ConstantPointerNull::get(Builder
.getPtrTy(0));
7138 Info
.RTArgs
.MapNamesArray
= ConstantPointerNull::get(Builder
.getPtrTy(0));
7139 Info
.RTArgs
.MappersArray
= ConstantPointerNull::get(Builder
.getPtrTy(0));
7140 Info
.NumberOfPtrs
= 4;
7141 Info
.EmitDebug
= false;
7142 OMPBuilder
.emitOffloadingArraysArgument(Builder
, RTArgs
, Info
, false);
7144 EXPECT_NE(RTArgs
.BasePointersArray
, nullptr);
7145 EXPECT_NE(RTArgs
.PointersArray
, nullptr);
7146 EXPECT_NE(RTArgs
.SizesArray
, nullptr);
7147 EXPECT_NE(RTArgs
.MapTypesArray
, nullptr);
7148 EXPECT_NE(RTArgs
.MappersArray
, nullptr);
7149 EXPECT_NE(RTArgs
.MapNamesArray
, nullptr);
7150 EXPECT_EQ(RTArgs
.MapTypesArrayEnd
, nullptr);
7152 EXPECT_EQ(RTArgs
.BasePointersArray
->getType(), VoidPtrPtrTy
);
7153 EXPECT_EQ(RTArgs
.PointersArray
->getType(), VoidPtrPtrTy
);
7154 EXPECT_EQ(RTArgs
.SizesArray
->getType(), Int64PtrTy
);
7155 EXPECT_EQ(RTArgs
.MapTypesArray
->getType(), Int64PtrTy
);
7156 EXPECT_EQ(RTArgs
.MappersArray
->getType(), VoidPtrPtrTy
);
7157 EXPECT_EQ(RTArgs
.MapNamesArray
->getType(), VoidPtrPtrTy
);
7160 TEST_F(OpenMPIRBuilderTest
, OffloadEntriesInfoManager
) {
7161 OpenMPIRBuilder
OMPBuilder(*M
);
7162 OMPBuilder
.setConfig(
7163 OpenMPIRBuilderConfig(true, false, false, false, false, false, false));
7164 OffloadEntriesInfoManager
&InfoManager
= OMPBuilder
.OffloadInfoManager
;
7165 TargetRegionEntryInfo
EntryInfo("parent", 1, 2, 4, 0);
7166 InfoManager
.initializeTargetRegionEntryInfo(EntryInfo
, 0);
7167 EXPECT_TRUE(InfoManager
.hasTargetRegionEntryInfo(EntryInfo
));
7168 InfoManager
.initializeDeviceGlobalVarEntryInfo(
7169 "gvar", OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo
, 0);
7170 InfoManager
.registerTargetRegionEntryInfo(
7171 EntryInfo
, nullptr, nullptr,
7172 OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion
);
7173 InfoManager
.registerDeviceGlobalVarEntryInfo(
7174 "gvar", 0x0, 8, OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo
,
7175 GlobalValue::WeakAnyLinkage
);
7176 EXPECT_TRUE(InfoManager
.hasDeviceGlobalVarEntryInfo("gvar"));
7179 // Tests both registerTargetGlobalVariable and getAddrOfDeclareTargetVar as they
7180 // call each other (recursively in some cases). The test case test these
7181 // functions by utilising them for host code generation for declare target
7183 TEST_F(OpenMPIRBuilderTest
, registerTargetGlobalVariable
) {
7184 OpenMPIRBuilder
OMPBuilder(*M
);
7185 OMPBuilder
.initialize();
7186 OpenMPIRBuilderConfig
Config(false, false, false, false, false, false, false);
7187 OMPBuilder
.setConfig(Config
);
7189 std::vector
<llvm::Triple
> TargetTriple
;
7190 TargetTriple
.emplace_back("amdgcn-amd-amdhsa");
7192 TargetRegionEntryInfo
EntryInfo("", 42, 4711, 17);
7193 std::vector
<GlobalVariable
*> RefsGathered
;
7195 std::vector
<Constant
*> Globals
;
7196 auto *IntTy
= Type::getInt32Ty(Ctx
);
7197 for (int I
= 0; I
< 2; ++I
) {
7198 Globals
.push_back(M
->getOrInsertGlobal(
7199 "test_data_int_" + std::to_string(I
), IntTy
, [&]() -> GlobalVariable
* {
7200 return new GlobalVariable(
7201 *M
, IntTy
, false, GlobalValue::LinkageTypes::WeakAnyLinkage
,
7202 ConstantInt::get(IntTy
, I
), "test_data_int_" + std::to_string(I
));
7206 OMPBuilder
.registerTargetGlobalVariable(
7207 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo
,
7208 OffloadEntriesInfoManager::OMPTargetDeviceClauseAny
, false, true,
7209 EntryInfo
, Globals
[0]->getName(), RefsGathered
, false, TargetTriple
,
7210 nullptr, nullptr, Globals
[0]->getType(), Globals
[0]);
7212 OMPBuilder
.registerTargetGlobalVariable(
7213 OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink
,
7214 OffloadEntriesInfoManager::OMPTargetDeviceClauseAny
, false, true,
7215 EntryInfo
, Globals
[1]->getName(), RefsGathered
, false, TargetTriple
,
7216 nullptr, nullptr, Globals
[1]->getType(), Globals
[1]);
7218 llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy
&&ErrorReportfn
=
7219 [](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind
,
7220 const llvm::TargetRegionEntryInfo
&EntryInfo
) -> void {
7221 // If this is invoked, then we want to emit an error, even if it is not
7222 // neccesarily the most readable, as something has went wrong. The
7223 // test-suite unfortunately eats up all error output
7224 ASSERT_EQ(Kind
, Kind
);
7227 OMPBuilder
.createOffloadEntriesAndInfoMetadata(ErrorReportfn
);
7229 // Clauses for data_int_0 with To + Any clauses for the host
7230 std::vector
<GlobalVariable
*> OffloadEntries
;
7231 OffloadEntries
.push_back(M
->getNamedGlobal(".offloading.entry_name"));
7232 OffloadEntries
.push_back(
7233 M
->getNamedGlobal(".offloading.entry.test_data_int_0"));
7235 // Clauses for data_int_1 with Link + Any clauses for the host
7236 OffloadEntries
.push_back(
7237 M
->getNamedGlobal("test_data_int_1_decl_tgt_ref_ptr"));
7238 OffloadEntries
.push_back(M
->getNamedGlobal(".offloading.entry_name.1"));
7239 OffloadEntries
.push_back(
7240 M
->getNamedGlobal(".offloading.entry.test_data_int_1_decl_tgt_ref_ptr"));
7242 for (unsigned I
= 0; I
< OffloadEntries
.size(); ++I
)
7243 EXPECT_NE(OffloadEntries
[I
], nullptr);
7245 // Metadata generated for the host offload module
7246 NamedMDNode
*OffloadMetadata
= M
->getNamedMetadata("omp_offload.info");
7247 ASSERT_THAT(OffloadMetadata
, testing::NotNull());
7248 StringRef Nodes
[2] = {
7249 cast
<MDString
>(OffloadMetadata
->getOperand(0)->getOperand(1))
7251 cast
<MDString
>(OffloadMetadata
->getOperand(1)->getOperand(1))
7254 Nodes
, testing::UnorderedElementsAre("test_data_int_0",
7255 "test_data_int_1_decl_tgt_ref_ptr"));
7258 TEST_F(OpenMPIRBuilderTest
, createGPUOffloadEntry
) {
7259 OpenMPIRBuilder
OMPBuilder(*M
);
7260 OMPBuilder
.initialize();
7261 OpenMPIRBuilderConfig
Config(/* IsTargetDevice = */ true,
7263 /* OpenMPOffloadMandatory = */ false,
7264 /* HasRequiresReverseOffload = */ false,
7265 /* HasRequiresUnifiedAddress = */ false,
7266 /* HasRequiresUnifiedSharedMemory = */ false,
7267 /* HasRequiresDynamicAllocators = */ false);
7268 OMPBuilder
.setConfig(Config
);
7270 FunctionCallee FnTypeAndCallee
=
7271 M
->getOrInsertFunction("test_kernel", Type::getVoidTy(Ctx
));
7273 auto *Fn
= cast
<Function
>(FnTypeAndCallee
.getCallee());
7274 OMPBuilder
.createOffloadEntry(/* ID = */ nullptr, Fn
,
7276 /* Flags = */ 0, GlobalValue::WeakAnyLinkage
);
7278 // Check nvvm.annotations only created for GPU kernels
7279 NamedMDNode
*MD
= M
->getNamedMetadata("nvvm.annotations");
7280 EXPECT_NE(MD
, nullptr);
7281 EXPECT_EQ(MD
->getNumOperands(), 1u);
7283 MDNode
*Annotations
= MD
->getOperand(0);
7284 EXPECT_EQ(Annotations
->getNumOperands(), 3u);
7286 Constant
*ConstVal
=
7287 dyn_cast
<ConstantAsMetadata
>(Annotations
->getOperand(0))->getValue();
7288 EXPECT_TRUE(isa
<Function
>(Fn
));
7289 EXPECT_EQ(ConstVal
, cast
<Function
>(Fn
));
7291 EXPECT_TRUE(Annotations
->getOperand(1).equalsStr("kernel"));
7293 EXPECT_TRUE(mdconst::hasa
<ConstantInt
>(Annotations
->getOperand(2)));
7295 mdconst::extract
<ConstantInt
>(Annotations
->getOperand(2))->getValue();
7296 EXPECT_EQ(IntVal
, 1);
7298 // Check kernel attributes
7299 EXPECT_TRUE(Fn
->hasFnAttribute("kernel"));
7300 EXPECT_TRUE(Fn
->hasFnAttribute(Attribute::MustProgress
));