1 //===---- Delinearization.cpp - MultiDimensional Index Delinearization ----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This implements an analysis pass that tries to delinearize all GEP
10 // instructions in all loops using the SCEV analysis functionality. This pass is
11 // only used for testing purposes: if your pass needs delinearization, please
12 // use the on-demand SCEVAddRecExpr::delinearize() function.
14 //===----------------------------------------------------------------------===//
16 #include "llvm/Analysis/Delinearization.h"
17 #include "llvm/Analysis/LoopInfo.h"
18 #include "llvm/Analysis/ScalarEvolution.h"
19 #include "llvm/Analysis/ScalarEvolutionDivision.h"
20 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
21 #include "llvm/IR/Constants.h"
22 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/IR/Function.h"
24 #include "llvm/IR/InstIterator.h"
25 #include "llvm/IR/Instructions.h"
26 #include "llvm/IR/PassManager.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Support/raw_ostream.h"
32 #define DL_NAME "delinearize"
33 #define DEBUG_TYPE DL_NAME
35 // Return true when S contains at least an undef value.
36 static inline bool containsUndefs(const SCEV
*S
) {
37 return SCEVExprContains(S
, [](const SCEV
*S
) {
38 if (const auto *SU
= dyn_cast
<SCEVUnknown
>(S
))
39 return isa
<UndefValue
>(SU
->getValue());
46 // Collect all steps of SCEV expressions.
47 struct SCEVCollectStrides
{
49 SmallVectorImpl
<const SCEV
*> &Strides
;
51 SCEVCollectStrides(ScalarEvolution
&SE
, SmallVectorImpl
<const SCEV
*> &S
)
52 : SE(SE
), Strides(S
) {}
54 bool follow(const SCEV
*S
) {
55 if (const SCEVAddRecExpr
*AR
= dyn_cast
<SCEVAddRecExpr
>(S
))
56 Strides
.push_back(AR
->getStepRecurrence(SE
));
60 bool isDone() const { return false; }
63 // Collect all SCEVUnknown and SCEVMulExpr expressions.
64 struct SCEVCollectTerms
{
65 SmallVectorImpl
<const SCEV
*> &Terms
;
67 SCEVCollectTerms(SmallVectorImpl
<const SCEV
*> &T
) : Terms(T
) {}
69 bool follow(const SCEV
*S
) {
70 if (isa
<SCEVUnknown
>(S
) || isa
<SCEVMulExpr
>(S
) ||
71 isa
<SCEVSignExtendExpr
>(S
)) {
72 if (!containsUndefs(S
))
75 // Stop recursion: once we collected a term, do not walk its operands.
83 bool isDone() const { return false; }
86 // Check if a SCEV contains an AddRecExpr.
87 struct SCEVHasAddRec
{
90 SCEVHasAddRec(bool &ContainsAddRec
) : ContainsAddRec(ContainsAddRec
) {
91 ContainsAddRec
= false;
94 bool follow(const SCEV
*S
) {
95 if (isa
<SCEVAddRecExpr
>(S
)) {
96 ContainsAddRec
= true;
98 // Stop recursion: once we collected a term, do not walk its operands.
106 bool isDone() const { return false; }
109 // Find factors that are multiplied with an expression that (possibly as a
110 // subexpression) contains an AddRecExpr. In the expression:
112 // 8 * (100 + %p * %q * (%a + {0, +, 1}_loop))
114 // "%p * %q" are factors multiplied by the expression "(%a + {0, +, 1}_loop)"
115 // that contains the AddRec {0, +, 1}_loop. %p * %q are likely to be array size
116 // parameters as they form a product with an induction variable.
118 // This collector expects all array size parameters to be in the same MulExpr.
119 // It might be necessary to later add support for collecting parameters that are
120 // spread over different nested MulExpr.
121 struct SCEVCollectAddRecMultiplies
{
122 SmallVectorImpl
<const SCEV
*> &Terms
;
125 SCEVCollectAddRecMultiplies(SmallVectorImpl
<const SCEV
*> &T
,
127 : Terms(T
), SE(SE
) {}
129 bool follow(const SCEV
*S
) {
130 if (auto *Mul
= dyn_cast
<SCEVMulExpr
>(S
)) {
131 bool HasAddRec
= false;
132 SmallVector
<const SCEV
*, 0> Operands
;
133 for (const SCEV
*Op
: Mul
->operands()) {
134 const SCEVUnknown
*Unknown
= dyn_cast
<SCEVUnknown
>(Op
);
135 if (Unknown
&& !isa
<CallInst
>(Unknown
->getValue())) {
136 Operands
.push_back(Op
);
137 } else if (Unknown
) {
140 bool ContainsAddRec
= false;
141 SCEVHasAddRec
ContiansAddRec(ContainsAddRec
);
142 visitAll(Op
, ContiansAddRec
);
143 HasAddRec
|= ContainsAddRec
;
146 if (Operands
.size() == 0)
152 Terms
.push_back(SE
.getMulExpr(Operands
));
153 // Stop recursion: once we collected a term, do not walk its operands.
161 bool isDone() const { return false; }
164 } // end anonymous namespace
166 /// Find parametric terms in this SCEVAddRecExpr. We first for parameters in
168 /// 1) The strides of AddRec expressions.
169 /// 2) Unknowns that are multiplied with AddRec expressions.
170 void llvm::collectParametricTerms(ScalarEvolution
&SE
, const SCEV
*Expr
,
171 SmallVectorImpl
<const SCEV
*> &Terms
) {
172 SmallVector
<const SCEV
*, 4> Strides
;
173 SCEVCollectStrides
StrideCollector(SE
, Strides
);
174 visitAll(Expr
, StrideCollector
);
177 dbgs() << "Strides:\n";
178 for (const SCEV
*S
: Strides
)
179 dbgs() << *S
<< "\n";
182 for (const SCEV
*S
: Strides
) {
183 SCEVCollectTerms
TermCollector(Terms
);
184 visitAll(S
, TermCollector
);
188 dbgs() << "Terms:\n";
189 for (const SCEV
*T
: Terms
)
190 dbgs() << *T
<< "\n";
193 SCEVCollectAddRecMultiplies
MulCollector(Terms
, SE
);
194 visitAll(Expr
, MulCollector
);
197 static bool findArrayDimensionsRec(ScalarEvolution
&SE
,
198 SmallVectorImpl
<const SCEV
*> &Terms
,
199 SmallVectorImpl
<const SCEV
*> &Sizes
) {
200 int Last
= Terms
.size() - 1;
201 const SCEV
*Step
= Terms
[Last
];
205 if (const SCEVMulExpr
*M
= dyn_cast
<SCEVMulExpr
>(Step
)) {
206 SmallVector
<const SCEV
*, 2> Qs
;
207 for (const SCEV
*Op
: M
->operands())
208 if (!isa
<SCEVConstant
>(Op
))
211 Step
= SE
.getMulExpr(Qs
);
214 Sizes
.push_back(Step
);
218 for (const SCEV
*&Term
: Terms
) {
219 // Normalize the terms before the next call to findArrayDimensionsRec.
221 SCEVDivision::divide(SE
, Term
, Step
, &Q
, &R
);
223 // Bail out when GCD does not evenly divide one of the terms.
230 // Remove all SCEVConstants.
231 erase_if(Terms
, [](const SCEV
*E
) { return isa
<SCEVConstant
>(E
); });
233 if (Terms
.size() > 0)
234 if (!findArrayDimensionsRec(SE
, Terms
, Sizes
))
237 Sizes
.push_back(Step
);
241 // Returns true when one of the SCEVs of Terms contains a SCEVUnknown parameter.
242 static inline bool containsParameters(SmallVectorImpl
<const SCEV
*> &Terms
) {
243 for (const SCEV
*T
: Terms
)
244 if (SCEVExprContains(T
, [](const SCEV
*S
) { return isa
<SCEVUnknown
>(S
); }))
250 // Return the number of product terms in S.
251 static inline int numberOfTerms(const SCEV
*S
) {
252 if (const SCEVMulExpr
*Expr
= dyn_cast
<SCEVMulExpr
>(S
))
253 return Expr
->getNumOperands();
257 static const SCEV
*removeConstantFactors(ScalarEvolution
&SE
, const SCEV
*T
) {
258 if (isa
<SCEVConstant
>(T
))
261 if (isa
<SCEVUnknown
>(T
))
264 if (const SCEVMulExpr
*M
= dyn_cast
<SCEVMulExpr
>(T
)) {
265 SmallVector
<const SCEV
*, 2> Factors
;
266 for (const SCEV
*Op
: M
->operands())
267 if (!isa
<SCEVConstant
>(Op
))
268 Factors
.push_back(Op
);
270 return SE
.getMulExpr(Factors
);
276 void llvm::findArrayDimensions(ScalarEvolution
&SE
,
277 SmallVectorImpl
<const SCEV
*> &Terms
,
278 SmallVectorImpl
<const SCEV
*> &Sizes
,
279 const SCEV
*ElementSize
) {
280 if (Terms
.size() < 1 || !ElementSize
)
283 // Early return when Terms do not contain parameters: we do not delinearize
284 // non parametric SCEVs.
285 if (!containsParameters(Terms
))
289 dbgs() << "Terms:\n";
290 for (const SCEV
*T
: Terms
)
291 dbgs() << *T
<< "\n";
294 // Remove duplicates.
295 array_pod_sort(Terms
.begin(), Terms
.end());
296 Terms
.erase(llvm::unique(Terms
), Terms
.end());
298 // Put larger terms first.
299 llvm::sort(Terms
, [](const SCEV
*LHS
, const SCEV
*RHS
) {
300 return numberOfTerms(LHS
) > numberOfTerms(RHS
);
303 // Try to divide all terms by the element size. If term is not divisible by
304 // element size, proceed with the original term.
305 for (const SCEV
*&Term
: Terms
) {
307 SCEVDivision::divide(SE
, Term
, ElementSize
, &Q
, &R
);
312 SmallVector
<const SCEV
*, 4> NewTerms
;
314 // Remove constant factors.
315 for (const SCEV
*T
: Terms
)
316 if (const SCEV
*NewT
= removeConstantFactors(SE
, T
))
317 NewTerms
.push_back(NewT
);
320 dbgs() << "Terms after sorting:\n";
321 for (const SCEV
*T
: NewTerms
)
322 dbgs() << *T
<< "\n";
325 if (NewTerms
.empty() || !findArrayDimensionsRec(SE
, NewTerms
, Sizes
)) {
330 // The last element to be pushed into Sizes is the size of an element.
331 Sizes
.push_back(ElementSize
);
334 dbgs() << "Sizes:\n";
335 for (const SCEV
*S
: Sizes
)
336 dbgs() << *S
<< "\n";
340 void llvm::computeAccessFunctions(ScalarEvolution
&SE
, const SCEV
*Expr
,
341 SmallVectorImpl
<const SCEV
*> &Subscripts
,
342 SmallVectorImpl
<const SCEV
*> &Sizes
) {
343 // Early exit in case this SCEV is not an affine multivariate function.
347 if (auto *AR
= dyn_cast
<SCEVAddRecExpr
>(Expr
))
351 const SCEV
*Res
= Expr
;
352 int Last
= Sizes
.size() - 1;
353 for (int i
= Last
; i
>= 0; i
--) {
355 SCEVDivision::divide(SE
, Res
, Sizes
[i
], &Q
, &R
);
358 dbgs() << "Res: " << *Res
<< "\n";
359 dbgs() << "Sizes[i]: " << *Sizes
[i
] << "\n";
360 dbgs() << "Res divided by Sizes[i]:\n";
361 dbgs() << "Quotient: " << *Q
<< "\n";
362 dbgs() << "Remainder: " << *R
<< "\n";
367 // Do not record the last subscript corresponding to the size of elements in
371 // Bail out if the byte offset is non-zero.
381 // Record the access function for the current subscript.
382 Subscripts
.push_back(R
);
385 // Also push in last position the remainder of the last division: it will be
386 // the access function of the innermost dimension.
387 Subscripts
.push_back(Res
);
389 std::reverse(Subscripts
.begin(), Subscripts
.end());
392 dbgs() << "Subscripts:\n";
393 for (const SCEV
*S
: Subscripts
)
394 dbgs() << *S
<< "\n";
398 /// Splits the SCEV into two vectors of SCEVs representing the subscripts and
399 /// sizes of an array access. Returns the remainder of the delinearization that
400 /// is the offset start of the array. The SCEV->delinearize algorithm computes
401 /// the multiples of SCEV coefficients: that is a pattern matching of sub
402 /// expressions in the stride and base of a SCEV corresponding to the
403 /// computation of a GCD (greatest common divisor) of base and stride. When
404 /// SCEV->delinearize fails, it returns the SCEV unchanged.
406 /// For example: when analyzing the memory access A[i][j][k] in this loop nest
408 /// void foo(long n, long m, long o, double A[n][m][o]) {
410 /// for (long i = 0; i < n; i++)
411 /// for (long j = 0; j < m; j++)
412 /// for (long k = 0; k < o; k++)
413 /// A[i][j][k] = 1.0;
416 /// the delinearization input is the following AddRec SCEV:
418 /// AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
420 /// From this SCEV, we are able to say that the base offset of the access is %A
421 /// because it appears as an offset that does not divide any of the strides in
424 /// CHECK: Base offset: %A
426 /// and then SCEV->delinearize determines the size of some of the dimensions of
427 /// the array as these are the multiples by which the strides are happening:
429 /// CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of sizeof(double)
432 /// Note that the outermost dimension remains of UnknownSize because there are
433 /// no strides that would help identifying the size of the last dimension: when
434 /// the array has been statically allocated, one could compute the size of that
435 /// dimension by dividing the overall size of the array by the size of the known
436 /// dimensions: %m * %o * 8.
438 /// Finally delinearize provides the access functions for the array reference
439 /// that does correspond to A[i][j][k] of the above C testcase:
441 /// CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>]
443 /// The testcases are checking the output of a function pass:
444 /// DelinearizationPass that walks through all loads and stores of a function
445 /// asking for the SCEV of the memory access with respect to all enclosing
446 /// loops, calling SCEV->delinearize on that and printing the results.
447 void llvm::delinearize(ScalarEvolution
&SE
, const SCEV
*Expr
,
448 SmallVectorImpl
<const SCEV
*> &Subscripts
,
449 SmallVectorImpl
<const SCEV
*> &Sizes
,
450 const SCEV
*ElementSize
) {
451 // First step: collect parametric terms.
452 SmallVector
<const SCEV
*, 4> Terms
;
453 collectParametricTerms(SE
, Expr
, Terms
);
458 // Second step: find subscript sizes.
459 findArrayDimensions(SE
, Terms
, Sizes
, ElementSize
);
464 // Third step: compute the access functions for each subscript.
465 computeAccessFunctions(SE
, Expr
, Subscripts
, Sizes
);
467 if (Subscripts
.empty())
471 dbgs() << "succeeded to delinearize " << *Expr
<< "\n";
472 dbgs() << "ArrayDecl[UnknownSize]";
473 for (const SCEV
*S
: Sizes
)
474 dbgs() << "[" << *S
<< "]";
476 dbgs() << "\nArrayRef";
477 for (const SCEV
*S
: Subscripts
)
478 dbgs() << "[" << *S
<< "]";
483 bool llvm::getIndexExpressionsFromGEP(ScalarEvolution
&SE
,
484 const GetElementPtrInst
*GEP
,
485 SmallVectorImpl
<const SCEV
*> &Subscripts
,
486 SmallVectorImpl
<int> &Sizes
) {
487 assert(Subscripts
.empty() && Sizes
.empty() &&
488 "Expected output lists to be empty on entry to this function.");
489 assert(GEP
&& "getIndexExpressionsFromGEP called with a null GEP");
491 bool DroppedFirstDim
= false;
492 for (unsigned i
= 1; i
< GEP
->getNumOperands(); i
++) {
493 const SCEV
*Expr
= SE
.getSCEV(GEP
->getOperand(i
));
495 Ty
= GEP
->getSourceElementType();
496 if (auto *Const
= dyn_cast
<SCEVConstant
>(Expr
))
497 if (Const
->getValue()->isZero()) {
498 DroppedFirstDim
= true;
501 Subscripts
.push_back(Expr
);
505 auto *ArrayTy
= dyn_cast
<ArrayType
>(Ty
);
512 Subscripts
.push_back(Expr
);
513 if (!(DroppedFirstDim
&& i
== 2))
514 Sizes
.push_back(ArrayTy
->getNumElements());
516 Ty
= ArrayTy
->getElementType();
518 return !Subscripts
.empty();
521 bool llvm::tryDelinearizeFixedSizeImpl(
522 ScalarEvolution
*SE
, Instruction
*Inst
, const SCEV
*AccessFn
,
523 SmallVectorImpl
<const SCEV
*> &Subscripts
, SmallVectorImpl
<int> &Sizes
) {
524 Value
*SrcPtr
= getLoadStorePointerOperand(Inst
);
526 // Check the simple case where the array dimensions are fixed size.
527 auto *SrcGEP
= dyn_cast
<GetElementPtrInst
>(SrcPtr
);
531 getIndexExpressionsFromGEP(*SE
, SrcGEP
, Subscripts
, Sizes
);
533 // Check that the two size arrays are non-empty and equal in length and
535 // TODO: it would be better to let the caller to clear Subscripts, similar
536 // to how we handle Sizes.
537 if (Sizes
.empty() || Subscripts
.size() <= 1) {
542 // Check that for identical base pointers we do not miss index offsets
543 // that have been added before this GEP is applied.
544 Value
*SrcBasePtr
= SrcGEP
->getOperand(0)->stripPointerCasts();
545 const SCEVUnknown
*SrcBase
=
546 dyn_cast
<SCEVUnknown
>(SE
->getPointerBase(AccessFn
));
547 if (!SrcBase
|| SrcBasePtr
!= SrcBase
->getValue()) {
552 assert(Subscripts
.size() == Sizes
.size() + 1 &&
553 "Expected equal number of entries in the list of size and "
561 void printDelinearization(raw_ostream
&O
, Function
*F
, LoopInfo
*LI
,
562 ScalarEvolution
*SE
) {
563 O
<< "Delinearization on function " << F
->getName() << ":\n";
564 for (Instruction
&Inst
: instructions(F
)) {
565 // Only analyze loads and stores.
566 if (!isa
<StoreInst
>(&Inst
) && !isa
<LoadInst
>(&Inst
) &&
567 !isa
<GetElementPtrInst
>(&Inst
))
570 const BasicBlock
*BB
= Inst
.getParent();
571 // Delinearize the memory access as analyzed in all the surrounding loops.
572 // Do not analyze memory accesses outside loops.
573 for (Loop
*L
= LI
->getLoopFor(BB
); L
!= nullptr; L
= L
->getParentLoop()) {
574 const SCEV
*AccessFn
= SE
->getSCEVAtScope(getPointerOperand(&Inst
), L
);
576 const SCEVUnknown
*BasePointer
=
577 dyn_cast
<SCEVUnknown
>(SE
->getPointerBase(AccessFn
));
578 // Do not delinearize if we cannot find the base pointer.
581 AccessFn
= SE
->getMinusSCEV(AccessFn
, BasePointer
);
584 O
<< "Inst:" << Inst
<< "\n";
585 O
<< "In Loop with Header: " << L
->getHeader()->getName() << "\n";
586 O
<< "AccessFunction: " << *AccessFn
<< "\n";
588 SmallVector
<const SCEV
*, 3> Subscripts
, Sizes
;
589 delinearize(*SE
, AccessFn
, Subscripts
, Sizes
, SE
->getElementSize(&Inst
));
590 if (Subscripts
.size() == 0 || Sizes
.size() == 0 ||
591 Subscripts
.size() != Sizes
.size()) {
592 O
<< "failed to delinearize\n";
596 O
<< "Base offset: " << *BasePointer
<< "\n";
597 O
<< "ArrayDecl[UnknownSize]";
598 int Size
= Subscripts
.size();
599 for (int i
= 0; i
< Size
- 1; i
++)
600 O
<< "[" << *Sizes
[i
] << "]";
601 O
<< " with elements of " << *Sizes
[Size
- 1] << " bytes.\n";
604 for (int i
= 0; i
< Size
; i
++)
605 O
<< "[" << *Subscripts
[i
] << "]";
611 } // end anonymous namespace
613 DelinearizationPrinterPass::DelinearizationPrinterPass(raw_ostream
&OS
)
615 PreservedAnalyses
DelinearizationPrinterPass::run(Function
&F
,
616 FunctionAnalysisManager
&AM
) {
617 printDelinearization(OS
, &F
, &AM
.getResult
<LoopAnalysis
>(F
),
618 &AM
.getResult
<ScalarEvolutionAnalysis
>(F
));
619 return PreservedAnalyses::all();