| blob | ad2c195ba1e8ef5b52076d7719965bf61a485eec |
1 ; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true \
2 ; RUN: -polly-target-throughput-vector-fma=1 \
3 ; RUN: -polly-target-latency-vector-fma=8 \
4 ; RUN: -polly-target-1st-cache-level-associativity=8 \
5 ; RUN: -polly-target-2nd-cache-level-associativity=8 \
6 ; RUN: -polly-target-1st-cache-level-size=32768 \
7 ; RUN: -polly-target-vector-register-bitwidth=256 \
8 ; RUN: -polly-target-2nd-cache-level-size=262144 \
9 ; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s
10 ;
11 ; opt %loadNPMPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \
12 ; -polly-target-throughput-vector-fma=1 \
13 ; -polly-target-latency-vector-fma=8 \
14 ; -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \
15 ; -polly-target-2nd-cache-level-associativity=8 \
16 ; -polly-target-1st-cache-level-size=32768 \
17 ; -polly-target-vector-register-bitwidth=256 \
18 ; -polly-target-2nd-cache-level-size=262144 -gvn -licm -slp-vectorizer \
19 ; -mcpu=corei7 -stats -S < %s 2>&1 | FileCheck %s --check-prefix=AUTO-VECTORIZATION
20 ;
21 ;
22 ; /* We isolate a set of partial tile prefixes, which contains only partial
23 ; tile prefixes that have exactly Mr x Nr iterations of the two innermost
24 ; loops produced by the optimization of the matrix multiplication. Mr and
25 ; Nr are parameters of the micro-kernel (see getMicroKernelParams and
26 ; getMacroKernelParams from lib/Transform/ScheduleOptimizer.cpp for
27 ; details). This test check that in case of parametric bounds it helps to
28 ; get rid of the conditional expressions of the unrolled innermost loops,
29 ; which prevents stores and loads of the unrolled loops from being sunk
30 ; and hoisted. Otherwise, it causes a run-time regression in comparison
31 ; to the vectorized code with sunk and hoisted memory accesses. */
32 ;
33 ; /* C := A * B + C */
34 ; for (i = 0; i < _PB_NI; i++)
35 ; for (j = 0; j < _PB_NJ; j++)
36 ; for (k = 0; k < _PB_NK; ++k)
37 ; C[i][j] += A[i][k] * B[k][j];
38 ;
39 ; CHECK: if (ni >= 1) {
40 ; CHECK-NEXT: // 1st level tiling - Tiles
41 ; CHECK-NEXT: for (int c0 = 0; c0 <= floord(nj - 1, 2048); c0 += 1)
42 ; CHECK-NEXT: for (int c1 = 0; c1 <= floord(nk - 1, 256); c1 += 1) {
43 ; CHECK-NEXT: for (int c3 = 2048 * c0; c3 <= min(nj - 1, 2048 * c0 + 2047); c3 += 1)
44 ; CHECK-NEXT: for (int c4 = 256 * c1; c4 <= min(nk - 1, 256 * c1 + 255); c4 += 1)
45 ; CHECK-NEXT: CopyStmt_0(0, c3, c4);
46 ; CHECK-NEXT: for (int c2 = 0; c2 <= floord(ni - 1, 96); c2 += 1) {
47 ; CHECK-NEXT: for (int c6 = 96 * c2; c6 <= min(ni - 1, 96 * c2 + 95); c6 += 1)
48 ; CHECK-NEXT: for (int c7 = 256 * c1; c7 <= min(nk - 1, 256 * c1 + 255); c7 += 1)
49 ; CHECK-NEXT: CopyStmt_1(c0, c1, c2, c6, c7);
50 ; CHECK-NEXT: // 1st level tiling - Points
51 ; CHECK-NEXT: // Register tiling - Tiles
52 ; CHECK-NEXT: {
53 ; CHECK-NEXT: if (ni >= 96 * c2 + 4)
54 ; CHECK-NEXT: for (int c3 = 0; c3 <= min(255, -256 * c0 + nj / 8 - 1); c3 += 1) {
55 ; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + ni / 4 - 1); c4 += 1)
56 ; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {
57 ; CHECK-NEXT: // Loop Vectorizer Disabled
58 ; CHECK-NEXT: // Register tiling - Points
59 ; CHECK-NEXT: {
60 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3, 256 * c1 + c5);
61 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
62 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
63 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
64 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
65 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
66 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
67 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
68 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);
69 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
70 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
71 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
72 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
73 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
74 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
75 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
76 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3, 256 * c1 + c5);
77 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
78 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
79 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
80 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
81 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
82 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
83 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
84 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3, 256 * c1 + c5);
85 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
86 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
87 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
88 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
89 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
90 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
91 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
92 ; CHECK-NEXT: }
93 ; CHECK-NEXT: }
94 ; CHECK-NEXT: if ((ni >= 96 * c2 + 5 && 96 * c2 + 7 >= ni && c3 >= 0) || (ni >= 96 * c2 + 8 && 96 * c2 + 95 >= ni && ni % 4 >= 1)) {
95 ; CHECK-NEXT: if (96 * c2 + 7 >= ni) {
96 ; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {
97 ; CHECK-NEXT: // Loop Vectorizer Disabled
98 ; CHECK-NEXT: // Register tiling - Points
99 ; CHECK-NEXT: {
100 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3, 256 * c1 + c5);
101 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
102 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
103 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
104 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
105 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
106 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
107 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
108 ; CHECK-NEXT: if (ni >= 96 * c2 + 6) {
109 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3, 256 * c1 + c5);
110 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
111 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
112 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
113 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
114 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
115 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
116 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
117 ; CHECK-NEXT: if (96 * c2 + 7 == ni) {
118 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);
119 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
120 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
121 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
122 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
123 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
124 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
125 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
126 ; CHECK-NEXT: }
127 ; CHECK-NEXT: }
128 ; CHECK-NEXT: }
129 ; CHECK-NEXT: }
130 ; CHECK-NEXT: } else {
131 ; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {
132 ; CHECK-NEXT: // Loop Vectorizer Disabled
133 ; CHECK-NEXT: // Register tiling - Points
134 ; CHECK-NEXT: {
135 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);
136 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
137 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
138 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
139 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
140 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
141 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
142 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
143 ; CHECK-NEXT: if (ni % 4 >= 2) {
144 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3, 256 * c1 + c5);
145 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
146 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
147 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
148 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
149 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
150 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
151 ; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
152 ; CHECK-NEXT: if ((ni + 1) % 4 == 0) {
153 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);
154 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
155 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
156 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
157 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
158 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
159 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
160 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
161 ; CHECK-NEXT: }
162 ; CHECK-NEXT: }
163 ; CHECK-NEXT: }
164 ; CHECK-NEXT: }
165 ; CHECK-NEXT: }
166 ; CHECK-NEXT: }
167 ; CHECK-NEXT: }
168 ; CHECK-NEXT: if (96 * c2 + 3 >= ni || 2048 * c0 + 7 >= nj || (2048 * c0 + 2047 >= nj && nj % 8 >= 1)) {
169 ; CHECK-NEXT: if (2048 * c0 + 7 >= nj) {
170 ; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + (ni - 1) / 4); c4 += 1)
171 ; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {
172 ; CHECK-NEXT: // Loop Vectorizer Disabled
173 ; CHECK-NEXT: // Register tiling - Points
174 ; CHECK-NEXT: {
175 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0, 256 * c1 + c5);
176 ; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {
177 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 1, 256 * c1 + c5);
178 ; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {
179 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 2, 256 * c1 + c5);
180 ; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {
181 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 3, 256 * c1 + c5);
182 ; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {
183 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 4, 256 * c1 + c5);
184 ; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {
185 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 5, 256 * c1 + c5);
186 ; CHECK-NEXT: if (2048 * c0 + 7 == nj)
187 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, nj - 1, 256 * c1 + c5);
188 ; CHECK-NEXT: }
189 ; CHECK-NEXT: }
190 ; CHECK-NEXT: }
191 ; CHECK-NEXT: }
192 ; CHECK-NEXT: }
193 ; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 2) {
194 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0, 256 * c1 + c5);
195 ; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {
196 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 1, 256 * c1 + c5);
197 ; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {
198 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 2, 256 * c1 + c5);
199 ; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {
200 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 3, 256 * c1 + c5);
201 ; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {
202 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 4, 256 * c1 + c5);
203 ; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {
204 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 5, 256 * c1 + c5);
205 ; CHECK-NEXT: if (2048 * c0 + 7 == nj)
206 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, nj - 1, 256 * c1 + c5);
207 ; CHECK-NEXT: }
208 ; CHECK-NEXT: }
209 ; CHECK-NEXT: }
210 ; CHECK-NEXT: }
211 ; CHECK-NEXT: }
212 ; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 3) {
213 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0, 256 * c1 + c5);
214 ; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {
215 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 1, 256 * c1 + c5);
216 ; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {
217 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 2, 256 * c1 + c5);
218 ; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {
219 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 3, 256 * c1 + c5);
220 ; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {
221 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 4, 256 * c1 + c5);
222 ; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {
223 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 5, 256 * c1 + c5);
224 ; CHECK-NEXT: if (2048 * c0 + 7 == nj)
225 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, nj - 1, 256 * c1 + c5);
226 ; CHECK-NEXT: }
227 ; CHECK-NEXT: }
228 ; CHECK-NEXT: }
229 ; CHECK-NEXT: }
230 ; CHECK-NEXT: }
231 ; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 4) {
232 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0, 256 * c1 + c5);
233 ; CHECK-NEXT: if (nj >= 2048 * c0 + 2) {
234 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 1, 256 * c1 + c5);
235 ; CHECK-NEXT: if (nj >= 2048 * c0 + 3) {
236 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 2, 256 * c1 + c5);
237 ; CHECK-NEXT: if (nj >= 2048 * c0 + 4) {
238 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 3, 256 * c1 + c5);
239 ; CHECK-NEXT: if (nj >= 2048 * c0 + 5) {
240 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 4, 256 * c1 + c5);
241 ; CHECK-NEXT: if (nj >= 2048 * c0 + 6) {
242 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 5, 256 * c1 + c5);
243 ; CHECK-NEXT: if (2048 * c0 + 7 == nj)
244 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, nj - 1, 256 * c1 + c5);
245 ; CHECK-NEXT: }
246 ; CHECK-NEXT: }
247 ; CHECK-NEXT: }
248 ; CHECK-NEXT: }
249 ; CHECK-NEXT: }
250 ; CHECK-NEXT: }
251 ; CHECK-NEXT: }
252 ; CHECK-NEXT: }
253 ; CHECK-NEXT: }
254 ; CHECK-NEXT: }
255 ; CHECK-NEXT: } else if (96 * c2 + 3 >= ni) {
256 ; CHECK-NEXT: for (int c3 = 0; c3 <= min(255, -256 * c0 + (nj - 1) / 8); c3 += 1)
257 ; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {
258 ; CHECK-NEXT: // Loop Vectorizer Disabled
259 ; CHECK-NEXT: // Register tiling - Points
260 ; CHECK-NEXT: {
261 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3, 256 * c1 + c5);
262 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) {
263 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
264 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) {
265 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
266 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) {
267 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
268 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) {
269 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
270 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) {
271 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
272 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) {
273 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
274 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8)
275 ; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
276 ; CHECK-NEXT: }
277 ; CHECK-NEXT: }
278 ; CHECK-NEXT: }
279 ; CHECK-NEXT: }
280 ; CHECK-NEXT: }
281 ; CHECK-NEXT: }
282 ; CHECK-NEXT: if (ni >= 96 * c2 + 2) {
283 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);
284 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) {
285 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
286 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) {
287 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
288 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) {
289 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
290 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) {
291 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
292 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) {
293 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
294 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) {
295 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
296 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8)
297 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
298 ; CHECK-NEXT: }
299 ; CHECK-NEXT: }
300 ; CHECK-NEXT: }
301 ; CHECK-NEXT: }
302 ; CHECK-NEXT: }
303 ; CHECK-NEXT: }
304 ; CHECK-NEXT: if (96 * c2 + 3 == ni) {
305 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5);
306 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) {
307 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5);
308 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) {
309 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5);
310 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) {
311 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5);
312 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) {
313 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5);
314 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) {
315 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5);
316 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) {
317 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5);
318 ; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8)
319 ; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5);
320 ; CHECK-NEXT: }
321 ; CHECK-NEXT: }
322 ; CHECK-NEXT: }
323 ; CHECK-NEXT: }
324 ; CHECK-NEXT: }
325 ; CHECK-NEXT: }
326 ; CHECK-NEXT: }
327 ; CHECK-NEXT: }
328 ; CHECK-NEXT: }
329 ; CHECK-NEXT: }
330 ; CHECK-NEXT: } else {
331 ; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + (ni - 1) / 4); c4 += 1)
332 ; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) {
333 ; CHECK-NEXT: // Loop Vectorizer Disabled
334 ; CHECK-NEXT: // Register tiling - Points
335 ; CHECK-NEXT: {
336 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);
337 ; CHECK-NEXT: if (nj % 8 >= 2) {
338 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj, 256 * c1 + c5);
339 ; CHECK-NEXT: if (nj % 8 >= 3) {
340 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);
341 ; CHECK-NEXT: if (nj % 8 >= 4) {
342 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);
343 ; CHECK-NEXT: if (nj % 8 >= 5) {
344 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);
345 ; CHECK-NEXT: if (nj % 8 >= 6) {
346 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);
347 ; CHECK-NEXT: if ((nj + 1) % 8 == 0)
348 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, nj - 1, 256 * c1 + c5);
349 ; CHECK-NEXT: }
350 ; CHECK-NEXT: }
351 ; CHECK-NEXT: }
352 ; CHECK-NEXT: }
353 ; CHECK-NEXT: }
354 ; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 2) {
355 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);
356 ; CHECK-NEXT: if (nj % 8 >= 2) {
357 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj, 256 * c1 + c5);
358 ; CHECK-NEXT: if (nj % 8 >= 3) {
359 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);
360 ; CHECK-NEXT: if (nj % 8 >= 4) {
361 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);
362 ; CHECK-NEXT: if (nj % 8 >= 5) {
363 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);
364 ; CHECK-NEXT: if (nj % 8 >= 6) {
365 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);
366 ; CHECK-NEXT: if ((nj + 1) % 8 == 0)
367 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, nj - 1, 256 * c1 + c5);
368 ; CHECK-NEXT: }
369 ; CHECK-NEXT: }
370 ; CHECK-NEXT: }
371 ; CHECK-NEXT: }
372 ; CHECK-NEXT: }
373 ; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 3) {
374 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);
375 ; CHECK-NEXT: if (nj % 8 >= 2) {
376 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj, 256 * c1 + c5);
377 ; CHECK-NEXT: if (nj % 8 >= 3) {
378 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);
379 ; CHECK-NEXT: if (nj % 8 >= 4) {
380 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);
381 ; CHECK-NEXT: if (nj % 8 >= 5) {
382 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);
383 ; CHECK-NEXT: if (nj % 8 >= 6) {
384 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);
385 ; CHECK-NEXT: if ((nj + 1) % 8 == 0)
386 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, nj - 1, 256 * c1 + c5);
387 ; CHECK-NEXT: }
388 ; CHECK-NEXT: }
389 ; CHECK-NEXT: }
390 ; CHECK-NEXT: }
391 ; CHECK-NEXT: }
392 ; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 4) {
393 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5);
394 ; CHECK-NEXT: if (nj % 8 >= 2) {
395 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj, 256 * c1 + c5);
396 ; CHECK-NEXT: if (nj % 8 >= 3) {
397 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5);
398 ; CHECK-NEXT: if (nj % 8 >= 4) {
399 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5);
400 ; CHECK-NEXT: if (nj % 8 >= 5) {
401 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5);
402 ; CHECK-NEXT: if (nj % 8 >= 6) {
403 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5);
404 ; CHECK-NEXT: if ((nj + 1) % 8 == 0)
405 ; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, nj - 1, 256 * c1 + c5);
406 ; CHECK-NEXT: }
407 ; CHECK-NEXT: }
408 ; CHECK-NEXT: }
409 ; CHECK-NEXT: }
410 ; CHECK-NEXT: }
411 ; CHECK-NEXT: }
412 ; CHECK-NEXT: }
413 ; CHECK-NEXT: }
414 ; CHECK-NEXT: }
415 ; CHECK-NEXT: }
416 ; CHECK-NEXT: }
417 ; CHECK-NEXT: }
418 ; CHECK-NEXT: }
419 ; CHECK-NEXT: }
420 ; CHECK-NEXT: }
421 ; CHECK-NEXT: }
422 ;
424 ; AUTO-VECTORIZATION: fmul <4 x double>
425 ; AUTO-VECTORIZATION: fadd <4 x double>
427 ; AUTO-VECTORIZATION: 36 SLP - Number of vector instructions generated
428 ; AUTO-VECTORIZATION: 453 licm - Number of instructions hoisted out of loop
429 ; AUTO-VECTORIZATION: 2 licm - Number of load insts hoisted or sunk
430 ; AUTO-VECTORIZATION: 32 licm - Number of memory locations promoted to registers
431 ;
432 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
433 target triple = "x86_64-unknown-unknown"
435 define internal void @kernel_gemm(i32 %ni, i32 %nj, i32 %nk, double %alpha, double %beta, ptr %C, ptr %A, ptr %B) #0 {
436 entry:
437 br label %entry.split
439 entry.split: ; preds = %entry
440 %cmp39 = icmp sgt i32 %ni, 0
441 br i1 %cmp39, label %for.cond1.preheader.lr.ph, label %for.end22
443 for.cond1.preheader.lr.ph: ; preds = %entry.split
444 br label %for.cond1.preheader
446 for.cond1.preheader: ; preds = %for.inc20, %for.cond1.preheader.lr.ph
447 %indvars.iv45 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next46, %for.inc20 ]
448 %cmp237 = icmp sgt i32 %nj, 0
449 br i1 %cmp237, label %for.cond4.preheader.lr.ph, label %for.inc20
451 for.cond4.preheader.lr.ph: ; preds = %for.cond1.preheader
452 br label %for.cond4.preheader
454 for.cond4.preheader: ; preds = %for.inc17, %for.cond4.preheader.lr.ph
455 %indvars.iv41 = phi i64 [ 0, %for.cond4.preheader.lr.ph ], [ %indvars.iv.next42, %for.inc17 ]
456 %cmp535 = icmp sgt i32 %nk, 0
457 br i1 %cmp535, label %for.body6.lr.ph, label %for.inc17
459 for.body6.lr.ph: ; preds = %for.cond4.preheader
460 br label %for.body6
462 for.body6: ; preds = %for.body6, %for.body6.lr.ph
463 %indvars.iv = phi i64 [ 0, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ]
464 %arrayidx8 = getelementptr inbounds [1024 x double], ptr %A, i64 %indvars.iv45, i64 %indvars.iv
465 %tmp = load double, ptr %arrayidx8, align 8
466 %arrayidx12 = getelementptr inbounds [1024 x double], ptr %B, i64 %indvars.iv, i64 %indvars.iv41
467 %tmp1 = load double, ptr %arrayidx12, align 8
468 %mul = fmul double %tmp, %tmp1
469 %arrayidx16 = getelementptr inbounds [1024 x double], ptr %C, i64 %indvars.iv45, i64 %indvars.iv41
470 %tmp2 = load double, ptr %arrayidx16, align 8
471 %add = fadd double %tmp2, %mul
472 store double %add, ptr %arrayidx16, align 8
473 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
474 %wide.trip.count = zext i32 %nk to i64
475 %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
476 br i1 %exitcond, label %for.body6, label %for.cond4.for.inc17_crit_edge
478 for.cond4.for.inc17_crit_edge: ; preds = %for.body6
479 br label %for.inc17
481 for.inc17: ; preds = %for.cond4.for.inc17_crit_edge, %for.cond4.preheader
482 %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1
483 %wide.trip.count43 = zext i32 %nj to i64
484 %exitcond44 = icmp ne i64 %indvars.iv.next42, %wide.trip.count43
485 br i1 %exitcond44, label %for.cond4.preheader, label %for.cond1.for.inc20_crit_edge
487 for.cond1.for.inc20_crit_edge: ; preds = %for.inc17
488 br label %for.inc20
490 for.inc20: ; preds = %for.cond1.for.inc20_crit_edge, %for.cond1.preheader
491 %indvars.iv.next46 = add nuw nsw i64 %indvars.iv45, 1
492 %wide.trip.count47 = zext i32 %ni to i64
493 %exitcond48 = icmp ne i64 %indvars.iv.next46, %wide.trip.count47
494 br i1 %exitcond48, label %for.cond1.preheader, label %for.cond.for.end22_crit_edge
496 for.cond.for.end22_crit_edge: ; preds = %for.inc20
497 br label %for.end22
499 for.end22: ; preds = %for.cond.for.end22_crit_edge, %entry.split
500 ret void
501 }
503 attributes #0 = { nounwind uwtable "target-cpu"="x86-64" "target-features"="+aes,+avx,+cmov,+cx16,+fxsr,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" }