| blob | dc0edc6c5a3ba6c100bd2d5fb9b1c8d504a7c55a |
1 ; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
2 ; RUN: -debug -polly-tc-opt=true -disable-output < %s 2>&1 | FileCheck %s
3 ; RUN: opt %loadPolly -polly-opt-isl -polly-pattern-matching-based-opts=true \
4 ; RUN: -polly-target-throughput-vector-fma=1 \
5 ; RUN: -polly-target-latency-vector-fma=8 \
6 ; RUN: -polly-target-1st-cache-level-size=32768 \
7 ; RUN: -polly-target-vector-register-bitwidth=256 \
8 ; RUN: -polly-target-2nd-cache-level-size=262144 -polly-print-ast \
9 ; RUN: -polly-tc-opt=true -disable-output -polly-opt-isl < %s | \
10 ; RUN: FileCheck %s --check-prefix=PATTERN-MATCHING-OPTS
11 ; REQUIRES: asserts
12 ;
13 ; C := A * B + C
14 ; Check that the pattern matching optimizations can detect different
15 ; permutations of GEMM loop and produce the correct ISL AST. In this case,
16 ; dimensions of band nodes can be implicitly permuted by the algorithm
17 ; applied during the schedule generation. It should be taken into the
18 ; account during the pattern matching optimizations.
19 ; for (i = 0; i < _PB_NI; i++)
20 ; for (k = 0; k < _PB_NK; ++k)
21 ; for (j = 0; j < _PB_NJ; j++)
22 ; C[i][j] += A[i][k] * B[k][j];
23 ;
24 ; CHECK: The tensor contraction pattern was detected
25 ; CHECK: The matrix multiplication pattern was detected
26 ;
27 ; PATTERN-MATCHING-OPTS: // 1st level tiling - Tiles
28 ; PATTERN-MATCHING-OPTS-NEXT: for (int c1 = 0; c1 <= 3; c1 += 1) {
29 ; PATTERN-MATCHING-OPTS-NEXT: for (int c3 = 256 * c1; c3 <= 256 * c1 + 255; c3 += 1)
30 ; PATTERN-MATCHING-OPTS-NEXT: for (int c4 = 0; c4 <= 1023; c4 += 1)
31 ; PATTERN-MATCHING-OPTS-NEXT: CopyStmt_0(0, c3, c4);
32 ; PATTERN-MATCHING-OPTS-NEXT: for (int c2 = 0; c2 <= 10; c2 += 1) {
33 ; PATTERN-MATCHING-OPTS-NEXT: for (int c6 = 96 * c2; c6 <= min(1023, 96 * c2 + 95); c6 += 1)
34 ; PATTERN-MATCHING-OPTS-NEXT: for (int c7 = 256 * c1; c7 <= 256 * c1 + 255; c7 += 1)
35 ; PATTERN-MATCHING-OPTS-NEXT: CopyStmt_1(0, c1, c2, c6, c7);
36 ; PATTERN-MATCHING-OPTS-NEXT: // 1st level tiling - Points
37 ; PATTERN-MATCHING-OPTS-NEXT: // Register tiling - Tiles
38 ; PATTERN-MATCHING-OPTS-NEXT: for (int c3 = 0; c3 <= 127; c3 += 1)
39 ; PATTERN-MATCHING-OPTS-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + 255); c4 += 1)
40 ; PATTERN-MATCHING-OPTS-NEXT: for (int c5 = 0; c5 <= 255; c5 += 1) {
41 ; PATTERN-MATCHING-OPTS-NEXT: // Loop Vectorizer Disabled
42 ; PATTERN-MATCHING-OPTS-NEXT: // Register tiling - Points
43 ; PATTERN-MATCHING-OPTS-NEXT: {
44 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3);
45 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 1);
46 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 2);
47 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 3);
48 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 4);
49 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 5);
50 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 6);
51 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 256 * c1 + c5, 8 * c3 + 7);
52 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3);
53 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 1);
54 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 2);
55 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 3);
56 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 4);
57 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 5);
58 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 6);
59 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 256 * c1 + c5, 8 * c3 + 7);
60 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3);
61 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 1);
62 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 2);
63 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 3);
64 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 4);
65 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 5);
66 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 6);
67 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 256 * c1 + c5, 8 * c3 + 7);
68 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3);
69 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 1);
70 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 2);
71 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 3);
72 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 4);
73 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 5);
74 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 6);
75 ; PATTERN-MATCHING-OPTS-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 256 * c1 + c5, 8 * c3 + 7);
76 ; PATTERN-MATCHING-OPTS-NEXT: }
77 ; PATTERN-MATCHING-OPTS-NEXT: }
78 ; PATTERN-MATCHING-OPTS-NEXT: }
79 ; PATTERN-MATCHING-OPTS-NEXT: }
80 ;
81 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
82 target triple = "x86_64-unknown-unknown"
84 define internal void @kernel_gemm(i32 %ni, i32 %nj, i32 %nk, double %alpha, double %beta, ptr %C, ptr %A, ptr %B) {
85 entry:
86 br label %entry.split
88 entry.split: ; preds = %entry
89 br label %for.cond1.preheader
91 for.cond1.preheader: ; preds = %for.inc20, %entry.split
92 %indvars.iv41 = phi i64 [ 0, %entry.split ], [ %indvars.iv.next42, %for.inc20 ]
93 br label %for.cond4.preheader
95 for.cond4.preheader: ; preds = %for.inc17, %for.cond1.preheader
96 %indvars.iv38 = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next39, %for.inc17 ]
97 br label %for.body6
99 for.body6: ; preds = %for.body6, %for.cond4.preheader
100 %indvars.iv = phi i64 [ 0, %for.cond4.preheader ], [ %indvars.iv.next, %for.body6 ]
101 %arrayidx8 = getelementptr inbounds [1024 x double], ptr %A, i64 %indvars.iv41, i64 %indvars.iv38
102 %tmp = load double, ptr %arrayidx8, align 8
103 %arrayidx12 = getelementptr inbounds [1024 x double], ptr %B, i64 %indvars.iv38, i64 %indvars.iv
104 %tmp1 = load double, ptr %arrayidx12, align 8
105 %mul = fmul double %tmp, %tmp1
106 %arrayidx16 = getelementptr inbounds [1024 x double], ptr %C, i64 %indvars.iv41, i64 %indvars.iv
107 %tmp2 = load double, ptr %arrayidx16, align 8
108 %add = fadd double %tmp2, %mul
109 store double %add, ptr %arrayidx16, align 8
110 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
111 %exitcond = icmp ne i64 %indvars.iv.next, 1024
112 br i1 %exitcond, label %for.body6, label %for.inc17
114 for.inc17: ; preds = %for.body6
115 %indvars.iv.next39 = add nuw nsw i64 %indvars.iv38, 1
116 %exitcond40 = icmp ne i64 %indvars.iv.next39, 1024
117 br i1 %exitcond40, label %for.cond4.preheader, label %for.inc20
119 for.inc20: ; preds = %for.inc17
120 %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1
121 %exitcond43 = icmp ne i64 %indvars.iv.next42, 1024
122 br i1 %exitcond43, label %for.cond1.preheader, label %for.end22
124 for.end22: ; preds = %for.inc20
125 ret void
126 }