| blob | a82c5215f3b2c65fdf5484bae7fb742f9a067964 |
1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2 ; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
3 ; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
4 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
5 ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
6 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
7 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX10-DL %s
8 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11-DL %s
10 define amdgpu_kernel void @udot4_acc32(ptr addrspace(1) %src1,
11 ; GFX7-LABEL: udot4_acc32:
12 ; GFX7: ; %bb.0: ; %entry
13 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
14 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
15 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
16 ; GFX7-NEXT: s_mov_b32 s10, 0
17 ; GFX7-NEXT: s_mov_b32 s11, s3
18 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
19 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
20 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
21 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
22 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
23 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
24 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
25 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
26 ; GFX7-NEXT: s_mov_b32 s2, -1
27 ; GFX7-NEXT: s_waitcnt vmcnt(1)
28 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
29 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
30 ; GFX7-NEXT: s_waitcnt vmcnt(0)
31 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v0
32 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
33 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
34 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, s4
35 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
36 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
37 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
38 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
39 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
40 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
41 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
42 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
43 ; GFX7-NEXT: s_endpgm
44 ;
45 ; GFX8-LABEL: udot4_acc32:
46 ; GFX8: ; %bb.0: ; %entry
47 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
48 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
49 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
50 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
51 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
52 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
53 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
54 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
55 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
56 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
57 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
58 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
59 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
60 ; GFX8-NEXT: s_waitcnt vmcnt(1)
61 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
62 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
63 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
64 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
65 ; GFX8-NEXT: s_waitcnt vmcnt(0)
66 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v0
67 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
68 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
69 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s2
70 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
71 ; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
72 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
73 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
74 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
75 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
76 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
77 ; GFX8-NEXT: flat_store_dword v[0:1], v2
78 ; GFX8-NEXT: s_endpgm
79 ;
80 ; GFX9-NODL-LABEL: udot4_acc32:
81 ; GFX9-NODL: ; %bb.0: ; %entry
82 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
83 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
84 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
85 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
86 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
87 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
88 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
89 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
90 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
91 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
92 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
93 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
94 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
95 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
96 ; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
97 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
98 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
99 ; GFX9-NODL-NEXT: s_endpgm
100 ;
101 ; GFX9-DL-LABEL: udot4_acc32:
102 ; GFX9-DL: ; %bb.0: ; %entry
103 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
104 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
105 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
106 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
107 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
108 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
109 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
110 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
111 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
112 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s0
113 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
114 ; GFX9-DL-NEXT: s_endpgm
115 ;
116 ; GFX10-DL-LABEL: udot4_acc32:
117 ; GFX10-DL: ; %bb.0: ; %entry
118 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
119 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
120 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
121 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
122 ; GFX10-DL-NEXT: s_clause 0x1
123 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
124 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
125 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
126 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
127 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
128 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s2
129 ; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
130 ; GFX10-DL-NEXT: s_endpgm
131 ;
132 ; GFX11-DL-LABEL: udot4_acc32:
133 ; GFX11-DL: ; %bb.0: ; %entry
134 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
135 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
136 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
137 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
138 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
139 ; GFX11-DL-NEXT: s_clause 0x1
140 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
141 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
142 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
143 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
144 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
145 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
146 ; GFX11-DL-NEXT: s_nop 0
147 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
148 ; GFX11-DL-NEXT: s_endpgm
149 ptr addrspace(1) %src2,
150 ptr addrspace(1) nocapture %dst) {
151 entry:
152 %idx = call i32 @llvm.amdgcn.workitem.id.x()
153 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
154 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
155 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
156 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
158 %v1e0 = extractelement <4 x i8> %vec1, i64 0
159 %cv1e0 = zext i8 %v1e0 to i32
160 %v2e0 = extractelement <4 x i8> %vec2, i64 0
161 %cv2e0 = zext i8 %v2e0 to i32
162 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
164 %v1e1 = extractelement <4 x i8> %vec1, i64 1
165 %cv1e1 = zext i8 %v1e1 to i32
166 %v2e1 = extractelement <4 x i8> %vec2, i64 1
167 %cv2e1 = zext i8 %v2e1 to i32
168 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
170 %v1e2 = extractelement <4 x i8> %vec1, i64 2
171 %cv1e2 = zext i8 %v1e2 to i32
172 %v2e2 = extractelement <4 x i8> %vec2, i64 2
173 %cv2e2 = zext i8 %v2e2 to i32
174 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
176 %v1e3 = extractelement <4 x i8> %vec1, i64 3
177 %cv1e3 = zext i8 %v1e3 to i32
178 %v2e3 = extractelement <4 x i8> %vec2, i64 3
179 %cv2e3 = zext i8 %v2e3 to i32
180 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
182 %acc = load i32, ptr addrspace(1) %dst, align 4
183 %mad1 = add i32 %mul1, %acc
184 %mad2 = add i32 %mad1, %mul2
185 %mad3 = add i32 %mad2, %mul3
186 %mad4 = add i32 %mad3, %mul4
188 store i32 %mad4, ptr addrspace(1) %dst, align 4
189 ret void
190 }
192 define amdgpu_kernel void @udot4_acc16(ptr addrspace(1) %src1,
193 ; GFX7-LABEL: udot4_acc16:
194 ; GFX7: ; %bb.0: ; %entry
195 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
196 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
197 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
198 ; GFX7-NEXT: s_mov_b32 s10, 0
199 ; GFX7-NEXT: s_mov_b32 s11, s3
200 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
201 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
202 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
203 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
204 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
205 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
206 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
207 ; GFX7-NEXT: s_mov_b32 s2, -1
208 ; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0
209 ; GFX7-NEXT: s_waitcnt vmcnt(2)
210 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
211 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
212 ; GFX7-NEXT: s_waitcnt vmcnt(1)
213 ; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0
214 ; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
215 ; GFX7-NEXT: s_waitcnt vmcnt(0)
216 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
217 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
218 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
219 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
220 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
221 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
222 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
223 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
224 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
225 ; GFX7-NEXT: s_endpgm
226 ;
227 ; GFX8-LABEL: udot4_acc16:
228 ; GFX8: ; %bb.0: ; %entry
229 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
230 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
231 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
232 ; GFX8-NEXT: v_mov_b32_e32 v5, 0xff
233 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
234 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
235 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
236 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
237 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
238 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
239 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
240 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
241 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
242 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
243 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
244 ; GFX8-NEXT: flat_load_ushort v4, v[0:1]
245 ; GFX8-NEXT: s_waitcnt vmcnt(2)
246 ; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v3
247 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3
248 ; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8
249 ; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
250 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
251 ; GFX8-NEXT: s_waitcnt vmcnt(1)
252 ; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v2
253 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
254 ; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
255 ; GFX8-NEXT: s_waitcnt vmcnt(0)
256 ; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4
257 ; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
258 ; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4
259 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
260 ; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4
261 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
262 ; GFX8-NEXT: flat_store_short v[0:1], v2
263 ; GFX8-NEXT: s_endpgm
264 ;
265 ; GFX9-NODL-LABEL: udot4_acc16:
266 ; GFX9-NODL: ; %bb.0: ; %entry
267 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
268 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
269 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
270 ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
271 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
272 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
273 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
274 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
275 ; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
276 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
277 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v1
278 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
279 ; GFX9-NODL-NEXT: v_and_b32_e32 v5, 0xff, v2
280 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
281 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
282 ; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6
283 ; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
284 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
285 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3
286 ; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
287 ; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
288 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v6, v7, v3
289 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
290 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
291 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v9, v3
292 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
293 ; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
294 ; GFX9-NODL-NEXT: s_endpgm
295 ;
296 ; GFX9-DL-LABEL: udot4_acc16:
297 ; GFX9-DL: ; %bb.0: ; %entry
298 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
299 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
300 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
301 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
302 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
303 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
304 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
305 ; GFX9-DL-NEXT: global_load_ushort v4, v1, s[2:3]
306 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
307 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4
308 ; GFX9-DL-NEXT: global_store_short v1, v0, s[2:3]
309 ; GFX9-DL-NEXT: s_endpgm
310 ;
311 ; GFX10-DL-LABEL: udot4_acc16:
312 ; GFX10-DL: ; %bb.0: ; %entry
313 ; GFX10-DL-NEXT: s_clause 0x1
314 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
315 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
316 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
317 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
318 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
319 ; GFX10-DL-NEXT: s_clause 0x1
320 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
321 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
322 ; GFX10-DL-NEXT: global_load_ushort v4, v1, s[2:3]
323 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
324 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4
325 ; GFX10-DL-NEXT: global_store_short v1, v0, s[2:3]
326 ; GFX10-DL-NEXT: s_endpgm
327 ;
328 ; GFX11-DL-LABEL: udot4_acc16:
329 ; GFX11-DL: ; %bb.0: ; %entry
330 ; GFX11-DL-NEXT: s_clause 0x1
331 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
332 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
333 ; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
334 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
335 ; GFX11-DL-NEXT: s_clause 0x1
336 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]
337 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
338 ; GFX11-DL-NEXT: global_load_u16 v3, v1, s[0:1]
339 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
340 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, v3
341 ; GFX11-DL-NEXT: global_store_b16 v1, v0, s[0:1]
342 ; GFX11-DL-NEXT: s_nop 0
343 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
344 ; GFX11-DL-NEXT: s_endpgm
345 ptr addrspace(1) %src2,
346 ptr addrspace(1) nocapture %dst) {
347 entry:
348 %idx = call i32 @llvm.amdgcn.workitem.id.x()
349 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
350 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
351 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
352 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
354 %v1e0 = extractelement <4 x i8> %vec1, i64 0
355 %cv1e0 = zext i8 %v1e0 to i16
356 %v2e0 = extractelement <4 x i8> %vec2, i64 0
357 %cv2e0 = zext i8 %v2e0 to i16
358 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
360 %v1e1 = extractelement <4 x i8> %vec1, i64 1
361 %cv1e1 = zext i8 %v1e1 to i16
362 %v2e1 = extractelement <4 x i8> %vec2, i64 1
363 %cv2e1 = zext i8 %v2e1 to i16
364 %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1
366 %v1e2 = extractelement <4 x i8> %vec1, i64 2
367 %cv1e2 = zext i8 %v1e2 to i16
368 %v2e2 = extractelement <4 x i8> %vec2, i64 2
369 %cv2e2 = zext i8 %v2e2 to i16
370 %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2
372 %v1e3 = extractelement <4 x i8> %vec1, i64 3
373 %cv1e3 = zext i8 %v1e3 to i16
374 %v2e3 = extractelement <4 x i8> %vec2, i64 3
375 %cv2e3 = zext i8 %v2e3 to i16
376 %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3
378 %acc = load i16, ptr addrspace(1) %dst, align 2
379 %mad1 = add i16 %mul1, %acc
380 %mad2 = add i16 %mad1, %mul2
381 %mad3 = add i16 %mad2, %mul3
382 %mad4 = add i16 %mad3, %mul4
384 store i16 %mad4, ptr addrspace(1) %dst, align 2
385 ret void
386 }
388 define amdgpu_kernel void @udot4_acc8(ptr addrspace(1) %src1,
389 ; GFX7-LABEL: udot4_acc8:
390 ; GFX7: ; %bb.0: ; %entry
391 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
392 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
393 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
394 ; GFX7-NEXT: s_mov_b32 s10, 0
395 ; GFX7-NEXT: s_mov_b32 s11, s3
396 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
397 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
398 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
399 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
400 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
401 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
402 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
403 ; GFX7-NEXT: s_mov_b32 s2, -1
404 ; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
405 ; GFX7-NEXT: s_waitcnt vmcnt(2)
406 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
407 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
408 ; GFX7-NEXT: s_waitcnt vmcnt(1)
409 ; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0
410 ; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
411 ; GFX7-NEXT: s_waitcnt vmcnt(0)
412 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
413 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
414 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
415 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
416 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
417 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
418 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
419 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
420 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
421 ; GFX7-NEXT: s_endpgm
422 ;
423 ; GFX8-LABEL: udot4_acc8:
424 ; GFX8: ; %bb.0: ; %entry
425 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
426 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
427 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
428 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
429 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
430 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
431 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
432 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
433 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
434 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
435 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
436 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
437 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
438 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
439 ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
440 ; GFX8-NEXT: s_waitcnt vmcnt(2)
441 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
442 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
443 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3
444 ; GFX8-NEXT: s_waitcnt vmcnt(1)
445 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
446 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2
447 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2
448 ; GFX8-NEXT: s_waitcnt vmcnt(0)
449 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
450 ; GFX8-NEXT: v_mad_u16 v2, v7, v8, v2
451 ; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2
452 ; GFX8-NEXT: v_mad_u16 v2, v9, v10, v2
453 ; GFX8-NEXT: flat_store_byte v[0:1], v2
454 ; GFX8-NEXT: s_endpgm
455 ;
456 ; GFX9-NODL-LABEL: udot4_acc8:
457 ; GFX9-NODL: ; %bb.0: ; %entry
458 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
459 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
460 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
461 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
462 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
463 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
464 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
465 ; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[2:3]
466 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
467 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
468 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
469 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
470 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
471 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1
472 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
473 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
474 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
475 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v6, v7, v1
476 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v2
477 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
478 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v8, v9, v1
479 ; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
480 ; GFX9-NODL-NEXT: s_endpgm
481 ;
482 ; GFX9-DL-LABEL: udot4_acc8:
483 ; GFX9-DL: ; %bb.0: ; %entry
484 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
485 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
486 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
487 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
488 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
489 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
490 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
491 ; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
492 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
493 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4
494 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
495 ; GFX9-DL-NEXT: s_endpgm
496 ;
497 ; GFX10-DL-LABEL: udot4_acc8:
498 ; GFX10-DL: ; %bb.0: ; %entry
499 ; GFX10-DL-NEXT: s_clause 0x1
500 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
501 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
502 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
503 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
504 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
505 ; GFX10-DL-NEXT: s_clause 0x1
506 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
507 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
508 ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
509 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
510 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v2, v3, v4
511 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
512 ; GFX10-DL-NEXT: s_endpgm
513 ;
514 ; GFX11-DL-LABEL: udot4_acc8:
515 ; GFX11-DL: ; %bb.0: ; %entry
516 ; GFX11-DL-NEXT: s_clause 0x1
517 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
518 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
519 ; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
520 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
521 ; GFX11-DL-NEXT: s_clause 0x1
522 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]
523 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
524 ; GFX11-DL-NEXT: global_load_u8 v3, v1, s[0:1]
525 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
526 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v2, v0, v3
527 ; GFX11-DL-NEXT: global_store_b8 v1, v0, s[0:1]
528 ; GFX11-DL-NEXT: s_nop 0
529 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
530 ; GFX11-DL-NEXT: s_endpgm
531 ptr addrspace(1) %src2,
532 ptr addrspace(1) nocapture %dst) {
533 entry:
534 %idx = call i32 @llvm.amdgcn.workitem.id.x()
535 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
536 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
537 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
538 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
540 %v1e0 = extractelement <4 x i8> %vec1, i64 0
541 %v2e0 = extractelement <4 x i8> %vec2, i64 0
542 %mul1 = mul nuw nsw i8 %v1e0, %v2e0
544 %v1e1 = extractelement <4 x i8> %vec1, i64 1
545 %v2e1 = extractelement <4 x i8> %vec2, i64 1
546 %mul2 = mul nuw nsw i8 %v1e1, %v2e1
548 %v1e2 = extractelement <4 x i8> %vec1, i64 2
549 %v2e2 = extractelement <4 x i8> %vec2, i64 2
550 %mul3 = mul nuw nsw i8 %v1e2, %v2e2
552 %v1e3 = extractelement <4 x i8> %vec1, i64 3
553 %v2e3 = extractelement <4 x i8> %vec2, i64 3
554 %mul4 = mul nuw nsw i8 %v1e3, %v2e3
556 %acc = load i8, ptr addrspace(1) %dst, align 2
557 %mad1 = add i8 %mul1, %acc
558 %mad2 = add i8 %mad1, %mul2
559 %mad3 = add i8 %mad2, %mul3
560 %mad4 = add i8 %mad3, %mul4
562 store i8 %mad4, ptr addrspace(1) %dst, align 2
563 ret void
564 }
566 define amdgpu_kernel void @udot2_8(ptr addrspace(1) %src1,
567 ; GFX7-LABEL: udot2_8:
568 ; GFX7: ; %bb.0: ; %entry
569 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
570 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
571 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
572 ; GFX7-NEXT: s_mov_b32 s10, 0
573 ; GFX7-NEXT: s_mov_b32 s11, s3
574 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
575 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
576 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
577 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
578 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
579 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
580 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
581 ; GFX7-NEXT: s_mov_b32 s2, -1
582 ; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
583 ; GFX7-NEXT: s_waitcnt vmcnt(2)
584 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
585 ; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 8
586 ; GFX7-NEXT: s_waitcnt vmcnt(1)
587 ; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
588 ; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8
589 ; GFX7-NEXT: s_waitcnt vmcnt(0)
590 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v4, v1
591 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
592 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
593 ; GFX7-NEXT: s_endpgm
594 ;
595 ; GFX8-LABEL: udot2_8:
596 ; GFX8: ; %bb.0: ; %entry
597 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
598 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
599 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
600 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
601 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
602 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
603 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
604 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
605 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
606 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
607 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
608 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
609 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
610 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
611 ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
612 ; GFX8-NEXT: s_waitcnt vmcnt(2)
613 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v3
614 ; GFX8-NEXT: s_waitcnt vmcnt(1)
615 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v2
616 ; GFX8-NEXT: s_waitcnt vmcnt(0)
617 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
618 ; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2
619 ; GFX8-NEXT: flat_store_byte v[0:1], v2
620 ; GFX8-NEXT: s_endpgm
621 ;
622 ; GFX9-NODL-LABEL: udot2_8:
623 ; GFX9-NODL: ; %bb.0: ; %entry
624 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
625 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
626 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
627 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, 0
628 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
629 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
630 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[6:7]
631 ; GFX9-NODL-NEXT: global_load_ubyte v4, v1, s[2:3]
632 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
633 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v0, 8, v2
634 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
635 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 8, v3
636 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
637 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v2, v2, v3, v4
638 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v0, v0, v5, v2
639 ; GFX9-NODL-NEXT: global_store_byte v1, v0, s[2:3]
640 ; GFX9-NODL-NEXT: s_endpgm
641 ;
642 ; GFX9-DL-LABEL: udot2_8:
643 ; GFX9-DL: ; %bb.0: ; %entry
644 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
645 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
646 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
647 ; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c0100
648 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
649 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
650 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
651 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
652 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
653 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
654 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s0
655 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
656 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s0
657 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
658 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
659 ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
660 ; GFX9-DL-NEXT: s_endpgm
661 ;
662 ; GFX10-DL-LABEL: udot2_8:
663 ; GFX10-DL: ; %bb.0: ; %entry
664 ; GFX10-DL-NEXT: s_clause 0x1
665 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
666 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
667 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
668 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
669 ; GFX10-DL-NEXT: s_clause 0x1
670 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
671 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
672 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
673 ; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
674 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
675 ; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100
676 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
677 ; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0100
678 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
679 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
680 ; GFX10-DL-NEXT: global_store_byte v0, v1, s[2:3]
681 ; GFX10-DL-NEXT: s_endpgm
682 ;
683 ; GFX11-DL-LABEL: udot2_8:
684 ; GFX11-DL: ; %bb.0: ; %entry
685 ; GFX11-DL-NEXT: s_clause 0x1
686 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
687 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
688 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
689 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
690 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
691 ; GFX11-DL-NEXT: s_clause 0x1
692 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
693 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
694 ; GFX11-DL-NEXT: global_load_u8 v3, v2, s[0:1]
695 ; GFX11-DL-NEXT: s_waitcnt vmcnt(2)
696 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100
697 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
698 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0100
699 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
700 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
701 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v3
702 ; GFX11-DL-NEXT: global_store_b8 v2, v0, s[0:1]
703 ; GFX11-DL-NEXT: s_nop 0
704 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
705 ; GFX11-DL-NEXT: s_endpgm
706 ptr addrspace(1) %src2,
707 ptr addrspace(1) nocapture %dst) {
708 entry:
709 %idx = call i32 @llvm.amdgcn.workitem.id.x()
710 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
711 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
712 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
713 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
715 %v1e0 = extractelement <4 x i8> %vec1, i64 0
716 %v2e0 = extractelement <4 x i8> %vec2, i64 0
717 %mul1 = mul nuw nsw i8 %v1e0, %v2e0
719 %v1e1 = extractelement <4 x i8> %vec1, i64 1
720 %v2e1 = extractelement <4 x i8> %vec2, i64 1
721 %mul2 = mul nuw nsw i8 %v1e1, %v2e1
723 %acc = load i8, ptr addrspace(1) %dst, align 2
724 %mad1 = add i8 %mul1, %acc
725 %mad2 = add i8 %mad1, %mul2
726 store i8 %mad2, ptr addrspace(1) %dst, align 2
727 ret void
728 }
730 define amdgpu_kernel void @udot4_CommutationInsideMAD(ptr addrspace(1) %src1,
731 ; GFX7-LABEL: udot4_CommutationInsideMAD:
732 ; GFX7: ; %bb.0: ; %entry
733 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
734 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
735 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
736 ; GFX7-NEXT: s_mov_b32 s10, 0
737 ; GFX7-NEXT: s_mov_b32 s11, s3
738 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
739 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
740 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
741 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
742 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
743 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
744 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
745 ; GFX7-NEXT: s_mov_b32 s2, -1
746 ; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
747 ; GFX7-NEXT: s_waitcnt vmcnt(2)
748 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
749 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
750 ; GFX7-NEXT: s_waitcnt vmcnt(1)
751 ; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0
752 ; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
753 ; GFX7-NEXT: s_waitcnt vmcnt(0)
754 ; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v1
755 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
756 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
757 ; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1
758 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
759 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
760 ; GFX7-NEXT: v_mad_u32_u24 v1, v8, v5, v1
761 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
762 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
763 ; GFX7-NEXT: s_endpgm
764 ;
765 ; GFX8-LABEL: udot4_CommutationInsideMAD:
766 ; GFX8: ; %bb.0: ; %entry
767 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
768 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
769 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
770 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
771 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
772 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
773 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
774 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
775 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
776 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
777 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
778 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
779 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
780 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
781 ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
782 ; GFX8-NEXT: s_waitcnt vmcnt(2)
783 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
784 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
785 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3
786 ; GFX8-NEXT: s_waitcnt vmcnt(1)
787 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
788 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2
789 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2
790 ; GFX8-NEXT: s_waitcnt vmcnt(0)
791 ; GFX8-NEXT: v_mad_u16 v2, v2, v3, v4
792 ; GFX8-NEXT: v_mad_u16 v2, v8, v7, v2
793 ; GFX8-NEXT: v_mad_u16 v2, v6, v5, v2
794 ; GFX8-NEXT: v_mad_u16 v2, v10, v9, v2
795 ; GFX8-NEXT: flat_store_byte v[0:1], v2
796 ; GFX8-NEXT: s_endpgm
797 ;
798 ; GFX9-NODL-LABEL: udot4_CommutationInsideMAD:
799 ; GFX9-NODL: ; %bb.0: ; %entry
800 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
801 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
802 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
803 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
804 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
805 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
806 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
807 ; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[2:3]
808 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
809 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
810 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
811 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
812 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
813 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1
814 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
815 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v2, v1, v3
816 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
817 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v7, v6, v1
818 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v2
819 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v5, v4, v1
820 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v9, v8, v1
821 ; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
822 ; GFX9-NODL-NEXT: s_endpgm
823 ;
824 ; GFX9-DL-LABEL: udot4_CommutationInsideMAD:
825 ; GFX9-DL: ; %bb.0: ; %entry
826 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
827 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
828 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
829 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
830 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
831 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
832 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
833 ; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
834 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
835 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v4
836 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
837 ; GFX9-DL-NEXT: s_endpgm
838 ;
839 ; GFX10-DL-LABEL: udot4_CommutationInsideMAD:
840 ; GFX10-DL: ; %bb.0: ; %entry
841 ; GFX10-DL-NEXT: s_clause 0x1
842 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
843 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
844 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
845 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
846 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
847 ; GFX10-DL-NEXT: s_clause 0x1
848 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
849 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
850 ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
851 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
852 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v4
853 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
854 ; GFX10-DL-NEXT: s_endpgm
855 ;
856 ; GFX11-DL-LABEL: udot4_CommutationInsideMAD:
857 ; GFX11-DL: ; %bb.0: ; %entry
858 ; GFX11-DL-NEXT: s_clause 0x1
859 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
860 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
861 ; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
862 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
863 ; GFX11-DL-NEXT: s_clause 0x1
864 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]
865 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
866 ; GFX11-DL-NEXT: global_load_u8 v3, v1, s[0:1]
867 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
868 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v3
869 ; GFX11-DL-NEXT: global_store_b8 v1, v0, s[0:1]
870 ; GFX11-DL-NEXT: s_nop 0
871 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
872 ; GFX11-DL-NEXT: s_endpgm
873 ptr addrspace(1) %src2,
874 ptr addrspace(1) nocapture %dst) {
875 entry:
876 %idx = call i32 @llvm.amdgcn.workitem.id.x()
877 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
878 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
879 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
880 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
882 %v1e0 = extractelement <4 x i8> %vec1, i64 0
883 %v2e0 = extractelement <4 x i8> %vec2, i64 0
884 %mul1 = mul nuw nsw i8 %v2e0, %v1e0
886 %v1e1 = extractelement <4 x i8> %vec1, i64 1
887 %v2e1 = extractelement <4 x i8> %vec2, i64 1
888 %mul2 = mul nuw nsw i8 %v2e1, %v1e1
890 %v1e2 = extractelement <4 x i8> %vec1, i64 2
891 %v2e2 = extractelement <4 x i8> %vec2, i64 2
892 %mul3 = mul nuw nsw i8 %v2e2, %v1e2
894 %v1e3 = extractelement <4 x i8> %vec1, i64 3
895 %v2e3 = extractelement <4 x i8> %vec2, i64 3
896 %mul4 = mul nuw nsw i8 %v2e3, %v1e3
898 %acc = load i8, ptr addrspace(1) %dst, align 2
899 %mad1 = add i8 %acc, %mul1
900 %mad2 = add i8 %mul2, %mad1
901 %mad3 = add i8 %mul3, %mad2
902 %mad4 = add i8 %mul4, %mad3
904 store i8 %mad4, ptr addrspace(1) %dst, align 2
905 ret void
906 }
908 define amdgpu_kernel void @udot4_CommutationAccrossMADs(ptr addrspace(1) %src1,
909 ; GFX7-LABEL: udot4_CommutationAccrossMADs:
910 ; GFX7: ; %bb.0: ; %entry
911 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
912 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
913 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
914 ; GFX7-NEXT: s_mov_b32 s10, 0
915 ; GFX7-NEXT: s_mov_b32 s11, s3
916 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
917 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
918 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
919 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
920 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
921 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
922 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
923 ; GFX7-NEXT: s_mov_b32 s2, -1
924 ; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
925 ; GFX7-NEXT: s_waitcnt vmcnt(2)
926 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
927 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
928 ; GFX7-NEXT: s_waitcnt vmcnt(1)
929 ; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
930 ; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v0
931 ; GFX7-NEXT: s_waitcnt vmcnt(0)
932 ; GFX7-NEXT: v_mad_u32_u24 v1, v7, v4, v1
933 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
934 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
935 ; GFX7-NEXT: v_mad_u32_u24 v1, v6, v3, v1
936 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
937 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
938 ; GFX7-NEXT: v_mad_u32_u24 v1, v8, v5, v1
939 ; GFX7-NEXT: v_mad_u32_u24 v0, v0, v2, v1
940 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
941 ; GFX7-NEXT: s_endpgm
942 ;
943 ; GFX8-LABEL: udot4_CommutationAccrossMADs:
944 ; GFX8: ; %bb.0: ; %entry
945 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
946 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
947 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
948 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
949 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
950 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
951 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
952 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
953 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
954 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
955 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
956 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
957 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
958 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
959 ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
960 ; GFX8-NEXT: s_waitcnt vmcnt(2)
961 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v3
962 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
963 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v3
964 ; GFX8-NEXT: s_waitcnt vmcnt(1)
965 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2
966 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
967 ; GFX8-NEXT: v_lshrrev_b32_e32 v10, 24, v2
968 ; GFX8-NEXT: s_waitcnt vmcnt(0)
969 ; GFX8-NEXT: v_mad_u16 v4, v8, v7, v4
970 ; GFX8-NEXT: v_mad_u16 v2, v2, v3, v4
971 ; GFX8-NEXT: v_mad_u16 v2, v6, v5, v2
972 ; GFX8-NEXT: v_mad_u16 v2, v10, v9, v2
973 ; GFX8-NEXT: flat_store_byte v[0:1], v2
974 ; GFX8-NEXT: s_endpgm
975 ;
976 ; GFX9-NODL-LABEL: udot4_CommutationAccrossMADs:
977 ; GFX9-NODL: ; %bb.0: ; %entry
978 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
979 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
980 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
981 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
982 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
983 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
984 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
985 ; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[2:3]
986 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
987 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
988 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
989 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
990 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
991 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
992 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v7, v6, v3
993 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
994 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 24, v1
995 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v2, v1, v3
996 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v9, 24, v2
997 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v5, v4, v1
998 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v9, v8, v1
999 ; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
1000 ; GFX9-NODL-NEXT: s_endpgm
1001 ;
1002 ; GFX9-DL-LABEL: udot4_CommutationAccrossMADs:
1003 ; GFX9-DL: ; %bb.0: ; %entry
1004 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1005 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1006 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1007 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, 0
1008 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1009 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
1010 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[6:7]
1011 ; GFX9-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
1012 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1013 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v4
1014 ; GFX9-DL-NEXT: global_store_byte v1, v0, s[2:3]
1015 ; GFX9-DL-NEXT: s_endpgm
1016 ;
1017 ; GFX10-DL-LABEL: udot4_CommutationAccrossMADs:
1018 ; GFX10-DL: ; %bb.0: ; %entry
1019 ; GFX10-DL-NEXT: s_clause 0x1
1020 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1021 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1022 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1023 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
1024 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1025 ; GFX10-DL-NEXT: s_clause 0x1
1026 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
1027 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[6:7]
1028 ; GFX10-DL-NEXT: global_load_ubyte v4, v1, s[2:3]
1029 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1030 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v3, v2, v4
1031 ; GFX10-DL-NEXT: global_store_byte v1, v0, s[2:3]
1032 ; GFX10-DL-NEXT: s_endpgm
1033 ;
1034 ; GFX11-DL-LABEL: udot4_CommutationAccrossMADs:
1035 ; GFX11-DL: ; %bb.0: ; %entry
1036 ; GFX11-DL-NEXT: s_clause 0x1
1037 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1038 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1039 ; GFX11-DL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
1040 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1041 ; GFX11-DL-NEXT: s_clause 0x1
1042 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]
1043 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
1044 ; GFX11-DL-NEXT: global_load_u8 v3, v1, s[0:1]
1045 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
1046 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v3
1047 ; GFX11-DL-NEXT: global_store_b8 v1, v0, s[0:1]
1048 ; GFX11-DL-NEXT: s_nop 0
1049 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1050 ; GFX11-DL-NEXT: s_endpgm
1051 ptr addrspace(1) %src2,
1052 ptr addrspace(1) nocapture %dst) {
1053 entry:
1054 %idx = call i32 @llvm.amdgcn.workitem.id.x()
1055 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
1056 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
1057 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
1058 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
1060 %v1e0 = extractelement <4 x i8> %vec1, i64 0
1061 %v2e0 = extractelement <4 x i8> %vec2, i64 0
1062 %mul1 = mul nuw nsw i8 %v2e0, %v1e0
1064 %v1e1 = extractelement <4 x i8> %vec1, i64 1
1065 %v2e1 = extractelement <4 x i8> %vec2, i64 1
1066 %mul2 = mul nuw nsw i8 %v2e1, %v1e1
1068 %v1e2 = extractelement <4 x i8> %vec1, i64 2
1069 %v2e2 = extractelement <4 x i8> %vec2, i64 2
1070 %mul3 = mul nuw nsw i8 %v2e2, %v1e2
1072 %v1e3 = extractelement <4 x i8> %vec1, i64 3
1073 %v2e3 = extractelement <4 x i8> %vec2, i64 3
1074 %mul4 = mul nuw nsw i8 %v2e3, %v1e3
1076 %acc = load i8, ptr addrspace(1) %dst, align 2
1077 %mad1 = add i8 %acc, %mul2
1078 %mad2 = add i8 %mad1, %mul1
1079 %mad3 = add i8 %mad2, %mul3
1080 %mad4 = add i8 %mad3, %mul4
1082 store i8 %mad4, ptr addrspace(1) %dst, align 2
1083 ret void
1084 }
1086 define amdgpu_kernel void @udot4_multiuse_mul1(ptr addrspace(1) %src1,
1087 ; GFX7-LABEL: udot4_multiuse_mul1:
1088 ; GFX7: ; %bb.0: ; %entry
1089 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1090 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
1091 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
1092 ; GFX7-NEXT: s_mov_b32 s10, 0
1093 ; GFX7-NEXT: s_mov_b32 s11, s3
1094 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1095 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
1096 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1097 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
1098 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
1099 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
1100 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
1101 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
1102 ; GFX7-NEXT: s_mov_b32 s2, -1
1103 ; GFX7-NEXT: s_waitcnt vmcnt(1)
1104 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
1105 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
1106 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1107 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v0
1108 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
1109 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1110 ; GFX7-NEXT: v_mad_u32_u24 v8, v1, v5, s4
1111 ; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, v8
1112 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
1113 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
1114 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
1115 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
1116 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
1117 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
1118 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
1119 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
1120 ; GFX7-NEXT: s_endpgm
1121 ;
1122 ; GFX8-LABEL: udot4_multiuse_mul1:
1123 ; GFX8: ; %bb.0: ; %entry
1124 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1125 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1126 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
1127 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1128 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1129 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
1130 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1131 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1132 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1133 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
1134 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1135 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
1136 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
1137 ; GFX8-NEXT: s_waitcnt vmcnt(1)
1138 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
1139 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
1140 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
1141 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
1142 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1143 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v0
1144 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
1145 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1146 ; GFX8-NEXT: v_mad_u32_u24 v8, v1, v2, s2
1147 ; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v8
1148 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
1149 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v4
1150 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
1151 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
1152 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
1153 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1154 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1155 ; GFX8-NEXT: flat_store_dword v[0:1], v2
1156 ; GFX8-NEXT: s_endpgm
1157 ;
1158 ; GFX9-NODL-LABEL: udot4_multiuse_mul1:
1159 ; GFX9-NODL: ; %bb.0: ; %entry
1160 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1161 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1162 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1163 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1164 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
1165 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
1166 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
1167 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
1168 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
1169 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v1
1170 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
1171 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v2
1172 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
1173 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
1174 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
1175 ; GFX9-NODL-NEXT: v_mul_u32_u24_e32 v2, v3, v4
1176 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1177 ; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v3, v4, s0
1178 ; GFX9-NODL-NEXT: v_add3_u32 v2, v5, v3, v2
1179 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
1180 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
1181 ; GFX9-NODL-NEXT: s_endpgm
1182 ;
1183 ; GFX9-DL-LABEL: udot4_multiuse_mul1:
1184 ; GFX9-DL: ; %bb.0: ; %entry
1185 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1186 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1187 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1188 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1189 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1190 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1191 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
1192 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1193 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
1194 ; GFX9-DL-NEXT: v_and_b32_e32 v3, 0xff, v1
1195 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1196 ; GFX9-DL-NEXT: v_and_b32_e32 v4, 0xff, v2
1197 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1198 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, v3, v4, s0
1199 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
1200 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
1201 ; GFX9-DL-NEXT: s_endpgm
1202 ;
1203 ; GFX10-DL-LABEL: udot4_multiuse_mul1:
1204 ; GFX10-DL: ; %bb.0: ; %entry
1205 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1206 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1207 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1208 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1209 ; GFX10-DL-NEXT: s_clause 0x1
1210 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
1211 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
1212 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
1213 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
1214 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xff, v1
1215 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1216 ; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xff, v2
1217 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1218 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v3, s2
1219 ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
1220 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v2, v0
1221 ; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
1222 ; GFX10-DL-NEXT: s_endpgm
1223 ;
1224 ; GFX11-DL-LABEL: udot4_multiuse_mul1:
1225 ; GFX11-DL: ; %bb.0: ; %entry
1226 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1227 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1228 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1229 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1230 ; GFX11-DL-NEXT: s_clause 0x1
1231 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
1232 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
1233 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
1234 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
1235 ; GFX11-DL-NEXT: v_and_b32_e32 v2, 0xff, v1
1236 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
1237 ; GFX11-DL-NEXT: v_and_b32_e32 v3, 0xff, v0
1238 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1239 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
1240 ; GFX11-DL-NEXT: v_mad_u32_u24 v2, v2, v3, s2
1241 ; GFX11-DL-NEXT: v_mov_b32_e32 v3, 0
1242 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v2
1243 ; GFX11-DL-NEXT: global_store_b32 v3, v0, s[0:1]
1244 ; GFX11-DL-NEXT: s_nop 0
1245 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1246 ; GFX11-DL-NEXT: s_endpgm
1247 ptr addrspace(1) %src2,
1248 ptr addrspace(1) nocapture %dst) {
1249 entry:
1250 %idx = call i32 @llvm.amdgcn.workitem.id.x()
1251 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
1252 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
1253 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
1254 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
1256 %v1e0 = extractelement <4 x i8> %vec1, i64 0
1257 %cv1e0 = zext i8 %v1e0 to i32
1258 %v2e0 = extractelement <4 x i8> %vec2, i64 0
1259 %cv2e0 = zext i8 %v2e0 to i32
1260 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
1262 %v1e1 = extractelement <4 x i8> %vec1, i64 1
1263 %cv1e1 = zext i8 %v1e1 to i32
1264 %v2e1 = extractelement <4 x i8> %vec2, i64 1
1265 %cv2e1 = zext i8 %v2e1 to i32
1266 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
1268 %v1e2 = extractelement <4 x i8> %vec1, i64 2
1269 %cv1e2 = zext i8 %v1e2 to i32
1270 %v2e2 = extractelement <4 x i8> %vec2, i64 2
1271 %cv2e2 = zext i8 %v2e2 to i32
1272 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
1274 %v1e3 = extractelement <4 x i8> %vec1, i64 3
1275 %cv1e3 = zext i8 %v1e3 to i32
1276 %v2e3 = extractelement <4 x i8> %vec2, i64 3
1277 %cv2e3 = zext i8 %v2e3 to i32
1278 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
1280 %acc = load i32, ptr addrspace(1) %dst, align 4
1281 %add = add i32 %mul1, %acc
1282 %add1 = add i32 %mul2, %add
1283 %add2 = add i32 %add1, %mul1
1284 %add3 = add i32 %add2, %mul3
1285 %add4 = add i32 %add3, %mul4
1287 store i32 %add4, ptr addrspace(1) %dst, align 4
1288 ret void
1289 }
1291 define amdgpu_kernel void @udot4_multiuse_add1(ptr addrspace(1) %src1,
1292 ; GFX7-LABEL: udot4_multiuse_add1:
1293 ; GFX7: ; %bb.0: ; %entry
1294 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1295 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
1296 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
1297 ; GFX7-NEXT: s_mov_b32 s10, 0
1298 ; GFX7-NEXT: s_mov_b32 s11, s3
1299 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1300 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
1301 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1302 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
1303 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
1304 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
1305 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
1306 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
1307 ; GFX7-NEXT: s_mov_b32 s2, -1
1308 ; GFX7-NEXT: s_waitcnt vmcnt(1)
1309 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
1310 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
1311 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1312 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
1313 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v0
1314 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1315 ; GFX7-NEXT: v_mad_u32_u24 v3, v3, v6, s4
1316 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
1317 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
1318 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
1319 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
1320 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
1321 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
1322 ; GFX7-NEXT: v_add_i32_e32 v6, vcc, s4, v3
1323 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
1324 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v6
1325 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
1326 ; GFX7-NEXT: s_endpgm
1327 ;
1328 ; GFX8-LABEL: udot4_multiuse_add1:
1329 ; GFX8: ; %bb.0: ; %entry
1330 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1331 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1332 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
1333 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1334 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1335 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
1336 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1337 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1338 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1339 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
1340 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1341 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
1342 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
1343 ; GFX8-NEXT: s_waitcnt vmcnt(1)
1344 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
1345 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
1346 ; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 8
1347 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
1348 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1349 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
1350 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v0
1351 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1352 ; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, s2
1353 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
1354 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, v4
1355 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
1356 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v7, v1
1357 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, s2, v4
1358 ; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, v1
1359 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v0, v5
1360 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1361 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1362 ; GFX8-NEXT: flat_store_dword v[0:1], v2
1363 ; GFX8-NEXT: s_endpgm
1364 ;
1365 ; GFX9-NODL-LABEL: udot4_multiuse_add1:
1366 ; GFX9-NODL: ; %bb.0: ; %entry
1367 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1368 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1369 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1370 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1371 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
1372 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
1373 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
1374 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
1375 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
1376 ; GFX9-NODL-NEXT: v_bfe_u32 v4, v1, 8, 8
1377 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
1378 ; GFX9-NODL-NEXT: v_bfe_u32 v5, v2, 8, 8
1379 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
1380 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
1381 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
1382 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1383 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v4, v5, s0
1384 ; GFX9-NODL-NEXT: v_add_u32_e32 v4, s0, v2
1385 ; GFX9-NODL-NEXT: v_add3_u32 v2, v2, v3, v6
1386 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v1, v4
1387 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
1388 ; GFX9-NODL-NEXT: s_endpgm
1389 ;
1390 ; GFX9-DL-LABEL: udot4_multiuse_add1:
1391 ; GFX9-DL: ; %bb.0: ; %entry
1392 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1393 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1394 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1395 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1396 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1397 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1398 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
1399 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1400 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1401 ; GFX9-DL-NEXT: s_add_i32 s1, s0, s0
1402 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1403 ; GFX9-DL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
1404 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s0
1405 ; GFX9-DL-NEXT: v_add3_u32 v1, s1, v3, v1
1406 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
1407 ; GFX9-DL-NEXT: s_endpgm
1408 ;
1409 ; GFX10-DL-LABEL: udot4_multiuse_add1:
1410 ; GFX10-DL: ; %bb.0: ; %entry
1411 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1412 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1413 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1414 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1415 ; GFX10-DL-NEXT: s_clause 0x1
1416 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
1417 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
1418 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
1419 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1420 ; GFX10-DL-NEXT: v_mul_u32_u24_sdwa v0, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
1421 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1422 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s2
1423 ; GFX10-DL-NEXT: s_add_i32 s2, s2, s2
1424 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
1425 ; GFX10-DL-NEXT: v_add3_u32 v0, s2, v0, v1
1426 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
1427 ; GFX10-DL-NEXT: s_endpgm
1428 ;
1429 ; GFX11-DL-LABEL: udot4_multiuse_add1:
1430 ; GFX11-DL: ; %bb.0: ; %entry
1431 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1432 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1433 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1434 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1435 ; GFX11-DL-NEXT: s_clause 0x1
1436 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
1437 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
1438 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
1439 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
1440 ; GFX11-DL-NEXT: v_bfe_u32 v2, v1, 8, 8
1441 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
1442 ; GFX11-DL-NEXT: v_bfe_u32 v3, v0, 8, 8
1443 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1444 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
1445 ; GFX11-DL-NEXT: v_mov_b32_e32 v1, 0
1446 ; GFX11-DL-NEXT: s_add_i32 s2, s2, s2
1447 ; GFX11-DL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
1448 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
1449 ; GFX11-DL-NEXT: v_add3_u32 v0, s2, v2, v0
1450 ; GFX11-DL-NEXT: global_store_b32 v1, v0, s[0:1]
1451 ; GFX11-DL-NEXT: s_nop 0
1452 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1453 ; GFX11-DL-NEXT: s_endpgm
1454 ptr addrspace(1) %src2,
1455 ptr addrspace(1) nocapture %dst) {
1456 entry:
1457 %idx = call i32 @llvm.amdgcn.workitem.id.x()
1458 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
1459 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
1460 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
1461 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
1463 %v1e0 = extractelement <4 x i8> %vec1, i64 0
1464 %cv1e0 = zext i8 %v1e0 to i32
1465 %v2e0 = extractelement <4 x i8> %vec2, i64 0
1466 %cv2e0 = zext i8 %v2e0 to i32
1467 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
1469 %v1e1 = extractelement <4 x i8> %vec1, i64 1
1470 %cv1e1 = zext i8 %v1e1 to i32
1471 %v2e1 = extractelement <4 x i8> %vec2, i64 1
1472 %cv2e1 = zext i8 %v2e1 to i32
1473 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
1475 %v1e2 = extractelement <4 x i8> %vec1, i64 2
1476 %cv1e2 = zext i8 %v1e2 to i32
1477 %v2e2 = extractelement <4 x i8> %vec2, i64 2
1478 %cv2e2 = zext i8 %v2e2 to i32
1479 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
1481 %v1e3 = extractelement <4 x i8> %vec1, i64 3
1482 %cv1e3 = zext i8 %v1e3 to i32
1483 %v2e3 = extractelement <4 x i8> %vec2, i64 3
1484 %cv2e3 = zext i8 %v2e3 to i32
1485 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
1487 %acc = load i32, ptr addrspace(1) %dst, align 4
1488 %add1 = add i32 %mul2, %acc
1489 %add = add i32 %add1, %acc
1490 %add2 = add i32 %add1, %mul1
1491 %add3 = add i32 %add2, %mul3
1492 %add4 = add i32 %add3, %mul4
1493 %res = add i32 %add4, %add
1494 store i32 %res, ptr addrspace(1) %dst, align 4
1495 ret void
1496 }
1498 define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
1499 ; GFX7-LABEL: notdot4_mixedtypes:
1500 ; GFX7: ; %bb.0: ; %entry
1501 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1502 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
1503 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
1504 ; GFX7-NEXT: s_mov_b32 s10, 0
1505 ; GFX7-NEXT: s_mov_b32 s11, s3
1506 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1507 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
1508 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1509 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
1510 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
1511 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
1512 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
1513 ; GFX7-NEXT: s_mov_b32 s2, -1
1514 ; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0
1515 ; GFX7-NEXT: s_waitcnt vmcnt(2)
1516 ; GFX7-NEXT: v_bfe_i32 v3, v2, 0, 8
1517 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
1518 ; GFX7-NEXT: s_waitcnt vmcnt(1)
1519 ; GFX7-NEXT: v_bfe_i32 v6, v0, 0, 8
1520 ; GFX7-NEXT: v_bfe_u32 v7, v0, 8, 8
1521 ; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
1522 ; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
1523 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1524 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
1525 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
1526 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
1527 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v6, v1
1528 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
1529 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
1530 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
1531 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
1532 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
1533 ; GFX7-NEXT: s_endpgm
1534 ;
1535 ; GFX8-LABEL: notdot4_mixedtypes:
1536 ; GFX8: ; %bb.0: ; %entry
1537 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1538 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1539 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
1540 ; GFX8-NEXT: v_mov_b32_e32 v5, 0xff
1541 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1542 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1543 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
1544 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1545 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1546 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1547 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
1548 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1549 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
1550 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1551 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1552 ; GFX8-NEXT: flat_load_ushort v4, v[0:1]
1553 ; GFX8-NEXT: s_waitcnt vmcnt(2)
1554 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v3
1555 ; GFX8-NEXT: v_and_b32_e32 v8, 0xff, v8
1556 ; GFX8-NEXT: v_bfe_i32 v6, v3, 0, 8
1557 ; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
1558 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
1559 ; GFX8-NEXT: s_waitcnt vmcnt(1)
1560 ; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v2
1561 ; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v9
1562 ; GFX8-NEXT: v_bfe_i32 v7, v2, 0, 8
1563 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1564 ; GFX8-NEXT: v_mad_u16 v4, v8, v9, v4
1565 ; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
1566 ; GFX8-NEXT: v_mad_u16 v4, v6, v7, v4
1567 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
1568 ; GFX8-NEXT: v_mad_u16 v4, v10, v5, v4
1569 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
1570 ; GFX8-NEXT: flat_store_short v[0:1], v2
1571 ; GFX8-NEXT: s_endpgm
1572 ;
1573 ; GFX9-NODL-LABEL: notdot4_mixedtypes:
1574 ; GFX9-NODL: ; %bb.0: ; %entry
1575 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1576 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1577 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1578 ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
1579 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1580 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
1581 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
1582 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
1583 ; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
1584 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
1585 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
1586 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
1587 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
1588 ; GFX9-NODL-NEXT: v_and_b32_e32 v6, 0xff, v6
1589 ; GFX9-NODL-NEXT: v_and_b32_e32 v7, 0xff, v7
1590 ; GFX9-NODL-NEXT: v_bfe_i32 v4, v1, 0, 8
1591 ; GFX9-NODL-NEXT: v_bfe_i32 v5, v2, 0, 8
1592 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
1593 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v6, v7, v3
1594 ; GFX9-NODL-NEXT: v_and_b32_sdwa v8, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
1595 ; GFX9-NODL-NEXT: v_and_b32_sdwa v9, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
1596 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3
1597 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
1598 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v2, 24, v2
1599 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v3, v8, v9, v3
1600 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
1601 ; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
1602 ; GFX9-NODL-NEXT: s_endpgm
1603 ;
1604 ; GFX9-DL-LABEL: notdot4_mixedtypes:
1605 ; GFX9-DL: ; %bb.0: ; %entry
1606 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1607 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1608 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1609 ; GFX9-DL-NEXT: s_mov_b32 s0, 0xc0c0302
1610 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1611 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1612 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1613 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1614 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
1615 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
1616 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v1
1617 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
1618 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v2
1619 ; GFX9-DL-NEXT: v_and_b32_e32 v6, 0xff, v6
1620 ; GFX9-DL-NEXT: v_and_b32_e32 v7, 0xff, v7
1621 ; GFX9-DL-NEXT: v_bfe_i32 v4, v1, 0, 8
1622 ; GFX9-DL-NEXT: v_bfe_i32 v5, v2, 0, 8
1623 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
1624 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v6, v7, v3
1625 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s0
1626 ; GFX9-DL-NEXT: v_mad_legacy_u16 v3, v4, v5, v3
1627 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s0
1628 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
1629 ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
1630 ; GFX9-DL-NEXT: s_endpgm
1631 ;
1632 ; GFX10-DL-LABEL: notdot4_mixedtypes:
1633 ; GFX10-DL: ; %bb.0: ; %entry
1634 ; GFX10-DL-NEXT: s_clause 0x1
1635 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1636 ; GFX10-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1637 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1638 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1639 ; GFX10-DL-NEXT: s_clause 0x1
1640 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
1641 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
1642 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
1643 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[2:3]
1644 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
1645 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
1646 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
1647 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v2
1648 ; GFX10-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
1649 ; GFX10-DL-NEXT: v_bfe_i32 v7, v2, 0, 8
1650 ; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0302
1651 ; GFX10-DL-NEXT: v_and_b32_e32 v4, 0xff, v4
1652 ; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xff, v5
1653 ; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
1654 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
1655 ; GFX10-DL-NEXT: v_mad_u16 v3, v4, v5, v3
1656 ; GFX10-DL-NEXT: v_mad_u16 v3, v6, v7, v3
1657 ; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v3
1658 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
1659 ; GFX10-DL-NEXT: global_store_short v0, v1, s[2:3]
1660 ; GFX10-DL-NEXT: s_endpgm
1661 ;
1662 ; GFX11-DL-LABEL: notdot4_mixedtypes:
1663 ; GFX11-DL: ; %bb.0: ; %entry
1664 ; GFX11-DL-NEXT: s_clause 0x1
1665 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1666 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1667 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1668 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1669 ; GFX11-DL-NEXT: s_clause 0x1
1670 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
1671 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
1672 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
1673 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v1
1674 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
1675 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v0
1676 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
1677 ; GFX11-DL-NEXT: v_bfe_i32 v6, v1, 0, 8
1678 ; GFX11-DL-NEXT: v_bfe_i32 v7, v0, 0, 8
1679 ; GFX11-DL-NEXT: v_and_b32_e32 v4, 0xff, v4
1680 ; GFX11-DL-NEXT: v_and_b32_e32 v5, 0xff, v5
1681 ; GFX11-DL-NEXT: global_load_u16 v3, v2, s[0:1]
1682 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0302
1683 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
1684 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
1685 ; GFX11-DL-NEXT: v_mad_u16 v3, v4, v5, v3
1686 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
1687 ; GFX11-DL-NEXT: v_mad_u16 v3, v6, v7, v3
1688 ; GFX11-DL-NEXT: v_and_b32_e32 v3, 0xffff, v3
1689 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
1690 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v3
1691 ; GFX11-DL-NEXT: global_store_b16 v2, v0, s[0:1]
1692 ; GFX11-DL-NEXT: s_nop 0
1693 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1694 ; GFX11-DL-NEXT: s_endpgm
1695 ptr addrspace(1) %src2,
1696 ptr addrspace(1) nocapture %dst) {
1697 entry:
1698 %idx = call i32 @llvm.amdgcn.workitem.id.x()
1699 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
1700 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
1701 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
1702 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
1704 %v1e0 = extractelement <4 x i8> %vec1, i64 0
1705 %cv1e0 = sext i8 %v1e0 to i16
1706 %v2e0 = extractelement <4 x i8> %vec2, i64 0
1707 %cv2e0 = sext i8 %v2e0 to i16
1708 %mul1 = mul nuw nsw i16 %cv1e0, %cv2e0
1710 %v1e1 = extractelement <4 x i8> %vec1, i64 1
1711 %cv1e1 = zext i8 %v1e1 to i16
1712 %v2e1 = extractelement <4 x i8> %vec2, i64 1
1713 %cv2e1 = zext i8 %v2e1 to i16
1714 %mul2 = mul nuw nsw i16 %cv1e1, %cv2e1
1716 %v1e2 = extractelement <4 x i8> %vec1, i64 2
1717 %cv1e2 = zext i8 %v1e2 to i16
1718 %v2e2 = extractelement <4 x i8> %vec2, i64 2
1719 %cv2e2 = zext i8 %v2e2 to i16
1720 %mul3 = mul nuw nsw i16 %cv1e2, %cv2e2
1722 %v1e3 = extractelement <4 x i8> %vec1, i64 3
1723 %cv1e3 = zext i8 %v1e3 to i16
1724 %v2e3 = extractelement <4 x i8> %vec2, i64 3
1725 %cv2e3 = zext i8 %v2e3 to i16
1726 %mul4 = mul nuw nsw i16 %cv1e3, %cv2e3
1728 %acc = load i16, ptr addrspace(1) %dst, align 2
1729 %add1 = add i16 %mul2, %acc
1730 %add2 = add i16 %add1, %mul1
1731 %add3 = add i16 %add2, %mul3
1732 %add4 = add i16 %add3, %mul4
1734 store i16 %add4, ptr addrspace(1) %dst, align 2
1735 ret void
1736 }
1738 ; TODO: cleanup s_lshr_b32
1739 define amdgpu_kernel void @udot4_acc32_vecMul(ptr addrspace(1) %src1,
1740 ; GFX7-LABEL: udot4_acc32_vecMul:
1741 ; GFX7: ; %bb.0: ; %entry
1742 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1743 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
1744 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
1745 ; GFX7-NEXT: s_mov_b32 s10, 0
1746 ; GFX7-NEXT: s_mov_b32 s11, s3
1747 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1748 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
1749 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1750 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
1751 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
1752 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
1753 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
1754 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
1755 ; GFX7-NEXT: s_mov_b32 s2, -1
1756 ; GFX7-NEXT: s_waitcnt vmcnt(1)
1757 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
1758 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
1759 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
1760 ; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
1761 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1762 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v0
1763 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
1764 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
1765 ; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
1766 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1767 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, s4
1768 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v0
1769 ; GFX7-NEXT: v_mad_u32_u24 v0, v4, v7, v0
1770 ; GFX7-NEXT: v_mad_u32_u24 v0, v1, v5, v0
1771 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
1772 ; GFX7-NEXT: s_endpgm
1773 ;
1774 ; GFX8-LABEL: udot4_acc32_vecMul:
1775 ; GFX8: ; %bb.0: ; %entry
1776 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1777 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1778 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
1779 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1780 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1781 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
1782 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1783 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1784 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1785 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
1786 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1787 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
1788 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
1789 ; GFX8-NEXT: s_waitcnt vmcnt(1)
1790 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v3
1791 ; GFX8-NEXT: v_bfe_u32 v4, v3, 16, 8
1792 ; GFX8-NEXT: v_lshrrev_b16_e32 v5, 8, v3
1793 ; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v3
1794 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1795 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v0
1796 ; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 8
1797 ; GFX8-NEXT: v_lshrrev_b16_e32 v7, 8, v0
1798 ; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
1799 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1800 ; GFX8-NEXT: v_mad_u32_u24 v0, v3, v0, s2
1801 ; GFX8-NEXT: v_mad_u32_u24 v0, v5, v7, v0
1802 ; GFX8-NEXT: v_mad_u32_u24 v0, v4, v6, v0
1803 ; GFX8-NEXT: v_mad_u32_u24 v2, v1, v2, v0
1804 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1805 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1806 ; GFX8-NEXT: flat_store_dword v[0:1], v2
1807 ; GFX8-NEXT: s_endpgm
1808 ;
1809 ; GFX9-NODL-LABEL: udot4_acc32_vecMul:
1810 ; GFX9-NODL: ; %bb.0: ; %entry
1811 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1812 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1813 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1814 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1815 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
1816 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
1817 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
1818 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
1819 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
1820 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
1821 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
1822 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
1823 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
1824 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1825 ; GFX9-NODL-NEXT: v_add3_u32 v2, v3, s0, v4
1826 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
1827 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
1828 ; GFX9-NODL-NEXT: s_endpgm
1829 ;
1830 ; GFX9-DL-LABEL: udot4_acc32_vecMul:
1831 ; GFX9-DL: ; %bb.0: ; %entry
1832 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1833 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1834 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1835 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
1836 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
1837 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
1838 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
1839 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
1840 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1841 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s0
1842 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
1843 ; GFX9-DL-NEXT: s_endpgm
1844 ;
1845 ; GFX10-DL-LABEL: udot4_acc32_vecMul:
1846 ; GFX10-DL: ; %bb.0: ; %entry
1847 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1848 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1849 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1850 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
1851 ; GFX10-DL-NEXT: s_clause 0x1
1852 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
1853 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
1854 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
1855 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
1856 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1857 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, s2
1858 ; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
1859 ; GFX10-DL-NEXT: s_endpgm
1860 ;
1861 ; GFX11-DL-LABEL: udot4_acc32_vecMul:
1862 ; GFX11-DL: ; %bb.0: ; %entry
1863 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
1864 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1865 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
1866 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
1867 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
1868 ; GFX11-DL-NEXT: s_clause 0x1
1869 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
1870 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
1871 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
1872 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
1873 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
1874 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
1875 ; GFX11-DL-NEXT: s_nop 0
1876 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
1877 ; GFX11-DL-NEXT: s_endpgm
1878 ptr addrspace(1) %src2,
1879 ptr addrspace(1) nocapture %dst) {
1880 entry:
1881 %idx = call i32 @llvm.amdgcn.workitem.id.x()
1882 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
1883 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
1884 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
1885 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
1887 %cvec1 = zext <4 x i8> %vec1 to <4 x i32>
1888 %cvec2 = zext <4 x i8> %vec2 to <4 x i32>
1890 %mul = mul <4 x i32> %cvec1, %cvec2
1891 %mul0 = extractelement <4 x i32> %mul, i64 0
1892 %mul1 = extractelement <4 x i32> %mul, i64 1
1893 %mul2 = extractelement <4 x i32> %mul, i64 2
1894 %mul3 = extractelement <4 x i32> %mul, i64 3
1896 %acc = load i32, ptr addrspace(1) %dst, align 4
1897 %add1 = add i32 %mul0, %acc
1898 %add2 = add i32 %add1, %mul1
1899 %add3 = add i32 %add2, %mul2
1900 %add4 = add i32 %add3, %mul3
1902 store i32 %add4, ptr addrspace(1) %dst, align 4
1903 ret void
1904 }
1906 ; TODO: This pattern should be recognized.
1907 define amdgpu_kernel void @udot4_acc16_vecMul(ptr addrspace(1) %src1,
1908 ; GFX7-LABEL: udot4_acc16_vecMul:
1909 ; GFX7: ; %bb.0: ; %entry
1910 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
1911 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
1912 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
1913 ; GFX7-NEXT: s_mov_b32 s10, 0
1914 ; GFX7-NEXT: s_mov_b32 s11, s3
1915 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
1916 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
1917 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1918 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
1919 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
1920 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
1921 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
1922 ; GFX7-NEXT: s_mov_b32 s2, -1
1923 ; GFX7-NEXT: buffer_load_ushort v1, off, s[0:3], 0
1924 ; GFX7-NEXT: s_waitcnt vmcnt(2)
1925 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v2
1926 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v2
1927 ; GFX7-NEXT: s_waitcnt vmcnt(1)
1928 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v0
1929 ; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v0
1930 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
1931 ; GFX7-NEXT: v_alignbit_b32 v2, v3, v2, 16
1932 ; GFX7-NEXT: v_bfe_u32 v3, v0, 8, 8
1933 ; GFX7-NEXT: v_alignbit_b32 v0, v6, v0, 16
1934 ; GFX7-NEXT: s_waitcnt vmcnt(0)
1935 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v7, v1
1936 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v2
1937 ; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
1938 ; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v0
1939 ; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
1940 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v3, v1
1941 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
1942 ; GFX7-NEXT: v_mad_u32_u24 v0, v6, v5, v0
1943 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0
1944 ; GFX7-NEXT: s_endpgm
1945 ;
1946 ; GFX8-LABEL: udot4_acc16_vecMul:
1947 ; GFX8: ; %bb.0: ; %entry
1948 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1949 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
1950 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
1951 ; GFX8-NEXT: v_mov_b32_e32 v5, 0xff
1952 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
1953 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
1954 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
1955 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1956 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
1957 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
1958 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
1959 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
1960 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
1961 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
1962 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
1963 ; GFX8-NEXT: flat_load_ushort v4, v[0:1]
1964 ; GFX8-NEXT: s_waitcnt vmcnt(2)
1965 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 24, v3
1966 ; GFX8-NEXT: v_lshrrev_b16_e32 v7, 8, v3
1967 ; GFX8-NEXT: v_and_b32_sdwa v10, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
1968 ; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v3
1969 ; GFX8-NEXT: s_waitcnt vmcnt(1)
1970 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v2
1971 ; GFX8-NEXT: v_lshrrev_b16_e32 v9, 8, v2
1972 ; GFX8-NEXT: v_and_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
1973 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v2
1974 ; GFX8-NEXT: s_waitcnt vmcnt(0)
1975 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
1976 ; GFX8-NEXT: v_mad_u16 v2, v7, v9, v2
1977 ; GFX8-NEXT: v_mad_u16 v2, v10, v5, v2
1978 ; GFX8-NEXT: v_mad_u16 v2, v6, v8, v2
1979 ; GFX8-NEXT: flat_store_short v[0:1], v2
1980 ; GFX8-NEXT: s_endpgm
1981 ;
1982 ; GFX9-NODL-LABEL: udot4_acc16_vecMul:
1983 ; GFX9-NODL: ; %bb.0: ; %entry
1984 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
1985 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
1986 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
1987 ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff
1988 ; GFX9-NODL-NEXT: s_mov_b32 s1, 0x5040100
1989 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
1990 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
1991 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
1992 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
1993 ; GFX9-NODL-NEXT: global_load_ushort v3, v0, s[2:3]
1994 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
1995 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
1996 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
1997 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
1998 ; GFX9-NODL-NEXT: v_lshrrev_b16_e32 v6, 8, v2
1999 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
2000 ; GFX9-NODL-NEXT: v_and_b32_e32 v8, 0xff, v1
2001 ; GFX9-NODL-NEXT: v_and_b32_sdwa v1, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2002 ; GFX9-NODL-NEXT: v_and_b32_e32 v9, 0xff, v2
2003 ; GFX9-NODL-NEXT: v_and_b32_sdwa v2, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2004 ; GFX9-NODL-NEXT: v_perm_b32 v2, v7, v2, s1
2005 ; GFX9-NODL-NEXT: v_perm_b32 v1, v5, v1, s1
2006 ; GFX9-NODL-NEXT: v_perm_b32 v5, v6, v9, s1
2007 ; GFX9-NODL-NEXT: v_perm_b32 v4, v4, v8, s1
2008 ; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
2009 ; GFX9-NODL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
2010 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
2011 ; GFX9-NODL-NEXT: v_add_u16_e32 v3, v2, v3
2012 ; GFX9-NODL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2013 ; GFX9-NODL-NEXT: v_add_u16_e32 v2, v2, v1
2014 ; GFX9-NODL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2015 ; GFX9-NODL-NEXT: global_store_short v0, v1, s[2:3]
2016 ; GFX9-NODL-NEXT: s_endpgm
2017 ;
2018 ; GFX9-DL-LABEL: udot4_acc16_vecMul:
2019 ; GFX9-DL: ; %bb.0: ; %entry
2020 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2021 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2022 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2023 ; GFX9-DL-NEXT: s_movk_i32 s0, 0xff
2024 ; GFX9-DL-NEXT: s_mov_b32 s1, 0x5040100
2025 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2026 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
2027 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
2028 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2029 ; GFX9-DL-NEXT: global_load_ushort v3, v0, s[2:3]
2030 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
2031 ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v4, 8, v1
2032 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
2033 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2034 ; GFX9-DL-NEXT: v_lshrrev_b16_e32 v6, 8, v2
2035 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
2036 ; GFX9-DL-NEXT: v_and_b32_e32 v8, 0xff, v1
2037 ; GFX9-DL-NEXT: v_and_b32_sdwa v1, v1, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2038 ; GFX9-DL-NEXT: v_and_b32_e32 v9, 0xff, v2
2039 ; GFX9-DL-NEXT: v_and_b32_sdwa v2, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2040 ; GFX9-DL-NEXT: v_perm_b32 v2, v7, v2, s1
2041 ; GFX9-DL-NEXT: v_perm_b32 v1, v5, v1, s1
2042 ; GFX9-DL-NEXT: v_perm_b32 v5, v6, v9, s1
2043 ; GFX9-DL-NEXT: v_perm_b32 v4, v4, v8, s1
2044 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
2045 ; GFX9-DL-NEXT: v_pk_mul_lo_u16 v2, v4, v5
2046 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2047 ; GFX9-DL-NEXT: v_add_u16_e32 v3, v2, v3
2048 ; GFX9-DL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2049 ; GFX9-DL-NEXT: v_add_u16_e32 v2, v2, v1
2050 ; GFX9-DL-NEXT: v_add_u16_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
2051 ; GFX9-DL-NEXT: global_store_short v0, v1, s[2:3]
2052 ; GFX9-DL-NEXT: s_endpgm
2053 ;
2054 ; GFX10-DL-LABEL: udot4_acc16_vecMul:
2055 ; GFX10-DL: ; %bb.0: ; %entry
2056 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2057 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2058 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2059 ; GFX10-DL-NEXT: v_mov_b32_e32 v8, 0xff
2060 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2061 ; GFX10-DL-NEXT: s_clause 0x1
2062 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
2063 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
2064 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
2065 ; GFX10-DL-NEXT: global_load_ushort v3, v0, s[0:1]
2066 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
2067 ; GFX10-DL-NEXT: v_lshrrev_b16 v4, 8, v1
2068 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2069 ; GFX10-DL-NEXT: v_lshrrev_b16 v5, 8, v2
2070 ; GFX10-DL-NEXT: v_and_b32_e32 v6, 0xff, v2
2071 ; GFX10-DL-NEXT: v_and_b32_e32 v7, 0xff, v1
2072 ; GFX10-DL-NEXT: v_perm_b32 v5, v5, v6, 0x5040100
2073 ; GFX10-DL-NEXT: v_perm_b32 v4, v4, v7, 0x5040100
2074 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v1
2075 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 24, v2
2076 ; GFX10-DL-NEXT: v_and_b32_sdwa v2, v2, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2077 ; GFX10-DL-NEXT: v_and_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
2078 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v5
2079 ; GFX10-DL-NEXT: v_perm_b32 v2, v7, v2, 0x5040100
2080 ; GFX10-DL-NEXT: v_perm_b32 v1, v6, v1, 0x5040100
2081 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
2082 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2083 ; GFX10-DL-NEXT: v_add_nc_u16 v3, v4, v3
2084 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v1, v1, v2
2085 ; GFX10-DL-NEXT: v_add_nc_u16 v2, v3, v5
2086 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
2087 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v2, v1
2088 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v3
2089 ; GFX10-DL-NEXT: global_store_short v0, v1, s[0:1]
2090 ; GFX10-DL-NEXT: s_endpgm
2091 ;
2092 ; GFX11-DL-LABEL: udot4_acc16_vecMul:
2093 ; GFX11-DL: ; %bb.0: ; %entry
2094 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
2095 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2096 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
2097 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2098 ; GFX11-DL-NEXT: s_clause 0x1
2099 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
2100 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
2101 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
2102 ; GFX11-DL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v7, 0xff, v1
2103 ; GFX11-DL-NEXT: global_load_u16 v3, v2, s[0:1]
2104 ; GFX11-DL-NEXT: v_lshrrev_b16 v4, 8, v1
2105 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
2106 ; GFX11-DL-NEXT: v_lshrrev_b16 v5, 8, v0
2107 ; GFX11-DL-NEXT: v_and_b32_e32 v6, 0xff, v0
2108 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v1
2109 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v9, 16, v0
2110 ; GFX11-DL-NEXT: v_perm_b32 v4, v4, v7, 0x5040100
2111 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v1, 24, v1
2112 ; GFX11-DL-NEXT: v_perm_b32 v5, v5, v6, 0x5040100
2113 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v0, 24, v0
2114 ; GFX11-DL-NEXT: v_and_b32_e32 v6, 0xff, v9
2115 ; GFX11-DL-NEXT: v_and_b32_e32 v7, 0xff, v8
2116 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
2117 ; GFX11-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v5
2118 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v6, 0x5040100
2119 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
2120 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v7, 0x5040100
2121 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
2122 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
2123 ; GFX11-DL-NEXT: v_pk_mul_lo_u16 v0, v1, v0
2124 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
2125 ; GFX11-DL-NEXT: v_add_nc_u16 v3, v4, v3
2126 ; GFX11-DL-NEXT: v_add_nc_u16 v1, v3, v5
2127 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
2128 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
2129 ; GFX11-DL-NEXT: v_add_nc_u16 v0, v1, v0
2130 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
2131 ; GFX11-DL-NEXT: v_add_nc_u16 v0, v0, v3
2132 ; GFX11-DL-NEXT: global_store_b16 v2, v0, s[0:1]
2133 ; GFX11-DL-NEXT: s_nop 0
2134 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2135 ; GFX11-DL-NEXT: s_endpgm
2136 ptr addrspace(1) %src2,
2137 ptr addrspace(1) nocapture %dst) {
2138 entry:
2139 %idx = call i32 @llvm.amdgcn.workitem.id.x()
2140 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
2141 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
2142 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
2143 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
2145 %cvec1 = zext <4 x i8> %vec1 to <4 x i16>
2146 %cvec2 = zext <4 x i8> %vec2 to <4 x i16>
2148 %mul = mul <4 x i16> %cvec1, %cvec2
2149 %mul0 = extractelement <4 x i16> %mul, i64 0
2150 %mul1 = extractelement <4 x i16> %mul, i64 1
2151 %mul2 = extractelement <4 x i16> %mul, i64 2
2152 %mul3 = extractelement <4 x i16> %mul, i64 3
2154 %acc = load i16, ptr addrspace(1) %dst, align 4
2155 %add1 = add i16 %mul0, %acc
2156 %add2 = add i16 %add1, %mul1
2157 %add3 = add i16 %add2, %mul2
2158 %add4 = add i16 %add3, %mul3
2160 store i16 %add4, ptr addrspace(1) %dst, align 4
2161 ret void
2162 }
2164 ; TODO: Support this pattern.
2165 define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,
2166 ; GFX7-LABEL: udot4_acc8_vecMul:
2167 ; GFX7: ; %bb.0: ; %entry
2168 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
2169 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
2170 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
2171 ; GFX7-NEXT: s_mov_b32 s10, 0
2172 ; GFX7-NEXT: s_mov_b32 s11, s3
2173 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2174 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
2175 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2176 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
2177 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
2178 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
2179 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
2180 ; GFX7-NEXT: s_mov_b32 s2, -1
2181 ; GFX7-NEXT: buffer_load_ubyte v1, off, s[0:3], 0
2182 ; GFX7-NEXT: s_waitcnt vmcnt(2)
2183 ; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v2
2184 ; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 8
2185 ; GFX7-NEXT: s_waitcnt vmcnt(1)
2186 ; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v0
2187 ; GFX7-NEXT: v_bfe_u32 v8, v0, 8, 8
2188 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2189 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
2190 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v2
2191 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
2192 ; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v0
2193 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
2194 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v8, v1
2195 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
2196 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v6, v0
2197 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0
2198 ; GFX7-NEXT: s_endpgm
2199 ;
2200 ; GFX8-LABEL: udot4_acc8_vecMul:
2201 ; GFX8: ; %bb.0: ; %entry
2202 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2203 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2204 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
2205 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2206 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2207 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
2208 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2209 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2210 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2211 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
2212 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2213 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
2214 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2215 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2216 ; GFX8-NEXT: flat_load_ubyte v4, v[0:1]
2217 ; GFX8-NEXT: s_waitcnt vmcnt(2)
2218 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
2219 ; GFX8-NEXT: s_waitcnt vmcnt(1)
2220 ; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
2221 ; GFX8-NEXT: v_mul_lo_u16_sdwa v7, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
2222 ; GFX8-NEXT: v_mul_lo_u16_e32 v9, v5, v6
2223 ; GFX8-NEXT: v_or_b32_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2224 ; GFX8-NEXT: v_mul_lo_u16_sdwa v8, v3, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2225 ; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v7
2226 ; GFX8-NEXT: v_or_b32_e32 v8, v8, v9
2227 ; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v8
2228 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2229 ; GFX8-NEXT: v_mad_u16 v2, v3, v2, v4
2230 ; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
2231 ; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v7
2232 ; GFX8-NEXT: v_mad_u16 v2, v5, v6, v2
2233 ; GFX8-NEXT: v_add_u16_e32 v2, v2, v7
2234 ; GFX8-NEXT: flat_store_byte v[0:1], v2
2235 ; GFX8-NEXT: s_endpgm
2236 ;
2237 ; GFX9-NODL-LABEL: udot4_acc8_vecMul:
2238 ; GFX9-NODL: ; %bb.0: ; %entry
2239 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2240 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2241 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2242 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2243 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
2244 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
2245 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
2246 ; GFX9-NODL-NEXT: global_load_ubyte v3, v0, s[2:3]
2247 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
2248 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
2249 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
2250 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
2251 ; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
2252 ; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v7, v4, v5
2253 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v8, 8, v6
2254 ; GFX9-NODL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2255 ; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PRESERVE src0_sel:BYTE_1 src1_sel:BYTE_1
2256 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
2257 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
2258 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
2259 ; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v6
2260 ; GFX9-NODL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
2261 ; GFX9-NODL-NEXT: v_add_u16_e32 v1, v1, v8
2262 ; GFX9-NODL-NEXT: global_store_byte v0, v1, s[2:3]
2263 ; GFX9-NODL-NEXT: s_endpgm
2264 ;
2265 ; GFX9-DL-LABEL: udot4_acc8_vecMul:
2266 ; GFX9-DL: ; %bb.0: ; %entry
2267 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2268 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2269 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2270 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2271 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
2272 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
2273 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2274 ; GFX9-DL-NEXT: global_load_ubyte v3, v0, s[2:3]
2275 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
2276 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
2277 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2278 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
2279 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
2280 ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, v4, v5
2281 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v6
2282 ; GFX9-DL-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2283 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, v1, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PRESERVE src0_sel:BYTE_1 src1_sel:BYTE_1
2284 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
2285 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2286 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v1, v2, v3
2287 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v6
2288 ; GFX9-DL-NEXT: v_mad_legacy_u16 v1, v4, v5, v1
2289 ; GFX9-DL-NEXT: v_add_u16_e32 v1, v1, v8
2290 ; GFX9-DL-NEXT: global_store_byte v0, v1, s[2:3]
2291 ; GFX9-DL-NEXT: s_endpgm
2292 ;
2293 ; GFX10-DL-LABEL: udot4_acc8_vecMul:
2294 ; GFX10-DL: ; %bb.0: ; %entry
2295 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2296 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2297 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2298 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2299 ; GFX10-DL-NEXT: s_clause 0x1
2300 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
2301 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
2302 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
2303 ; GFX10-DL-NEXT: global_load_ubyte v3, v0, s[0:1]
2304 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
2305 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 24, v1
2306 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2307 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v2
2308 ; GFX10-DL-NEXT: v_lshrrev_b16 v6, 8, v1
2309 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v1
2310 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 16, v2
2311 ; GFX10-DL-NEXT: v_lshrrev_b16 v9, 8, v2
2312 ; GFX10-DL-NEXT: v_mul_lo_u16 v4, v4, v5
2313 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2314 ; GFX10-DL-NEXT: v_mad_u16 v1, v1, v2, v3
2315 ; GFX10-DL-NEXT: v_mul_lo_u16 v5, v7, v8
2316 ; GFX10-DL-NEXT: v_mul_lo_u16 v6, v6, v9
2317 ; GFX10-DL-NEXT: v_lshlrev_b16 v4, 8, v4
2318 ; GFX10-DL-NEXT: v_lshlrev_b16 v6, 8, v6
2319 ; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
2320 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v2, 8, v4
2321 ; GFX10-DL-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
2322 ; GFX10-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v5
2323 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v5
2324 ; GFX10-DL-NEXT: v_mad_u16 v1, v7, v8, v1
2325 ; GFX10-DL-NEXT: v_add_nc_u16 v1, v1, v2
2326 ; GFX10-DL-NEXT: global_store_byte v0, v1, s[0:1]
2327 ; GFX10-DL-NEXT: s_endpgm
2328 ;
2329 ; GFX11-DL-LABEL: udot4_acc8_vecMul:
2330 ; GFX11-DL: ; %bb.0: ; %entry
2331 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
2332 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2333 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
2334 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
2335 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2336 ; GFX11-DL-NEXT: s_clause 0x1
2337 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
2338 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
2339 ; GFX11-DL-NEXT: global_load_u8 v3, v2, s[0:1]
2340 ; GFX11-DL-NEXT: s_waitcnt vmcnt(2)
2341 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v4, 16, v1
2342 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v5, 24, v1
2343 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
2344 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v6, 24, v0
2345 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v7, 16, v0
2346 ; GFX11-DL-NEXT: v_lshrrev_b16 v8, 8, v1
2347 ; GFX11-DL-NEXT: v_lshrrev_b16 v9, 8, v0
2348 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
2349 ; GFX11-DL-NEXT: v_mad_u16 v0, v1, v0, v3
2350 ; GFX11-DL-NEXT: v_mul_lo_u16 v5, v5, v6
2351 ; GFX11-DL-NEXT: v_mul_lo_u16 v6, v4, v7
2352 ; GFX11-DL-NEXT: v_mul_lo_u16 v8, v8, v9
2353 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
2354 ; GFX11-DL-NEXT: v_lshlrev_b16 v5, 8, v5
2355 ; GFX11-DL-NEXT: v_and_b32_e32 v6, 0xff, v6
2356 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
2357 ; GFX11-DL-NEXT: v_lshlrev_b16 v8, 8, v8
2358 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v1, 8, v5
2359 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
2360 ; GFX11-DL-NEXT: v_or_b32_e32 v6, v6, v5
2361 ; GFX11-DL-NEXT: v_and_b32_e32 v8, 0xffff, v8
2362 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
2363 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
2364 ; GFX11-DL-NEXT: v_or_b32_e32 v6, v8, v6
2365 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2366 ; GFX11-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v6
2367 ; GFX11-DL-NEXT: v_add_nc_u16 v0, v0, v6
2368 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
2369 ; GFX11-DL-NEXT: v_mad_u16 v0, v4, v7, v0
2370 ; GFX11-DL-NEXT: v_add_nc_u16 v0, v0, v1
2371 ; GFX11-DL-NEXT: global_store_b8 v2, v0, s[0:1]
2372 ; GFX11-DL-NEXT: s_nop 0
2373 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2374 ; GFX11-DL-NEXT: s_endpgm
2375 ptr addrspace(1) %src2,
2376 ptr addrspace(1) nocapture %dst) {
2377 entry:
2378 %idx = call i32 @llvm.amdgcn.workitem.id.x()
2379 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
2380 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
2381 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
2382 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
2384 %mul = mul <4 x i8> %vec1, %vec2
2385 %mul0 = extractelement <4 x i8> %mul, i64 0
2386 %mul1 = extractelement <4 x i8> %mul, i64 1
2387 %mul2 = extractelement <4 x i8> %mul, i64 2
2388 %mul3 = extractelement <4 x i8> %mul, i64 3
2390 %acc = load i8, ptr addrspace(1) %dst, align 4
2391 %add1 = add i8 %mul0, %acc
2392 %add2 = add i8 %add1, %mul1
2393 %add3 = add i8 %add2, %mul2
2394 %add4 = add i8 %add3, %mul3
2396 store i8 %add4, ptr addrspace(1) %dst, align 4
2397 ret void
2398 }
2400 define amdgpu_kernel void @idot4_acc32_2ele(ptr addrspace(1) %src1,
2401 ; GFX7-LABEL: idot4_acc32_2ele:
2402 ; GFX7: ; %bb.0: ; %entry
2403 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
2404 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
2405 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
2406 ; GFX7-NEXT: s_mov_b32 s10, 0
2407 ; GFX7-NEXT: s_mov_b32 s11, s3
2408 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2409 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
2410 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2411 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
2412 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
2413 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
2414 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
2415 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
2416 ; GFX7-NEXT: s_mov_b32 s2, -1
2417 ; GFX7-NEXT: s_waitcnt vmcnt(1)
2418 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
2419 ; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 8
2420 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2421 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v0
2422 ; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8
2423 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2424 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v3, s4
2425 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
2426 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
2427 ; GFX7-NEXT: s_endpgm
2428 ;
2429 ; GFX8-LABEL: idot4_acc32_2ele:
2430 ; GFX8: ; %bb.0: ; %entry
2431 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2432 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2433 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
2434 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2435 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2436 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
2437 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2438 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2439 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2440 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
2441 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2442 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
2443 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
2444 ; GFX8-NEXT: s_waitcnt vmcnt(1)
2445 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
2446 ; GFX8-NEXT: v_bfe_u32 v3, v3, 8, 8
2447 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2448 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v0
2449 ; GFX8-NEXT: v_bfe_u32 v0, v0, 8, 8
2450 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2451 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s2
2452 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
2453 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2454 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2455 ; GFX8-NEXT: flat_store_dword v[0:1], v2
2456 ; GFX8-NEXT: s_endpgm
2457 ;
2458 ; GFX9-NODL-LABEL: idot4_acc32_2ele:
2459 ; GFX9-NODL: ; %bb.0: ; %entry
2460 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2461 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2462 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2463 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2464 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
2465 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
2466 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
2467 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
2468 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
2469 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2470 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2471 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2472 ; GFX9-NODL-NEXT: v_add3_u32 v1, v3, s0, v1
2473 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
2474 ; GFX9-NODL-NEXT: s_endpgm
2475 ;
2476 ; GFX9-DL-LABEL: idot4_acc32_2ele:
2477 ; GFX9-DL: ; %bb.0: ; %entry
2478 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2479 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2480 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2481 ; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0100
2482 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2483 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[6:7]
2484 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
2485 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
2486 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2487 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2488 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s1
2489 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2490 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s1
2491 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2492 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s0
2493 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
2494 ; GFX9-DL-NEXT: s_endpgm
2495 ;
2496 ; GFX10-DL-LABEL: idot4_acc32_2ele:
2497 ; GFX10-DL: ; %bb.0: ; %entry
2498 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2499 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2500 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2501 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2502 ; GFX10-DL-NEXT: s_clause 0x1
2503 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[6:7]
2504 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
2505 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
2506 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2507 ; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc0c0100
2508 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2509 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc0c0100
2510 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
2511 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2512 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
2513 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
2514 ; GFX10-DL-NEXT: s_endpgm
2515 ;
2516 ; GFX11-DL-LABEL: idot4_acc32_2ele:
2517 ; GFX11-DL: ; %bb.0: ; %entry
2518 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
2519 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2520 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
2521 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
2522 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2523 ; GFX11-DL-NEXT: s_clause 0x1
2524 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[6:7]
2525 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[4:5]
2526 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
2527 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
2528 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0100
2529 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
2530 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0100
2531 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2532 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
2533 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s2
2534 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
2535 ; GFX11-DL-NEXT: s_nop 0
2536 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2537 ; GFX11-DL-NEXT: s_endpgm
2538 ptr addrspace(1) %src2,
2539 ptr addrspace(1) nocapture %dst) {
2540 entry:
2541 %idx = call i32 @llvm.amdgcn.workitem.id.x()
2542 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
2543 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
2544 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
2545 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
2547 %v1e0 = extractelement <4 x i8> %vec1, i64 0
2548 %cv1e0 = zext i8 %v1e0 to i32
2549 %v2e0 = extractelement <4 x i8> %vec2, i64 0
2550 %cv2e0 = zext i8 %v2e0 to i32
2551 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
2553 %v1e1 = extractelement <4 x i8> %vec1, i64 1
2554 %cv1e1 = zext i8 %v1e1 to i32
2555 %v2e1 = extractelement <4 x i8> %vec2, i64 1
2556 %cv2e1 = zext i8 %v2e1 to i32
2557 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
2559 %acc = load i32, ptr addrspace(1) %dst, align 4
2560 %add1 = add i32 %mul1, %acc
2561 %add2 = add i32 %add1, %mul2
2562 store i32 %add2, ptr addrspace(1) %dst, align 4
2563 ret void
2564 }
2566 define amdgpu_kernel void @idot4_acc32_3ele(ptr addrspace(1) %src1,
2567 ; GFX7-LABEL: idot4_acc32_3ele:
2568 ; GFX7: ; %bb.0: ; %entry
2569 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
2570 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
2571 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
2572 ; GFX7-NEXT: s_mov_b32 s10, 0
2573 ; GFX7-NEXT: s_mov_b32 s11, s3
2574 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2575 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
2576 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2577 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
2578 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
2579 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
2580 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
2581 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
2582 ; GFX7-NEXT: s_mov_b32 s2, -1
2583 ; GFX7-NEXT: s_waitcnt vmcnt(1)
2584 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
2585 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
2586 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2587 ; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
2588 ; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 8
2589 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2590 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
2591 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
2592 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
2593 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v1
2594 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
2595 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
2596 ; GFX7-NEXT: s_endpgm
2597 ;
2598 ; GFX8-LABEL: idot4_acc32_3ele:
2599 ; GFX8: ; %bb.0: ; %entry
2600 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2601 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2602 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
2603 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2604 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2605 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
2606 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2607 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2608 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2609 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
2610 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2611 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
2612 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
2613 ; GFX8-NEXT: s_waitcnt vmcnt(1)
2614 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
2615 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
2616 ; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 8
2617 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2618 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v0
2619 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
2620 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2621 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s2
2622 ; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 8
2623 ; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
2624 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
2625 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2626 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2627 ; GFX8-NEXT: flat_store_dword v[0:1], v2
2628 ; GFX8-NEXT: s_endpgm
2629 ;
2630 ; GFX9-NODL-LABEL: idot4_acc32_3ele:
2631 ; GFX9-NODL: ; %bb.0: ; %entry
2632 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2633 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2634 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2635 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2636 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
2637 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
2638 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
2639 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
2640 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
2641 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v1
2642 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
2643 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v2
2644 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2645 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
2646 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2647 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, s0
2648 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
2649 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
2650 ; GFX9-NODL-NEXT: s_endpgm
2651 ;
2652 ; GFX9-DL-LABEL: idot4_acc32_3ele:
2653 ; GFX9-DL: ; %bb.0: ; %entry
2654 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2655 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2656 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2657 ; GFX9-DL-NEXT: s_mov_b32 s1, 0xc020100
2658 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2659 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[6:7]
2660 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
2661 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
2662 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2663 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2664 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s1
2665 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2666 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s1
2667 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2668 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s0
2669 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
2670 ; GFX9-DL-NEXT: s_endpgm
2671 ;
2672 ; GFX10-DL-LABEL: idot4_acc32_3ele:
2673 ; GFX10-DL: ; %bb.0: ; %entry
2674 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2675 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2676 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2677 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2678 ; GFX10-DL-NEXT: s_clause 0x1
2679 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[6:7]
2680 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
2681 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
2682 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2683 ; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc020100
2684 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2685 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc020100
2686 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
2687 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2688 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
2689 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
2690 ; GFX10-DL-NEXT: s_endpgm
2691 ;
2692 ; GFX11-DL-LABEL: idot4_acc32_3ele:
2693 ; GFX11-DL: ; %bb.0: ; %entry
2694 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
2695 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2696 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
2697 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
2698 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2699 ; GFX11-DL-NEXT: s_clause 0x1
2700 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[6:7]
2701 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[4:5]
2702 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
2703 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
2704 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc020100
2705 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
2706 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc020100
2707 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2708 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
2709 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s2
2710 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
2711 ; GFX11-DL-NEXT: s_nop 0
2712 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2713 ; GFX11-DL-NEXT: s_endpgm
2714 ptr addrspace(1) %src2,
2715 ptr addrspace(1) nocapture %dst) {
2716 entry:
2717 %idx = call i32 @llvm.amdgcn.workitem.id.x()
2718 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
2719 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
2720 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
2721 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
2723 %v1e0 = extractelement <4 x i8> %vec1, i64 0
2724 %cv1e0 = zext i8 %v1e0 to i32
2725 %v2e0 = extractelement <4 x i8> %vec2, i64 0
2726 %cv2e0 = zext i8 %v2e0 to i32
2727 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
2729 %v1e1 = extractelement <4 x i8> %vec1, i64 1
2730 %cv1e1 = zext i8 %v1e1 to i32
2731 %v2e1 = extractelement <4 x i8> %vec2, i64 1
2732 %cv2e1 = zext i8 %v2e1 to i32
2733 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
2735 %v1e2 = extractelement <4 x i8> %vec1, i64 2
2736 %cv1e2 = zext i8 %v1e2 to i32
2737 %v2e2 = extractelement <4 x i8> %vec2, i64 2
2738 %cv2e2 = zext i8 %v2e2 to i32
2739 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
2741 %acc = load i32, ptr addrspace(1) %dst, align 4
2742 %add1 = add i32 %mul1, %acc
2743 %add2 = add i32 %add1, %mul2
2744 %add3 = add i32 %add2, %mul3
2745 store i32 %add3, ptr addrspace(1) %dst, align 4
2746 ret void
2747 }
2749 define amdgpu_kernel void @idot4_acc32_3ele_permuted(ptr addrspace(1) %src1,
2750 ; GFX7-LABEL: idot4_acc32_3ele_permuted:
2751 ; GFX7: ; %bb.0: ; %entry
2752 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
2753 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
2754 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
2755 ; GFX7-NEXT: s_mov_b32 s10, 0
2756 ; GFX7-NEXT: s_mov_b32 s11, s3
2757 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2758 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
2759 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2760 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
2761 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
2762 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
2763 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
2764 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
2765 ; GFX7-NEXT: s_mov_b32 s2, -1
2766 ; GFX7-NEXT: s_waitcnt vmcnt(1)
2767 ; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v2
2768 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
2769 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2770 ; GFX7-NEXT: v_lshrrev_b32_e32 v4, 24, v0
2771 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v0
2772 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2773 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
2774 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
2775 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
2776 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v1
2777 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
2778 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
2779 ; GFX7-NEXT: s_endpgm
2780 ;
2781 ; GFX8-LABEL: idot4_acc32_3ele_permuted:
2782 ; GFX8: ; %bb.0: ; %entry
2783 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2784 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2785 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
2786 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2787 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2788 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
2789 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2790 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2791 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2792 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
2793 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2794 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
2795 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
2796 ; GFX8-NEXT: s_waitcnt vmcnt(1)
2797 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v3
2798 ; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v3
2799 ; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 8
2800 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2801 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v0
2802 ; GFX8-NEXT: v_and_b32_e32 v5, 0xff, v0
2803 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2804 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s2
2805 ; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 8
2806 ; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
2807 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
2808 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2809 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2810 ; GFX8-NEXT: flat_store_dword v[0:1], v2
2811 ; GFX8-NEXT: s_endpgm
2812 ;
2813 ; GFX9-NODL-LABEL: idot4_acc32_3ele_permuted:
2814 ; GFX9-NODL: ; %bb.0: ; %entry
2815 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2816 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2817 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2818 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2819 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
2820 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
2821 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
2822 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
2823 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
2824 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v1
2825 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
2826 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v4, 24, v2
2827 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
2828 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
2829 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
2830 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, s0
2831 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
2832 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
2833 ; GFX9-NODL-NEXT: s_endpgm
2834 ;
2835 ; GFX9-DL-LABEL: idot4_acc32_3ele_permuted:
2836 ; GFX9-DL: ; %bb.0: ; %entry
2837 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2838 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
2839 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2840 ; GFX9-DL-NEXT: s_mov_b32 s1, 0xc020003
2841 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2842 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[6:7]
2843 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
2844 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
2845 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
2846 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
2847 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s1
2848 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
2849 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s1
2850 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
2851 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s0
2852 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
2853 ; GFX9-DL-NEXT: s_endpgm
2854 ;
2855 ; GFX10-DL-LABEL: idot4_acc32_3ele_permuted:
2856 ; GFX10-DL: ; %bb.0: ; %entry
2857 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2858 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2859 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2860 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2861 ; GFX10-DL-NEXT: s_clause 0x1
2862 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[6:7]
2863 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
2864 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
2865 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
2866 ; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc020003
2867 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
2868 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc020003
2869 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
2870 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
2871 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
2872 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
2873 ; GFX10-DL-NEXT: s_endpgm
2874 ;
2875 ; GFX11-DL-LABEL: idot4_acc32_3ele_permuted:
2876 ; GFX11-DL: ; %bb.0: ; %entry
2877 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
2878 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2879 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
2880 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
2881 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2882 ; GFX11-DL-NEXT: s_clause 0x1
2883 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[6:7]
2884 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[4:5]
2885 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
2886 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
2887 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc020003
2888 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
2889 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc020003
2890 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
2891 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
2892 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s2
2893 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
2894 ; GFX11-DL-NEXT: s_nop 0
2895 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
2896 ; GFX11-DL-NEXT: s_endpgm
2897 ptr addrspace(1) %src2,
2898 ptr addrspace(1) nocapture %dst) {
2899 entry:
2900 %idx = call i32 @llvm.amdgcn.workitem.id.x()
2901 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
2902 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
2903 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
2904 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
2906 %v1e0 = extractelement <4 x i8> %vec1, i64 3
2907 %cv1e0 = zext i8 %v1e0 to i32
2908 %v2e0 = extractelement <4 x i8> %vec2, i64 3
2909 %cv2e0 = zext i8 %v2e0 to i32
2910 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
2912 %v1e1 = extractelement <4 x i8> %vec1, i64 0
2913 %cv1e1 = zext i8 %v1e1 to i32
2914 %v2e1 = extractelement <4 x i8> %vec2, i64 0
2915 %cv2e1 = zext i8 %v2e1 to i32
2916 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
2918 %v1e2 = extractelement <4 x i8> %vec1, i64 2
2919 %cv1e2 = zext i8 %v1e2 to i32
2920 %v2e2 = extractelement <4 x i8> %vec2, i64 2
2921 %cv2e2 = zext i8 %v2e2 to i32
2922 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
2924 %acc = load i32, ptr addrspace(1) %dst, align 4
2925 %add1 = add i32 %mul1, %acc
2926 %add2 = add i32 %add1, %mul2
2927 %add3 = add i32 %add2, %mul3
2928 store i32 %add3, ptr addrspace(1) %dst, align 4
2929 ret void
2930 }
2933 define amdgpu_kernel void @idot4_acc32_opt(ptr addrspace(1) %src1,
2934 ; GFX7-LABEL: idot4_acc32_opt:
2935 ; GFX7: ; %bb.0: ; %entry
2936 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
2937 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
2938 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
2939 ; GFX7-NEXT: s_mov_b32 s10, 0
2940 ; GFX7-NEXT: s_mov_b32 s11, s3
2941 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
2942 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
2943 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
2944 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
2945 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
2946 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
2947 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
2948 ; GFX7-NEXT: s_mov_b32 s2, -1
2949 ; GFX7-NEXT: s_waitcnt vmcnt(1)
2950 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
2951 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
2952 ; GFX7-NEXT: s_waitcnt vmcnt(0)
2953 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
2954 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v0
2955 ; GFX7-NEXT: v_mul_u32_u24_e32 v3, v3, v6
2956 ; GFX7-NEXT: v_bfe_u32 v4, v2, 16, 8
2957 ; GFX7-NEXT: v_bfe_u32 v7, v0, 16, 8
2958 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v5, v3
2959 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
2960 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
2961 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v7, v1
2962 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
2963 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
2964 ; GFX7-NEXT: s_endpgm
2965 ;
2966 ; GFX8-LABEL: idot4_acc32_opt:
2967 ; GFX8: ; %bb.0: ; %entry
2968 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
2969 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
2970 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
2971 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
2972 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
2973 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
2974 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2975 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
2976 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
2977 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
2978 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
2979 ; GFX8-NEXT: flat_load_dword v2, v[0:1]
2980 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
2981 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
2982 ; GFX8-NEXT: s_waitcnt vmcnt(1)
2983 ; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v3
2984 ; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 8
2985 ; GFX8-NEXT: s_waitcnt vmcnt(0)
2986 ; GFX8-NEXT: v_and_b32_e32 v5, 0xff, v2
2987 ; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
2988 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 8
2989 ; GFX8-NEXT: v_mad_u32_u24 v4, v4, v5, v6
2990 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
2991 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
2992 ; GFX8-NEXT: v_mad_u32_u24 v4, v7, v8, v4
2993 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v2, v4
2994 ; GFX8-NEXT: flat_store_dword v[0:1], v2
2995 ; GFX8-NEXT: s_endpgm
2996 ;
2997 ; GFX9-NODL-LABEL: idot4_acc32_opt:
2998 ; GFX9-NODL: ; %bb.0: ; %entry
2999 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3000 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
3001 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3002 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3003 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
3004 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
3005 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
3006 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
3007 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v1
3008 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
3009 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v2
3010 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
3011 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
3012 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
3013 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, v5
3014 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v6, v1
3015 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
3016 ; GFX9-NODL-NEXT: s_endpgm
3017 ;
3018 ; GFX9-DL-LABEL: idot4_acc32_opt:
3019 ; GFX9-DL: ; %bb.0: ; %entry
3020 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3021 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
3022 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3023 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3024 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
3025 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
3026 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3027 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3028 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, 0
3029 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
3030 ; GFX9-DL-NEXT: s_endpgm
3031 ;
3032 ; GFX10-DL-LABEL: idot4_acc32_opt:
3033 ; GFX10-DL: ; %bb.0: ; %entry
3034 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3035 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3036 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
3037 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3038 ; GFX10-DL-NEXT: s_clause 0x1
3039 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
3040 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
3041 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, 0
3042 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
3043 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, 0
3044 ; GFX10-DL-NEXT: global_store_dword v0, v1, s[0:1]
3045 ; GFX10-DL-NEXT: s_endpgm
3046 ;
3047 ; GFX11-DL-LABEL: idot4_acc32_opt:
3048 ; GFX11-DL: ; %bb.0: ; %entry
3049 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
3050 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3051 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
3052 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
3053 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3054 ; GFX11-DL-NEXT: s_clause 0x1
3055 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
3056 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
3057 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
3058 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, 0
3059 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
3060 ; GFX11-DL-NEXT: s_nop 0
3061 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3062 ; GFX11-DL-NEXT: s_endpgm
3063 ptr addrspace(1) %src2,
3064 ptr addrspace(1) nocapture %dst) {
3065 entry:
3066 %idx = call i32 @llvm.amdgcn.workitem.id.x()
3067 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
3068 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
3069 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
3070 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
3072 %v1e0 = extractelement <4 x i8> %vec1, i64 0
3073 %cv1e0 = zext i8 %v1e0 to i32
3074 %v2e0 = extractelement <4 x i8> %vec2, i64 0
3075 %cv2e0 = zext i8 %v2e0 to i32
3076 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
3078 %v1e1 = extractelement <4 x i8> %vec1, i64 1
3079 %cv1e1 = zext i8 %v1e1 to i32
3080 %v2e1 = extractelement <4 x i8> %vec2, i64 1
3081 %cv2e1 = zext i8 %v2e1 to i32
3082 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
3084 %v1e2 = extractelement <4 x i8> %vec1, i64 2
3085 %cv1e2 = zext i8 %v1e2 to i32
3086 %v2e2 = extractelement <4 x i8> %vec2, i64 2
3087 %cv2e2 = zext i8 %v2e2 to i32
3088 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
3090 %v1e3 = extractelement <4 x i8> %vec1, i64 3
3091 %cv1e3 = zext i8 %v1e3 to i32
3092 %v2e3 = extractelement <4 x i8> %vec2, i64 3
3093 %cv2e3 = zext i8 %v2e3 to i32
3094 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
3096 %add2 = add i32 %mul1, %mul2
3097 %add3 = add i32 %add2, %mul3
3098 %add4 = add i32 %add3, %mul4
3099 store i32 %add4, ptr addrspace(1) %dst, align 4
3100 ret void
3101 }
3103 define amdgpu_kernel void @udot4_acc32_3src(ptr addrspace(1) %src1,
3104 ; GFX7-LABEL: udot4_acc32_3src:
3105 ; GFX7: ; %bb.0: ; %entry
3106 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x9
3107 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
3108 ; GFX7-NEXT: s_mov_b32 s14, 0
3109 ; GFX7-NEXT: s_mov_b32 s15, s11
3110 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3111 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3112 ; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]
3113 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
3114 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr64
3115 ; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]
3116 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr64
3117 ; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]
3118 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
3119 ; GFX7-NEXT: s_load_dword s0, s[6:7], 0x0
3120 ; GFX7-NEXT: s_mov_b32 s10, -1
3121 ; GFX7-NEXT: s_mov_b32 s8, s6
3122 ; GFX7-NEXT: s_mov_b32 s9, s7
3123 ; GFX7-NEXT: s_waitcnt vmcnt(2)
3124 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
3125 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
3126 ; GFX7-NEXT: s_waitcnt vmcnt(1)
3127 ; GFX7-NEXT: v_bfe_u32 v3, v3, 8, 8
3128 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3129 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v1, s0
3130 ; GFX7-NEXT: v_bfe_u32 v5, v2, 16, 8
3131 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3132 ; GFX7-NEXT: v_bfe_u32 v6, v0, 16, 8
3133 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v3, v1
3134 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
3135 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
3136 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v6, v1
3137 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
3138 ; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 0
3139 ; GFX7-NEXT: s_endpgm
3140 ;
3141 ; GFX8-LABEL: udot4_acc32_3src:
3142 ; GFX8: ; %bb.0: ; %entry
3143 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3144 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
3145 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3146 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3147 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
3148 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3149 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
3150 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
3151 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
3152 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3153 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3154 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
3155 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
3156 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3157 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
3158 ; GFX8-NEXT: s_load_dword s0, s[6:7], 0x0
3159 ; GFX8-NEXT: s_waitcnt vmcnt(2)
3160 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
3161 ; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 8
3162 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3163 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s0
3164 ; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 8
3165 ; GFX8-NEXT: v_lshrrev_b32_e32 v3, 24, v3
3166 ; GFX8-NEXT: s_waitcnt vmcnt(1)
3167 ; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 8
3168 ; GFX8-NEXT: v_mad_u32_u24 v1, v2, v4, v1
3169 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3170 ; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 8
3171 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
3172 ; GFX8-NEXT: v_mad_u32_u24 v1, v5, v6, v1
3173 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
3174 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3175 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3176 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3177 ; GFX8-NEXT: s_endpgm
3178 ;
3179 ; GFX9-NODL-LABEL: udot4_acc32_3src:
3180 ; GFX9-NODL: ; %bb.0: ; %entry
3181 ; GFX9-NODL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3182 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3183 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3184 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]
3185 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]
3186 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[4:5]
3187 ; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
3188 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
3189 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
3190 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
3191 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
3192 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
3193 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
3194 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
3195 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
3196 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3197 ; GFX9-NODL-NEXT: v_add3_u32 v2, v4, s0, v2
3198 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
3199 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
3200 ; GFX9-NODL-NEXT: s_endpgm
3201 ;
3202 ; GFX9-DL-LABEL: udot4_acc32_3src:
3203 ; GFX9-DL: ; %bb.0: ; %entry
3204 ; GFX9-DL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3205 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3206 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3207 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]
3208 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
3209 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[0:1]
3210 ; GFX9-DL-NEXT: s_load_dword s1, s[6:7], 0x0
3211 ; GFX9-DL-NEXT: s_mov_b32 s0, 0x706010c
3212 ; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c0c00
3213 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3214 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
3215 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s0
3216 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3217 ; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s2
3218 ; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v2
3219 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3220 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v3, v1, s1
3221 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]
3222 ; GFX9-DL-NEXT: s_endpgm
3223 ;
3224 ; GFX10-DL-LABEL: udot4_acc32_3src:
3225 ; GFX10-DL: ; %bb.0: ; %entry
3226 ; GFX10-DL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3227 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3228 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3229 ; GFX10-DL-NEXT: s_clause 0x2
3230 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]
3231 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
3232 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[0:1]
3233 ; GFX10-DL-NEXT: s_waitcnt_depctr 0xffe3
3234 ; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0
3235 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
3236 ; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0x706010c
3237 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
3238 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c00
3239 ; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v1
3240 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, 0
3241 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3242 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v3, v0, s0
3243 ; GFX10-DL-NEXT: global_store_dword v1, v0, s[6:7]
3244 ; GFX10-DL-NEXT: s_endpgm
3245 ;
3246 ; GFX11-DL-LABEL: udot4_acc32_3src:
3247 ; GFX11-DL: ; %bb.0: ; %entry
3248 ; GFX11-DL-NEXT: s_load_b256 s[0:7], s[0:1], 0x24
3249 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3250 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3251 ; GFX11-DL-NEXT: s_clause 0x2
3252 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]
3253 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]
3254 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]
3255 ; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x0
3256 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
3257 ; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0x706010c
3258 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
3259 ; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c00
3260 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
3261 ; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v2
3262 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
3263 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3264 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s0
3265 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]
3266 ; GFX11-DL-NEXT: s_nop 0
3267 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3268 ; GFX11-DL-NEXT: s_endpgm
3269 ptr addrspace(1) %src2,
3270 ptr addrspace(1) %src3,
3271 ptr addrspace(1) nocapture %dst) {
3272 entry:
3273 %idx = call i32 @llvm.amdgcn.workitem.id.x()
3274 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
3275 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
3276 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
3277 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
3278 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx
3279 %vec3 = load <4 x i8>, ptr addrspace(1) %gep3
3281 %v1e0 = extractelement <4 x i8> %vec1, i64 0
3282 %cv1e0 = zext i8 %v1e0 to i32
3283 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e0
3285 %v1e1 = extractelement <4 x i8> %vec1, i64 1
3286 %cv1e1 = zext i8 %v1e1 to i32
3287 %v2e1 = extractelement <4 x i8> %vec2, i64 1
3288 %cv2e1 = zext i8 %v2e1 to i32
3289 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
3291 %v1e2 = extractelement <4 x i8> %vec1, i64 2
3292 %cv1e2 = zext i8 %v1e2 to i32
3293 %v3e2 = extractelement <4 x i8> %vec3, i64 2
3294 %cv3e2 = zext i8 %v3e2 to i32
3295 %mul3 = mul nuw nsw i32 %cv1e2, %cv3e2
3297 %v1e3 = extractelement <4 x i8> %vec1, i64 3
3298 %cv1e3 = zext i8 %v1e3 to i32
3299 %v3e3 = extractelement <4 x i8> %vec3, i64 3
3300 %cv3e3 = zext i8 %v3e3 to i32
3301 %mul4 = mul nuw nsw i32 %cv1e3, %cv3e3
3303 %acc = load i32, ptr addrspace(1) %dst, align 4
3304 %mad1 = add i32 %mul1, %acc
3305 %mad2 = add i32 %mad1, %mul2
3306 %mad3 = add i32 %mad2, %mul3
3307 %mad4 = add i32 %mad3, %mul4
3309 store i32 %mad4, ptr addrspace(1) %dst, align 4
3310 ret void
3311 }
3313 define amdgpu_kernel void @udot4_acc32_3src_3ele(ptr addrspace(1) %src1,
3314 ; GFX7-LABEL: udot4_acc32_3src_3ele:
3315 ; GFX7: ; %bb.0: ; %entry
3316 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x9
3317 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
3318 ; GFX7-NEXT: s_mov_b32 s14, 0
3319 ; GFX7-NEXT: s_mov_b32 s15, s11
3320 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3321 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3322 ; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]
3323 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
3324 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr64
3325 ; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]
3326 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr64
3327 ; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]
3328 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
3329 ; GFX7-NEXT: s_load_dword s0, s[6:7], 0x0
3330 ; GFX7-NEXT: s_mov_b32 s10, -1
3331 ; GFX7-NEXT: s_mov_b32 s8, s6
3332 ; GFX7-NEXT: s_mov_b32 s9, s7
3333 ; GFX7-NEXT: s_waitcnt vmcnt(2)
3334 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
3335 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
3336 ; GFX7-NEXT: s_waitcnt vmcnt(1)
3337 ; GFX7-NEXT: v_bfe_u32 v3, v3, 8, 8
3338 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3339 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v1, s0
3340 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
3341 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3342 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
3343 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v3, v1
3344 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
3345 ; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 0
3346 ; GFX7-NEXT: s_endpgm
3347 ;
3348 ; GFX8-LABEL: udot4_acc32_3src_3ele:
3349 ; GFX8: ; %bb.0: ; %entry
3350 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3351 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
3352 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3353 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3354 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
3355 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3356 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
3357 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
3358 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
3359 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3360 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3361 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
3362 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
3363 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3364 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
3365 ; GFX8-NEXT: s_load_dword s0, s[6:7], 0x0
3366 ; GFX8-NEXT: s_waitcnt vmcnt(2)
3367 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
3368 ; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 8
3369 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3370 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v1, s0
3371 ; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 8
3372 ; GFX8-NEXT: s_waitcnt vmcnt(1)
3373 ; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 8
3374 ; GFX8-NEXT: v_mad_u32_u24 v1, v2, v4, v1
3375 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3376 ; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 8
3377 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
3378 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3379 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3380 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3381 ; GFX8-NEXT: s_endpgm
3382 ;
3383 ; GFX9-NODL-LABEL: udot4_acc32_3src_3ele:
3384 ; GFX9-NODL: ; %bb.0: ; %entry
3385 ; GFX9-NODL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3386 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3387 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3388 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[0:1]
3389 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[2:3]
3390 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[4:5]
3391 ; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
3392 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
3393 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
3394 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v1
3395 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
3396 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
3397 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
3398 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
3399 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3400 ; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v4, s0
3401 ; GFX9-NODL-NEXT: v_add3_u32 v1, v3, v2, v1
3402 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
3403 ; GFX9-NODL-NEXT: s_endpgm
3404 ;
3405 ; GFX9-DL-LABEL: udot4_acc32_3src_3ele:
3406 ; GFX9-DL: ; %bb.0: ; %entry
3407 ; GFX9-DL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3408 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3409 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3410 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[2:3]
3411 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
3412 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[0:1]
3413 ; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
3414 ; GFX9-DL-NEXT: s_mov_b32 s0, 0xc06010c
3415 ; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0c00
3416 ; GFX9-DL-NEXT: s_mov_b32 s2, 0xc020100
3417 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3418 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
3419 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s0
3420 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3421 ; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s1
3422 ; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v2
3423 ; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s2
3424 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3425 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s3
3426 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]
3427 ; GFX9-DL-NEXT: s_endpgm
3428 ;
3429 ; GFX10-DL-LABEL: udot4_acc32_3src_3ele:
3430 ; GFX10-DL: ; %bb.0: ; %entry
3431 ; GFX10-DL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3432 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3433 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3434 ; GFX10-DL-NEXT: s_clause 0x2
3435 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[2:3]
3436 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
3437 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[0:1]
3438 ; GFX10-DL-NEXT: s_waitcnt_depctr 0xffe3
3439 ; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0
3440 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
3441 ; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc06010c
3442 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
3443 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c00
3444 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
3445 ; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v1
3446 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc020100
3447 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3448 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s0
3449 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]
3450 ; GFX10-DL-NEXT: s_endpgm
3451 ;
3452 ; GFX11-DL-LABEL: udot4_acc32_3src_3ele:
3453 ; GFX11-DL: ; %bb.0: ; %entry
3454 ; GFX11-DL-NEXT: s_load_b256 s[0:7], s[0:1], 0x24
3455 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3456 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3457 ; GFX11-DL-NEXT: s_clause 0x2
3458 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[2:3]
3459 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[4:5]
3460 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[0:1]
3461 ; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x0
3462 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
3463 ; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc06010c
3464 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
3465 ; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c00
3466 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc020100
3467 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
3468 ; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v2
3469 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
3470 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3471 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s0
3472 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]
3473 ; GFX11-DL-NEXT: s_nop 0
3474 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3475 ; GFX11-DL-NEXT: s_endpgm
3476 ptr addrspace(1) %src2,
3477 ptr addrspace(1) %src3,
3478 ptr addrspace(1) nocapture %dst) {
3479 entry:
3480 %idx = call i32 @llvm.amdgcn.workitem.id.x()
3481 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
3482 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
3483 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
3484 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
3485 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx
3486 %vec3 = load <4 x i8>, ptr addrspace(1) %gep3
3488 %v1e0 = extractelement <4 x i8> %vec1, i64 0
3489 %cv1e0 = zext i8 %v1e0 to i32
3490 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e0
3492 %v1e1 = extractelement <4 x i8> %vec1, i64 1
3493 %cv1e1 = zext i8 %v1e1 to i32
3494 %v2e1 = extractelement <4 x i8> %vec2, i64 1
3495 %cv2e1 = zext i8 %v2e1 to i32
3496 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
3498 %v1e2 = extractelement <4 x i8> %vec1, i64 2
3499 %cv1e2 = zext i8 %v1e2 to i32
3500 %v3e2 = extractelement <4 x i8> %vec3, i64 2
3501 %cv3e2 = zext i8 %v3e2 to i32
3502 %mul3 = mul nuw nsw i32 %cv1e2, %cv3e2
3505 %acc = load i32, ptr addrspace(1) %dst, align 4
3506 %mad1 = add i32 %mul1, %acc
3507 %mad2 = add i32 %mad1, %mul2
3508 %mad3 = add i32 %mad2, %mul3
3510 store i32 %mad3, ptr addrspace(1) %dst, align 4
3511 ret void
3512 }
3514 define amdgpu_kernel void @udot4_bad_source(ptr addrspace(1) %src1,
3515 ; GFX7-LABEL: udot4_bad_source:
3516 ; GFX7: ; %bb.0: ; %entry
3517 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
3518 ; GFX7-NEXT: s_load_dword s12, s[0:1], 0xf
3519 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
3520 ; GFX7-NEXT: s_mov_b32 s10, 0
3521 ; GFX7-NEXT: s_mov_b32 s11, s3
3522 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3523 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
3524 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3525 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
3526 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
3527 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
3528 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
3529 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x11
3530 ; GFX7-NEXT: s_and_b32 s5, s12, 0xffff
3531 ; GFX7-NEXT: s_mov_b32 s2, -1
3532 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3533 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
3534 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3535 ; GFX7-NEXT: v_mov_b32_e32 v1, s4
3536 ; GFX7-NEXT: s_waitcnt vmcnt(1)
3537 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v2
3538 ; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
3539 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3540 ; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 8
3541 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, s5, v1
3542 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
3543 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
3544 ; GFX7-NEXT: v_mad_u32_u24 v1, v4, v5, v1
3545 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
3546 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
3547 ; GFX7-NEXT: s_endpgm
3548 ;
3549 ; GFX8-LABEL: udot4_bad_source:
3550 ; GFX8: ; %bb.0: ; %entry
3551 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3552 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x3c
3553 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
3554 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
3555 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3556 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
3557 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
3558 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3559 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
3560 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3561 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
3562 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3563 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
3564 ; GFX8-NEXT: s_load_dword s3, s[0:1], 0x0
3565 ; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
3566 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3567 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
3568 ; GFX8-NEXT: s_waitcnt vmcnt(1)
3569 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v3
3570 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
3571 ; GFX8-NEXT: v_mad_u32_u24 v1, v2, s2, v1
3572 ; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 8
3573 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3574 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
3575 ; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 8
3576 ; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
3577 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
3578 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
3579 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3580 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3581 ; GFX8-NEXT: s_endpgm
3582 ;
3583 ; GFX9-NODL-LABEL: udot4_bad_source:
3584 ; GFX9-NODL: ; %bb.0: ; %entry
3585 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3586 ; GFX9-NODL-NEXT: s_load_dword s2, s[0:1], 0x3c
3587 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3588 ; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
3589 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3590 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
3591 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
3592 ; GFX9-NODL-NEXT: s_and_b32 s2, s2, 0xffff
3593 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
3594 ; GFX9-NODL-NEXT: s_load_dword s3, s[0:1], 0x0
3595 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
3596 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v1
3597 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
3598 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
3599 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
3600 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3601 ; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s3
3602 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, s2, v2
3603 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v4, v1
3604 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[0:1]
3605 ; GFX9-NODL-NEXT: s_endpgm
3606 ;
3607 ; GFX9-DL-LABEL: udot4_bad_source:
3608 ; GFX9-DL: ; %bb.0: ; %entry
3609 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3610 ; GFX9-DL-NEXT: s_load_dword s2, s[0:1], 0x3c
3611 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3612 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
3613 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3614 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
3615 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
3616 ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0201
3617 ; GFX9-DL-NEXT: s_and_b32 s2, s2, 0xffff
3618 ; GFX9-DL-NEXT: s_load_dword s3, s[0:1], 0x0
3619 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3620 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3621 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3
3622 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
3623 ; GFX9-DL-NEXT: v_and_b32_e32 v4, 0xff, v1
3624 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3625 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s4
3626 ; GFX9-DL-NEXT: v_mad_u32_u24 v3, v4, s2, v3
3627 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s4
3628 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v2, v3
3629 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[0:1]
3630 ; GFX9-DL-NEXT: s_endpgm
3631 ;
3632 ; GFX10-DL-LABEL: udot4_bad_source:
3633 ; GFX10-DL: ; %bb.0: ; %entry
3634 ; GFX10-DL-NEXT: s_clause 0x1
3635 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3636 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x3c
3637 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3638 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
3639 ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
3640 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3641 ; GFX10-DL-NEXT: s_clause 0x1
3642 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
3643 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
3644 ; GFX10-DL-NEXT: s_and_b32 s2, s2, 0xffff
3645 ; GFX10-DL-NEXT: s_load_dword s3, s[0:1], 0x0
3646 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
3647 ; GFX10-DL-NEXT: v_and_b32_e32 v0, 0xff, v1
3648 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
3649 ; GFX10-DL-NEXT: v_perm_b32 v2, v2, v2, 0xc0c0201
3650 ; GFX10-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0201
3651 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3652 ; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, s2, s3
3653 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v2, v0
3654 ; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
3655 ; GFX10-DL-NEXT: s_endpgm
3656 ;
3657 ; GFX11-DL-LABEL: udot4_bad_source:
3658 ; GFX11-DL: ; %bb.0: ; %entry
3659 ; GFX11-DL-NEXT: s_clause 0x1
3660 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
3661 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x3c
3662 ; GFX11-DL-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v0, 2, v0
3663 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x44
3664 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3665 ; GFX11-DL-NEXT: s_clause 0x1
3666 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
3667 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[6:7]
3668 ; GFX11-DL-NEXT: s_and_b32 s2, s2, 0xffff
3669 ; GFX11-DL-NEXT: s_load_b32 s3, s[0:1], 0x0
3670 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
3671 ; GFX11-DL-NEXT: v_and_b32_e32 v2, 0xff, v1
3672 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
3673 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0201
3674 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0201
3675 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3676 ; GFX11-DL-NEXT: v_mad_u32_u24 v2, v2, s2, s3
3677 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
3678 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, v2
3679 ; GFX11-DL-NEXT: global_store_b32 v3, v0, s[0:1]
3680 ; GFX11-DL-NEXT: s_nop 0
3681 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3682 ; GFX11-DL-NEXT: s_endpgm
3683 ptr addrspace(1) %src2,
3684 ptr addrspace(1) %src3,
3685 i16 %badsource,
3686 ptr addrspace(1) nocapture %dst) {
3687 entry:
3688 %idx = call i32 @llvm.amdgcn.workitem.id.x()
3689 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
3690 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
3691 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
3692 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
3693 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx
3694 %vec3 = load <4 x i8>, ptr addrspace(1) %gep3
3696 %v1e0 = extractelement <4 x i8> %vec1, i64 0
3697 %cv1e0 = zext i8 %v1e0 to i32
3698 %v2e0 = extractelement <4 x i8> %vec2, i64 0
3699 %other = zext i16 %badsource to i32
3700 %mul1 = mul nuw nsw i32 %cv1e0, %other
3702 %v1e1 = extractelement <4 x i8> %vec1, i64 1
3703 %cv1e1 = zext i8 %v1e1 to i32
3704 %v2e1 = extractelement <4 x i8> %vec2, i64 1
3705 %cv2e1 = zext i8 %v2e1 to i32
3706 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
3708 %v2e2 = extractelement <4 x i8> %vec2, i64 2
3709 %cv2e2 = zext i8 %v2e2 to i32
3710 %v1e2 = extractelement <4 x i8> %vec1, i64 2
3711 %cv1e2 = zext i8 %v1e2 to i32
3712 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
3715 %acc = load i32, ptr addrspace(1) %dst, align 4
3716 %mad1 = add i32 %mul1, %acc
3717 %mad2 = add i32 %mad1, %mul2
3718 %mad3 = add i32 %mad2, %mul3
3720 store i32 %mad3, ptr addrspace(1) %dst, align 4
3721 ret void
3722 }
3725 define amdgpu_kernel void @udot4_commutative(ptr addrspace(1) %src1,
3726 ; GFX7-LABEL: udot4_commutative:
3727 ; GFX7: ; %bb.0: ; %entry
3728 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
3729 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xf
3730 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
3731 ; GFX7-NEXT: s_mov_b32 s10, 0
3732 ; GFX7-NEXT: s_mov_b32 s11, s3
3733 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3734 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
3735 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3736 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
3737 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[8:11], 0 addr64
3738 ; GFX7-NEXT: s_mov_b64 s[8:9], s[6:7]
3739 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
3740 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
3741 ; GFX7-NEXT: s_mov_b32 s2, -1
3742 ; GFX7-NEXT: s_waitcnt vmcnt(1)
3743 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
3744 ; GFX7-NEXT: v_bfe_u32 v3, v2, 8, 8
3745 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3746 ; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
3747 ; GFX7-NEXT: v_bfe_u32 v5, v0, 8, 8
3748 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3749 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
3750 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
3751 ; GFX7-NEXT: v_bfe_u32 v2, v2, 16, 8
3752 ; GFX7-NEXT: v_mad_u32_u24 v1, v3, v5, v1
3753 ; GFX7-NEXT: v_mad_u32_u24 v0, v2, v0, v1
3754 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
3755 ; GFX7-NEXT: s_endpgm
3756 ;
3757 ; GFX8-LABEL: udot4_commutative:
3758 ; GFX8: ; %bb.0: ; %entry
3759 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3760 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x3c
3761 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
3762 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3763 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
3764 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
3765 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3766 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
3767 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3768 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
3769 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3770 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
3771 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
3772 ; GFX8-NEXT: s_waitcnt vmcnt(1)
3773 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
3774 ; GFX8-NEXT: v_bfe_u32 v4, v3, 8, 8
3775 ; GFX8-NEXT: v_bfe_u32 v3, v3, 16, 8
3776 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3777 ; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v0
3778 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
3779 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3780 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s2
3781 ; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 8
3782 ; GFX8-NEXT: v_mad_u32_u24 v1, v4, v5, v1
3783 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
3784 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
3785 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3786 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3787 ; GFX8-NEXT: s_endpgm
3788 ;
3789 ; GFX9-NODL-LABEL: udot4_commutative:
3790 ; GFX9-NODL: ; %bb.0: ; %entry
3791 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3792 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c
3793 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3794 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3795 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
3796 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
3797 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
3798 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
3799 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
3800 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xff, v1
3801 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
3802 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v2
3803 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
3804 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
3805 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3806 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, v3, v4, s0
3807 ; GFX9-NODL-NEXT: v_add3_u32 v1, v2, v5, v1
3808 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[2:3]
3809 ; GFX9-NODL-NEXT: s_endpgm
3810 ;
3811 ; GFX9-DL-LABEL: udot4_commutative:
3812 ; GFX9-DL: ; %bb.0: ; %entry
3813 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3814 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c
3815 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3816 ; GFX9-DL-NEXT: s_mov_b32 s1, 0xc020100
3817 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3818 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[6:7]
3819 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[4:5]
3820 ; GFX9-DL-NEXT: s_load_dword s0, s[2:3], 0x0
3821 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
3822 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
3823 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v1, s1
3824 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
3825 ; GFX9-DL-NEXT: v_perm_b32 v2, v2, v2, s1
3826 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
3827 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s0
3828 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[2:3]
3829 ; GFX9-DL-NEXT: s_endpgm
3830 ;
3831 ; GFX10-DL-LABEL: udot4_commutative:
3832 ; GFX10-DL: ; %bb.0: ; %entry
3833 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
3834 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3835 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x3c
3836 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3837 ; GFX10-DL-NEXT: s_clause 0x1
3838 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[6:7]
3839 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[4:5]
3840 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
3841 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
3842 ; GFX10-DL-NEXT: v_perm_b32 v0, v1, v1, 0xc020100
3843 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
3844 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v2, 0xc020100
3845 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
3846 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
3847 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
3848 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
3849 ; GFX10-DL-NEXT: s_endpgm
3850 ;
3851 ; GFX11-DL-LABEL: udot4_commutative:
3852 ; GFX11-DL: ; %bb.0: ; %entry
3853 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
3854 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3855 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x3c
3856 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
3857 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3858 ; GFX11-DL-NEXT: s_clause 0x1
3859 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[6:7]
3860 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[4:5]
3861 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
3862 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
3863 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v1, 0xc020100
3864 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
3865 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc020100
3866 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
3867 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_1)
3868 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s2
3869 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[0:1]
3870 ; GFX11-DL-NEXT: s_nop 0
3871 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
3872 ; GFX11-DL-NEXT: s_endpgm
3873 ptr addrspace(1) %src2,
3874 ptr addrspace(1) %src3,
3875 ptr addrspace(1) nocapture %dst) {
3876 entry:
3877 %idx = call i32 @llvm.amdgcn.workitem.id.x()
3878 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
3879 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
3880 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
3881 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
3882 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx
3883 %vec3 = load <4 x i8>, ptr addrspace(1) %gep3
3885 %v1e0 = extractelement <4 x i8> %vec1, i64 0
3886 %cv1e0 = zext i8 %v1e0 to i32
3887 %v2e0 = extractelement <4 x i8> %vec2, i64 0
3888 %cv2e0 = zext i8 %v2e0 to i32
3889 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
3891 %v1e1 = extractelement <4 x i8> %vec1, i64 1
3892 %cv1e1 = zext i8 %v1e1 to i32
3893 %v2e1 = extractelement <4 x i8> %vec2, i64 1
3894 %cv2e1 = zext i8 %v2e1 to i32
3895 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
3897 %v2e2 = extractelement <4 x i8> %vec2, i64 2
3898 %cv2e2 = zext i8 %v2e2 to i32
3899 %v1e2 = extractelement <4 x i8> %vec1, i64 2
3900 %cv1e2 = zext i8 %v1e2 to i32
3901 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
3904 %acc = load i32, ptr addrspace(1) %dst, align 4
3905 %mad1 = add i32 %mul1, %acc
3906 %mad2 = add i32 %mad1, %mul2
3907 %mad3 = add i32 %mad2, %mul3
3909 store i32 %mad3, ptr addrspace(1) %dst, align 4
3910 ret void
3911 }
3913 define amdgpu_kernel void @udot4_acc32_3src_3ele_src0(ptr addrspace(1) %src1,
3914 ; GFX7-LABEL: udot4_acc32_3src_3ele_src0:
3915 ; GFX7: ; %bb.0: ; %entry
3916 ; GFX7-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x9
3917 ; GFX7-NEXT: s_mov_b32 s11, 0xf000
3918 ; GFX7-NEXT: s_mov_b32 s14, 0
3919 ; GFX7-NEXT: s_mov_b32 s15, s11
3920 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3921 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3922 ; GFX7-NEXT: s_mov_b64 s[12:13], s[0:1]
3923 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
3924 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr64
3925 ; GFX7-NEXT: s_mov_b64 s[12:13], s[2:3]
3926 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr64
3927 ; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]
3928 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
3929 ; GFX7-NEXT: s_load_dword s0, s[6:7], 0x0
3930 ; GFX7-NEXT: s_mov_b32 s10, -1
3931 ; GFX7-NEXT: s_mov_b32 s8, s6
3932 ; GFX7-NEXT: s_mov_b32 s9, s7
3933 ; GFX7-NEXT: s_waitcnt vmcnt(2)
3934 ; GFX7-NEXT: v_bfe_u32 v1, v2, 8, 8
3935 ; GFX7-NEXT: s_waitcnt vmcnt(1)
3936 ; GFX7-NEXT: v_bfe_u32 v2, v3, 8, 8
3937 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
3938 ; GFX7-NEXT: v_mad_u32_u24 v4, v2, v2, s0
3939 ; GFX7-NEXT: v_bfe_u32 v3, v3, 16, 8
3940 ; GFX7-NEXT: s_waitcnt vmcnt(0)
3941 ; GFX7-NEXT: v_bfe_u32 v0, v0, 16, 8
3942 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v2, v4
3943 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
3944 ; GFX7-NEXT: buffer_store_dword v0, off, s[8:11], 0
3945 ; GFX7-NEXT: s_endpgm
3946 ;
3947 ; GFX8-LABEL: udot4_acc32_3src_3ele_src0:
3948 ; GFX8: ; %bb.0: ; %entry
3949 ; GFX8-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3950 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
3951 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3952 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
3953 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
3954 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3955 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
3956 ; GFX8-NEXT: v_mov_b32_e32 v1, s3
3957 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s2, v2
3958 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3959 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
3960 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
3961 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
3962 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
3963 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
3964 ; GFX8-NEXT: s_load_dword s0, s[6:7], 0x0
3965 ; GFX8-NEXT: s_waitcnt vmcnt(2)
3966 ; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 8
3967 ; GFX8-NEXT: s_waitcnt vmcnt(1)
3968 ; GFX8-NEXT: v_bfe_u32 v1, v4, 8, 8
3969 ; GFX8-NEXT: v_bfe_u32 v3, v4, 16, 8
3970 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
3971 ; GFX8-NEXT: v_mad_u32_u24 v4, v1, v1, s0
3972 ; GFX8-NEXT: v_mad_u32_u24 v1, v2, v1, v4
3973 ; GFX8-NEXT: s_waitcnt vmcnt(0)
3974 ; GFX8-NEXT: v_bfe_u32 v0, v0, 16, 8
3975 ; GFX8-NEXT: v_mad_u32_u24 v2, v3, v0, v1
3976 ; GFX8-NEXT: v_mov_b32_e32 v0, s6
3977 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
3978 ; GFX8-NEXT: flat_store_dword v[0:1], v2
3979 ; GFX8-NEXT: s_endpgm
3980 ;
3981 ; GFX9-NODL-LABEL: udot4_acc32_3src_3ele_src0:
3982 ; GFX9-NODL: ; %bb.0: ; %entry
3983 ; GFX9-NODL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
3984 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
3985 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3986 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[2:3]
3987 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[4:5]
3988 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[0:1]
3989 ; GFX9-NODL-NEXT: s_load_dword s0, s[6:7], 0x0
3990 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
3991 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
3992 ; GFX9-NODL-NEXT: v_bfe_u32 v4, v1, 8, 8
3993 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
3994 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
3995 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
3996 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
3997 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
3998 ; GFX9-NODL-NEXT: v_mad_u32_u24 v3, v4, v4, s0
3999 ; GFX9-NODL-NEXT: v_add3_u32 v1, v3, v2, v1
4000 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[6:7]
4001 ; GFX9-NODL-NEXT: s_endpgm
4002 ;
4003 ; GFX9-DL-LABEL: udot4_acc32_3src_3ele_src0:
4004 ; GFX9-DL: ; %bb.0: ; %entry
4005 ; GFX9-DL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
4006 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4007 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4008 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
4009 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[0:1]
4010 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[2:3]
4011 ; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0
4012 ; GFX9-DL-NEXT: s_mov_b32 s0, 0xc06010c
4013 ; GFX9-DL-NEXT: s_mov_b32 s1, 0xc0c0c01
4014 ; GFX9-DL-NEXT: s_mov_b32 s2, 0xc020101
4015 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
4016 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
4017 ; GFX9-DL-NEXT: v_perm_b32 v1, v1, v2, s0
4018 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
4019 ; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s1
4020 ; GFX9-DL-NEXT: v_or_b32_e32 v1, v1, v2
4021 ; GFX9-DL-NEXT: v_perm_b32 v2, v3, v3, s2
4022 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4023 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v2, v1, s3
4024 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[6:7]
4025 ; GFX9-DL-NEXT: s_endpgm
4026 ;
4027 ; GFX10-DL-LABEL: udot4_acc32_3src_3ele_src0:
4028 ; GFX10-DL: ; %bb.0: ; %entry
4029 ; GFX10-DL-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
4030 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4031 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
4032 ; GFX10-DL-NEXT: s_clause 0x2
4033 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
4034 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[0:1]
4035 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[2:3]
4036 ; GFX10-DL-NEXT: s_waitcnt_depctr 0xffe3
4037 ; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0
4038 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
4039 ; GFX10-DL-NEXT: v_perm_b32 v0, v1, v2, 0xc06010c
4040 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
4041 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc0c0c01
4042 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
4043 ; GFX10-DL-NEXT: v_or_b32_e32 v0, v0, v1
4044 ; GFX10-DL-NEXT: v_perm_b32 v1, v3, v3, 0xc020101
4045 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
4046 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s0
4047 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[6:7]
4048 ; GFX10-DL-NEXT: s_endpgm
4049 ;
4050 ; GFX11-DL-LABEL: udot4_acc32_3src_3ele_src0:
4051 ; GFX11-DL: ; %bb.0: ; %entry
4052 ; GFX11-DL-NEXT: s_load_b256 s[0:7], s[0:1], 0x24
4053 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4054 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
4055 ; GFX11-DL-NEXT: s_clause 0x2
4056 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
4057 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[0:1]
4058 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[2:3]
4059 ; GFX11-DL-NEXT: s_load_b32 s0, s[6:7], 0x0
4060 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
4061 ; GFX11-DL-NEXT: v_perm_b32 v1, v1, v2, 0xc06010c
4062 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
4063 ; GFX11-DL-NEXT: v_perm_b32 v2, v0, v0, 0xc0c0c01
4064 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v0, 0xc020101
4065 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
4066 ; GFX11-DL-NEXT: v_or_b32_e32 v1, v1, v2
4067 ; GFX11-DL-NEXT: v_mov_b32_e32 v2, 0
4068 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
4069 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v1, s0
4070 ; GFX11-DL-NEXT: global_store_b32 v2, v0, s[6:7]
4071 ; GFX11-DL-NEXT: s_nop 0
4072 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4073 ; GFX11-DL-NEXT: s_endpgm
4074 ptr addrspace(1) %src2,
4075 ptr addrspace(1) %src3,
4076 ptr addrspace(1) nocapture %dst) {
4077 entry:
4078 %idx = call i32 @llvm.amdgcn.workitem.id.x()
4079 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
4080 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
4081 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
4082 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
4083 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx
4084 %vec3 = load <4 x i8>, ptr addrspace(1) %gep3
4086 %v2e0 = extractelement <4 x i8> %vec2, i64 1
4087 %cv2e0 = zext i8 %v2e0 to i32
4088 %mul1 = mul nuw nsw i32 %cv2e0, %cv2e0
4090 %v1e1 = extractelement <4 x i8> %vec1, i64 1
4091 %cv1e1 = zext i8 %v1e1 to i32
4092 %v2e1 = extractelement <4 x i8> %vec2, i64 1
4093 %cv2e1 = zext i8 %v2e1 to i32
4094 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
4096 %v3e2 = extractelement <4 x i8> %vec3, i64 2
4097 %cv3e2 = zext i8 %v3e2 to i32
4098 %v2e2 = extractelement <4 x i8> %vec2, i64 2
4099 %cv2e2 = zext i8 %v2e2 to i32
4100 %mul3 = mul nuw nsw i32 %cv2e2, %cv3e2
4103 %acc = load i32, ptr addrspace(1) %dst, align 4
4104 %mad1 = add i32 %mul1, %acc
4105 %mad2 = add i32 %mad1, %mul2
4106 %mad3 = add i32 %mad2, %mul3
4108 store i32 %mad3, ptr addrspace(1) %dst, align 4
4109 ret void
4110 }
4112 define amdgpu_kernel void @udot4_4src(ptr addrspace(1) %src1,
4113 ; GFX7-LABEL: udot4_4src:
4114 ; GFX7: ; %bb.0: ; %entry
4115 ; GFX7-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x9
4116 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
4117 ; GFX7-NEXT: s_mov_b32 s14, 0
4118 ; GFX7-NEXT: s_mov_b32 s15, s3
4119 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4120 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4121 ; GFX7-NEXT: s_mov_b64 s[12:13], s[4:5]
4122 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
4123 ; GFX7-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr64
4124 ; GFX7-NEXT: s_mov_b64 s[12:13], s[6:7]
4125 ; GFX7-NEXT: buffer_load_dword v3, v[0:1], s[12:15], 0 addr64
4126 ; GFX7-NEXT: s_mov_b64 s[12:13], s[8:9]
4127 ; GFX7-NEXT: buffer_load_dword v4, v[0:1], s[12:15], 0 addr64
4128 ; GFX7-NEXT: s_mov_b64 s[12:13], s[10:11]
4129 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[12:15], 0 addr64
4130 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x11
4131 ; GFX7-NEXT: s_mov_b32 s2, -1
4132 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4133 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
4134 ; GFX7-NEXT: s_waitcnt vmcnt(3)
4135 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
4136 ; GFX7-NEXT: v_bfe_u32 v2, v2, 8, 8
4137 ; GFX7-NEXT: s_waitcnt vmcnt(2)
4138 ; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v3
4139 ; GFX7-NEXT: v_bfe_u32 v3, v3, 8, 8
4140 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4141 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v2, s4
4142 ; GFX7-NEXT: s_waitcnt vmcnt(1)
4143 ; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v4
4144 ; GFX7-NEXT: v_bfe_u32 v4, v4, 8, 8
4145 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v3, v1
4146 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4147 ; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v0
4148 ; GFX7-NEXT: v_bfe_u32 v0, v0, 8, 8
4149 ; GFX7-NEXT: v_mad_u32_u24 v1, v2, v4, v1
4150 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
4151 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
4152 ; GFX7-NEXT: s_endpgm
4153 ;
4154 ; GFX8-LABEL: udot4_4src:
4155 ; GFX8: ; %bb.0: ; %entry
4156 ; GFX8-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x24
4157 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 2, v0
4158 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
4159 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4160 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
4161 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
4162 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4163 ; GFX8-NEXT: flat_load_dword v3, v[0:1]
4164 ; GFX8-NEXT: v_mov_b32_e32 v1, s7
4165 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s6, v2
4166 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4167 ; GFX8-NEXT: flat_load_dword v4, v[0:1]
4168 ; GFX8-NEXT: v_mov_b32_e32 v1, s9
4169 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s8, v2
4170 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4171 ; GFX8-NEXT: flat_load_dword v5, v[0:1]
4172 ; GFX8-NEXT: v_mov_b32_e32 v1, s11
4173 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s10, v2
4174 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4175 ; GFX8-NEXT: flat_load_dword v0, v[0:1]
4176 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
4177 ; GFX8-NEXT: s_waitcnt vmcnt(3)
4178 ; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
4179 ; GFX8-NEXT: v_bfe_u32 v2, v3, 8, 8
4180 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4181 ; GFX8-NEXT: v_mad_u32_u24 v1, v1, v2, s2
4182 ; GFX8-NEXT: s_waitcnt vmcnt(2)
4183 ; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v4
4184 ; GFX8-NEXT: v_bfe_u32 v4, v4, 8, 8
4185 ; GFX8-NEXT: v_mad_u32_u24 v1, v3, v4, v1
4186 ; GFX8-NEXT: s_waitcnt vmcnt(1)
4187 ; GFX8-NEXT: v_and_b32_e32 v6, 0xff, v5
4188 ; GFX8-NEXT: v_bfe_u32 v5, v5, 8, 8
4189 ; GFX8-NEXT: v_mad_u32_u24 v1, v6, v5, v1
4190 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4191 ; GFX8-NEXT: v_and_b32_e32 v7, 0xff, v0
4192 ; GFX8-NEXT: v_bfe_u32 v0, v0, 8, 8
4193 ; GFX8-NEXT: v_mad_u32_u24 v2, v7, v0, v1
4194 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4195 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4196 ; GFX8-NEXT: flat_store_dword v[0:1], v2
4197 ; GFX8-NEXT: s_endpgm
4198 ;
4199 ; GFX9-NODL-LABEL: udot4_4src:
4200 ; GFX9-NODL: ; %bb.0: ; %entry
4201 ; GFX9-NODL-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x24
4202 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4203 ; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
4204 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
4205 ; GFX9-NODL-NEXT: global_load_dword v1, v0, s[4:5]
4206 ; GFX9-NODL-NEXT: global_load_dword v2, v0, s[6:7]
4207 ; GFX9-NODL-NEXT: global_load_dword v3, v0, s[8:9]
4208 ; GFX9-NODL-NEXT: global_load_dword v4, v0, s[10:11]
4209 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, 0
4210 ; GFX9-NODL-NEXT: s_load_dword s2, s[0:1], 0x0
4211 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(3)
4212 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1
4213 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(2)
4214 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1
4215 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
4216 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1
4217 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
4218 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1
4219 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
4220 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, s2, v2
4221 ; GFX9-NODL-NEXT: v_add3_u32 v1, v1, v3, v4
4222 ; GFX9-NODL-NEXT: global_store_dword v0, v1, s[0:1]
4223 ; GFX9-NODL-NEXT: s_endpgm
4224 ;
4225 ; GFX9-DL-LABEL: udot4_4src:
4226 ; GFX9-DL: ; %bb.0: ; %entry
4227 ; GFX9-DL-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x24
4228 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4229 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
4230 ; GFX9-DL-NEXT: s_mov_b32 s2, 0xc0c0501
4231 ; GFX9-DL-NEXT: s_mov_b32 s3, 0x5010c0c
4232 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4233 ; GFX9-DL-NEXT: global_load_dword v1, v0, s[4:5]
4234 ; GFX9-DL-NEXT: global_load_dword v2, v0, s[6:7]
4235 ; GFX9-DL-NEXT: global_load_dword v3, v0, s[8:9]
4236 ; GFX9-DL-NEXT: global_load_dword v4, v0, s[10:11]
4237 ; GFX9-DL-NEXT: s_mov_b32 s4, 0xc0c0400
4238 ; GFX9-DL-NEXT: s_load_dword s6, s[0:1], 0x0
4239 ; GFX9-DL-NEXT: s_mov_b32 s5, 0x4000c0c
4240 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, 0
4241 ; GFX9-DL-NEXT: s_waitcnt vmcnt(2)
4242 ; GFX9-DL-NEXT: v_perm_b32 v5, v2, v1, s2
4243 ; GFX9-DL-NEXT: v_perm_b32 v1, v2, v1, s4
4244 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
4245 ; GFX9-DL-NEXT: v_perm_b32 v6, v4, v3, s3
4246 ; GFX9-DL-NEXT: v_perm_b32 v2, v4, v3, s5
4247 ; GFX9-DL-NEXT: v_or_b32_e32 v3, v6, v5
4248 ; GFX9-DL-NEXT: v_or_b32_e32 v1, v2, v1
4249 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4250 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v1, v3, s6
4251 ; GFX9-DL-NEXT: global_store_dword v0, v1, s[0:1]
4252 ; GFX9-DL-NEXT: s_endpgm
4253 ;
4254 ; GFX10-DL-LABEL: udot4_4src:
4255 ; GFX10-DL: ; %bb.0: ; %entry
4256 ; GFX10-DL-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x24
4257 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4258 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x44
4259 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
4260 ; GFX10-DL-NEXT: s_clause 0x3
4261 ; GFX10-DL-NEXT: global_load_dword v1, v0, s[4:5]
4262 ; GFX10-DL-NEXT: global_load_dword v2, v0, s[6:7]
4263 ; GFX10-DL-NEXT: global_load_dword v3, v0, s[8:9]
4264 ; GFX10-DL-NEXT: global_load_dword v4, v0, s[10:11]
4265 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
4266 ; GFX10-DL-NEXT: s_waitcnt vmcnt(2)
4267 ; GFX10-DL-NEXT: v_perm_b32 v0, v2, v1, 0xc0c0501
4268 ; GFX10-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0400
4269 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
4270 ; GFX10-DL-NEXT: v_perm_b32 v5, v4, v3, 0x5010c0c
4271 ; GFX10-DL-NEXT: v_perm_b32 v2, v4, v3, 0x4000c0c
4272 ; GFX10-DL-NEXT: v_or_b32_e32 v0, v5, v0
4273 ; GFX10-DL-NEXT: v_or_b32_e32 v1, v2, v1
4274 ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0
4275 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
4276 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v1, v0, s2
4277 ; GFX10-DL-NEXT: global_store_dword v2, v0, s[0:1]
4278 ; GFX10-DL-NEXT: s_endpgm
4279 ;
4280 ; GFX11-DL-LABEL: udot4_4src:
4281 ; GFX11-DL: ; %bb.0: ; %entry
4282 ; GFX11-DL-NEXT: s_load_b256 s[4:11], s[0:1], 0x24
4283 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
4284 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x44
4285 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
4286 ; GFX11-DL-NEXT: s_clause 0x3
4287 ; GFX11-DL-NEXT: global_load_b32 v1, v0, s[4:5]
4288 ; GFX11-DL-NEXT: global_load_b32 v2, v0, s[6:7]
4289 ; GFX11-DL-NEXT: global_load_b32 v3, v0, s[8:9]
4290 ; GFX11-DL-NEXT: global_load_b32 v0, v0, s[10:11]
4291 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
4292 ; GFX11-DL-NEXT: s_waitcnt vmcnt(2)
4293 ; GFX11-DL-NEXT: v_perm_b32 v4, v2, v1, 0xc0c0501
4294 ; GFX11-DL-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0400
4295 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
4296 ; GFX11-DL-NEXT: v_perm_b32 v5, v0, v3, 0x5010c0c
4297 ; GFX11-DL-NEXT: v_perm_b32 v0, v0, v3, 0x4000c0c
4298 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
4299 ; GFX11-DL-NEXT: v_or_b32_e32 v2, v5, v4
4300 ; GFX11-DL-NEXT: v_or_b32_e32 v0, v0, v1
4301 ; GFX11-DL-NEXT: v_mov_b32_e32 v1, 0
4302 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
4303 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_2)
4304 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, s2
4305 ; GFX11-DL-NEXT: global_store_b32 v1, v0, s[0:1]
4306 ; GFX11-DL-NEXT: s_nop 0
4307 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4308 ; GFX11-DL-NEXT: s_endpgm
4309 ptr addrspace(1) %src2,
4310 ptr addrspace(1) %src3,
4311 ptr addrspace(1) %src4,
4312 ptr addrspace(1) nocapture %dst) {
4313 entry:
4314 %idx = call i32 @llvm.amdgcn.workitem.id.x()
4316 %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %src1, i32 %idx
4317 %vec1 = load <4 x i8>, ptr addrspace(1) %gep1
4318 %gep2 = getelementptr <4 x i8>, ptr addrspace(1) %src2, i32 %idx
4319 %vec2 = load <4 x i8>, ptr addrspace(1) %gep2
4320 %gep3 = getelementptr <4 x i8>, ptr addrspace(1) %src3, i32 %idx
4321 %vec3 = load <4 x i8>, ptr addrspace(1) %gep3
4322 %gep4 = getelementptr <4 x i8>, ptr addrspace(1) %src4, i32 %idx
4323 %vec4 = load <4 x i8>, ptr addrspace(1) %gep4
4326 %v1e0 = extractelement <4 x i8> %vec1, i64 0
4327 %cv1e0 = zext i8 %v1e0 to i32
4328 %v1e1 = extractelement <4 x i8> %vec1, i64 1
4329 %cv1e1 = zext i8 %v1e1 to i32
4330 %mul1 = mul nuw nsw i32 %cv1e0, %cv1e1
4332 %v2e0 = extractelement <4 x i8> %vec2, i64 0
4333 %cv2e0 = zext i8 %v2e0 to i32
4334 %v2e1 = extractelement <4 x i8> %vec2, i64 1
4335 %cv2e1 = zext i8 %v2e1 to i32
4336 %mul2 = mul nuw nsw i32 %cv2e0, %cv2e1
4338 %v3e0 = extractelement <4 x i8> %vec3, i64 0
4339 %cv3e0 = zext i8 %v3e0 to i32
4340 %v3e1 = extractelement <4 x i8> %vec3, i64 1
4341 %cv3e1 = zext i8 %v3e1 to i32
4342 %mul3 = mul nuw nsw i32 %cv3e0, %cv3e1
4344 %v4e0 = extractelement <4 x i8> %vec4, i64 0
4345 %cv4e0 = zext i8 %v4e0 to i32
4346 %v4e1 = extractelement <4 x i8> %vec4, i64 1
4347 %cv4e1 = zext i8 %v4e1 to i32
4348 %mul4 = mul nuw nsw i32 %cv4e0, %cv4e1
4351 %acc = load i32, ptr addrspace(1) %dst, align 4
4352 %mad1 = add i32 %mul1, %acc
4353 %mad2 = add i32 %mad1, %mul2
4354 %mad3 = add i32 %mad2, %mul3
4355 %mad4 = add i32 %mad3, %mul4
4357 store i32 %mad4, ptr addrspace(1) %dst, align 4
4358 ret void
4359 }
4361 define amdgpu_kernel void @udot4_acc32_multi(ptr addrspace(1) %src1,
4362 ; GFX7-LABEL: udot4_acc32_multi:
4363 ; GFX7: ; %bb.0: ; %entry
4364 ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
4365 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd
4366 ; GFX7-NEXT: s_mov_b32 s3, 0xf000
4367 ; GFX7-NEXT: s_mov_b32 s10, 0
4368 ; GFX7-NEXT: s_mov_b32 s11, s3
4369 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4370 ; GFX7-NEXT: s_mov_b64 s[8:9], s[4:5]
4371 ; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
4372 ; GFX7-NEXT: v_mov_b32_e32 v1, 0
4373 ; GFX7-NEXT: s_mov_b64 s[4:5], s[6:7]
4374 ; GFX7-NEXT: s_mov_b64 s[6:7], s[10:11]
4375 ; GFX7-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[8:11], 0 addr64
4376 ; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
4377 ; GFX7-NEXT: s_load_dword s4, s[0:1], 0x0
4378 ; GFX7-NEXT: s_mov_b32 s2, -1
4379 ; GFX7-NEXT: s_waitcnt vmcnt(1)
4380 ; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
4381 ; GFX7-NEXT: s_waitcnt vmcnt(0)
4382 ; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v0
4383 ; GFX7-NEXT: v_bfe_u32 v7, v2, 16, 8
4384 ; GFX7-NEXT: v_bfe_u32 v8, v0, 16, 8
4385 ; GFX7-NEXT: s_waitcnt lgkmcnt(0)
4386 ; GFX7-NEXT: v_mad_u32_u24 v1, v1, v4, s4
4387 ; GFX7-NEXT: v_and_b32_e32 v9, 0xff, v3
4388 ; GFX7-NEXT: v_mad_u32_u24 v1, v7, v8, v1
4389 ; GFX7-NEXT: v_bfe_u32 v11, v3, 16, 8
4390 ; GFX7-NEXT: v_mad_u32_u24 v1, v9, v4, v1
4391 ; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 8
4392 ; GFX7-NEXT: v_bfe_u32 v6, v0, 8, 8
4393 ; GFX7-NEXT: v_mad_u32_u24 v1, v11, v8, v1
4394 ; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
4395 ; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v0
4396 ; GFX7-NEXT: v_mad_u32_u24 v1, v5, v6, v1
4397 ; GFX7-NEXT: v_bfe_u32 v10, v3, 8, 8
4398 ; GFX7-NEXT: v_mad_u32_u24 v1, v2, v0, v1
4399 ; GFX7-NEXT: v_lshrrev_b32_e32 v3, 24, v3
4400 ; GFX7-NEXT: v_mad_u32_u24 v1, v10, v6, v1
4401 ; GFX7-NEXT: v_mad_u32_u24 v0, v3, v0, v1
4402 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
4403 ; GFX7-NEXT: s_endpgm
4404 ;
4405 ; GFX8-LABEL: udot4_acc32_multi:
4406 ; GFX8: ; %bb.0: ; %entry
4407 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4408 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4409 ; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4410 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4411 ; GFX8-NEXT: v_mov_b32_e32 v1, s5
4412 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v2
4413 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
4414 ; GFX8-NEXT: v_mov_b32_e32 v3, s7
4415 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v2
4416 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
4417 ; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
4418 ; GFX8-NEXT: flat_load_dword v2, v[2:3]
4419 ; GFX8-NEXT: s_load_dword s2, s[0:1], 0x0
4420 ; GFX8-NEXT: s_waitcnt vmcnt(1)
4421 ; GFX8-NEXT: v_and_b32_e32 v3, 0xff, v0
4422 ; GFX8-NEXT: s_waitcnt vmcnt(0)
4423 ; GFX8-NEXT: v_and_b32_e32 v4, 0xff, v2
4424 ; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 8
4425 ; GFX8-NEXT: v_bfe_u32 v8, v2, 16, 8
4426 ; GFX8-NEXT: s_waitcnt lgkmcnt(0)
4427 ; GFX8-NEXT: v_mad_u32_u24 v3, v3, v4, s2
4428 ; GFX8-NEXT: v_and_b32_e32 v9, 0xff, v1
4429 ; GFX8-NEXT: v_mad_u32_u24 v3, v7, v8, v3
4430 ; GFX8-NEXT: v_bfe_u32 v11, v1, 16, 8
4431 ; GFX8-NEXT: v_mad_u32_u24 v3, v9, v4, v3
4432 ; GFX8-NEXT: v_bfe_u32 v5, v0, 8, 8
4433 ; GFX8-NEXT: v_bfe_u32 v6, v2, 8, 8
4434 ; GFX8-NEXT: v_mad_u32_u24 v3, v11, v8, v3
4435 ; GFX8-NEXT: v_lshrrev_b32_e32 v0, 24, v0
4436 ; GFX8-NEXT: v_lshrrev_b32_e32 v2, 24, v2
4437 ; GFX8-NEXT: v_mad_u32_u24 v3, v5, v6, v3
4438 ; GFX8-NEXT: v_bfe_u32 v10, v1, 8, 8
4439 ; GFX8-NEXT: v_mad_u32_u24 v0, v0, v2, v3
4440 ; GFX8-NEXT: v_lshrrev_b32_e32 v1, 24, v1
4441 ; GFX8-NEXT: v_mad_u32_u24 v0, v10, v6, v0
4442 ; GFX8-NEXT: v_mad_u32_u24 v2, v1, v2, v0
4443 ; GFX8-NEXT: v_mov_b32_e32 v0, s0
4444 ; GFX8-NEXT: v_mov_b32_e32 v1, s1
4445 ; GFX8-NEXT: flat_store_dword v[0:1], v2
4446 ; GFX8-NEXT: s_endpgm
4447 ;
4448 ; GFX9-NODL-LABEL: udot4_acc32_multi:
4449 ; GFX9-NODL: ; %bb.0: ; %entry
4450 ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4451 ; GFX9-NODL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
4452 ; GFX9-NODL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4453 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
4454 ; GFX9-NODL-NEXT: global_load_dword v3, v2, s[6:7]
4455 ; GFX9-NODL-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
4456 ; GFX9-NODL-NEXT: s_load_dword s0, s[2:3], 0x0
4457 ; GFX9-NODL-NEXT: v_mov_b32_e32 v2, 0
4458 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(1)
4459 ; GFX9-NODL-NEXT: v_and_b32_e32 v4, 0xff, v3
4460 ; GFX9-NODL-NEXT: v_bfe_u32 v6, v3, 16, 8
4461 ; GFX9-NODL-NEXT: v_bfe_u32 v5, v3, 8, 8
4462 ; GFX9-NODL-NEXT: v_lshrrev_b32_e32 v3, 24, v3
4463 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0)
4464 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v7, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4465 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v9, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:DWORD
4466 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v8, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
4467 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
4468 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
4469 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
4470 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v6, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:DWORD
4471 ; GFX9-NODL-NEXT: v_mul_u32_u24_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
4472 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0)
4473 ; GFX9-NODL-NEXT: v_add3_u32 v3, v7, s0, v9
4474 ; GFX9-NODL-NEXT: v_add3_u32 v3, v3, v4, v6
4475 ; GFX9-NODL-NEXT: v_add3_u32 v0, v8, v3, v0
4476 ; GFX9-NODL-NEXT: v_add3_u32 v0, v0, v5, v1
4477 ; GFX9-NODL-NEXT: global_store_dword v2, v0, s[2:3]
4478 ; GFX9-NODL-NEXT: s_endpgm
4479 ;
4480 ; GFX9-DL-LABEL: udot4_acc32_multi:
4481 ; GFX9-DL: ; %bb.0: ; %entry
4482 ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4483 ; GFX9-DL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
4484 ; GFX9-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4485 ; GFX9-DL-NEXT: s_mov_b32 s0, 0x6040200
4486 ; GFX9-DL-NEXT: s_mov_b32 s1, 0x2000200
4487 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4488 ; GFX9-DL-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
4489 ; GFX9-DL-NEXT: global_load_dword v3, v2, s[6:7]
4490 ; GFX9-DL-NEXT: s_load_dword s5, s[2:3], 0x0
4491 ; GFX9-DL-NEXT: s_mov_b32 s4, 0x7050301
4492 ; GFX9-DL-NEXT: s_mov_b32 s6, 0x3010301
4493 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, 0
4494 ; GFX9-DL-NEXT: s_waitcnt vmcnt(1)
4495 ; GFX9-DL-NEXT: v_perm_b32 v4, v1, v0, s0
4496 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0)
4497 ; GFX9-DL-NEXT: v_perm_b32 v5, v3, v3, s1
4498 ; GFX9-DL-NEXT: v_perm_b32 v0, v1, v0, s4
4499 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0)
4500 ; GFX9-DL-NEXT: v_dot4_u32_u8 v1, v4, v5, s5
4501 ; GFX9-DL-NEXT: v_perm_b32 v3, v3, v3, s6
4502 ; GFX9-DL-NEXT: v_dot4_u32_u8 v0, v0, v3, v1
4503 ; GFX9-DL-NEXT: global_store_dword v2, v0, s[2:3]
4504 ; GFX9-DL-NEXT: s_endpgm
4505 ;
4506 ; GFX10-DL-LABEL: udot4_acc32_multi:
4507 ; GFX10-DL: ; %bb.0: ; %entry
4508 ; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
4509 ; GFX10-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4510 ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34
4511 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
4512 ; GFX10-DL-NEXT: s_clause 0x1
4513 ; GFX10-DL-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
4514 ; GFX10-DL-NEXT: global_load_dword v3, v2, s[6:7]
4515 ; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0
4516 ; GFX10-DL-NEXT: s_waitcnt vmcnt(1)
4517 ; GFX10-DL-NEXT: v_perm_b32 v2, v1, v0, 0x6040200
4518 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0)
4519 ; GFX10-DL-NEXT: v_perm_b32 v4, v3, v3, 0x2000200
4520 ; GFX10-DL-NEXT: v_perm_b32 v0, v1, v0, 0x7050301
4521 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0)
4522 ; GFX10-DL-NEXT: v_dot4_u32_u8 v1, v2, v4, s2
4523 ; GFX10-DL-NEXT: v_perm_b32 v2, v3, v3, 0x3010301
4524 ; GFX10-DL-NEXT: v_mov_b32_e32 v3, 0
4525 ; GFX10-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v1
4526 ; GFX10-DL-NEXT: global_store_dword v3, v0, s[0:1]
4527 ; GFX10-DL-NEXT: s_endpgm
4528 ;
4529 ; GFX11-DL-LABEL: udot4_acc32_multi:
4530 ; GFX11-DL: ; %bb.0: ; %entry
4531 ; GFX11-DL-NEXT: s_load_b128 s[4:7], s[0:1], 0x24
4532 ; GFX11-DL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
4533 ; GFX11-DL-NEXT: s_load_b64 s[0:1], s[0:1], 0x34
4534 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
4535 ; GFX11-DL-NEXT: s_clause 0x1
4536 ; GFX11-DL-NEXT: global_load_b64 v[0:1], v2, s[4:5]
4537 ; GFX11-DL-NEXT: global_load_b32 v2, v2, s[6:7]
4538 ; GFX11-DL-NEXT: s_load_b32 s2, s[0:1], 0x0
4539 ; GFX11-DL-NEXT: s_waitcnt vmcnt(1)
4540 ; GFX11-DL-NEXT: v_perm_b32 v3, v1, v0, 0x6040200
4541 ; GFX11-DL-NEXT: s_waitcnt vmcnt(0)
4542 ; GFX11-DL-NEXT: v_perm_b32 v4, v2, v2, 0x2000200
4543 ; GFX11-DL-NEXT: v_perm_b32 v0, v1, v0, 0x7050301
4544 ; GFX11-DL-NEXT: v_perm_b32 v2, v2, v2, 0x3010301
4545 ; GFX11-DL-NEXT: s_waitcnt lgkmcnt(0)
4546 ; GFX11-DL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
4547 ; GFX11-DL-NEXT: v_dot4_u32_u8 v1, v3, v4, s2
4548 ; GFX11-DL-NEXT: v_mov_b32_e32 v3, 0
4549 ; GFX11-DL-NEXT: v_dot4_u32_u8 v0, v0, v2, v1
4550 ; GFX11-DL-NEXT: global_store_b32 v3, v0, s[0:1]
4551 ; GFX11-DL-NEXT: s_nop 0
4552 ; GFX11-DL-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
4553 ; GFX11-DL-NEXT: s_endpgm
4554 ptr addrspace(1) %src2,
4555 ptr addrspace(1) nocapture %dst) {
4556 entry:
4557 %idx = call i32 @llvm.amdgcn.workitem.id.x()
4558 %gep1 = getelementptr <8 x i8>, ptr addrspace(1) %src1, i32 %idx
4559 %vec1 = load <8 x i8>, ptr addrspace(1) %gep1
4560 %gep2 = getelementptr <8 x i8>, ptr addrspace(1) %src2, i32 %idx
4561 %vec2 = load <8 x i8>, ptr addrspace(1) %gep2
4563 %v1e0 = extractelement <8 x i8> %vec1, i64 0
4564 %cv1e0 = zext i8 %v1e0 to i32
4565 %v2e0 = extractelement <8 x i8> %vec2, i64 0
4566 %cv2e0 = zext i8 %v2e0 to i32
4567 %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
4569 %v1e1 = extractelement <8 x i8> %vec1, i64 1
4570 %cv1e1 = zext i8 %v1e1 to i32
4571 %v2e1 = extractelement <8 x i8> %vec2, i64 1
4572 %cv2e1 = zext i8 %v2e1 to i32
4573 %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
4575 %v1e2 = extractelement <8 x i8> %vec1, i64 2
4576 %cv1e2 = zext i8 %v1e2 to i32
4577 %v2e2 = extractelement <8 x i8> %vec2, i64 2
4578 %cv2e2 = zext i8 %v2e2 to i32
4579 %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
4581 %v1e3 = extractelement <8 x i8> %vec1, i64 3
4582 %cv1e3 = zext i8 %v1e3 to i32
4583 %v2e3 = extractelement <8 x i8> %vec2, i64 3
4584 %cv2e3 = zext i8 %v2e3 to i32
4585 %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
4587 %v1e4 = extractelement <8 x i8> %vec1, i64 4
4588 %cv1e4 = zext i8 %v1e4 to i32
4589 %v2e4 = extractelement <8 x i8> %vec2, i64 4
4590 %cv2e4 = zext i8 %v2e4 to i32
4591 %mul5 = mul nuw nsw i32 %cv1e4, %cv2e0
4593 %v1e5 = extractelement <8 x i8> %vec1, i64 5
4594 %cv1e5 = zext i8 %v1e5 to i32
4595 %v2e5 = extractelement <8 x i8> %vec2, i64 5
4596 %cv2e5 = zext i8 %v2e5 to i32
4597 %mul6 = mul nuw nsw i32 %cv1e5, %cv2e1
4599 %v1e6 = extractelement <8 x i8> %vec1, i64 6
4600 %cv1e6 = zext i8 %v1e6 to i32
4601 %v2e6 = extractelement <8 x i8> %vec2, i64 6
4602 %cv2e6 = zext i8 %v2e6 to i32
4603 %mul7 = mul nuw nsw i32 %cv1e6, %cv2e2
4605 %v1e7 = extractelement <8 x i8> %vec1, i64 7
4606 %cv1e7 = zext i8 %v1e7 to i32
4607 %v2e7 = extractelement <8 x i8> %vec2, i64 7
4608 %cv2e7 = zext i8 %v2e7 to i32
4609 %mul8 = mul nuw nsw i32 %cv1e7, %cv2e3
4611 %acc = load i32, ptr addrspace(1) %dst, align 4
4612 %mad11 = add i32 %mul1, %acc
4613 %mad21 = add i32 %mad11, %mul3
4614 %mad31 = add i32 %mad21, %mul5
4615 %mad41 = add i32 %mad31, %mul7
4616 %mad12 = add i32 %mul2, %mad41
4617 %mad22 = add i32 %mad12, %mul4
4618 %mad32 = add i32 %mad22, %mul6
4619 %mad42 = add i32 %mad32, %mul8
4621 store i32 %mad42, ptr addrspace(1) %dst, align 4
4622 ret void
4623 }
4625 declare i32 @llvm.amdgcn.workitem.id.x()