1 ; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK -check-prefix=SI %s
2 ; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=CHECK -check-prefix=VI %s
4 ; Check that WQM isn't triggered by image load/store intrinsics.
6 ;CHECK-LABEL: {{^}}test1:
8 define amdgpu_ps <4 x float> @test1(<8 x i32> inreg %rsrc, i32 %c) {
10 %tex = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0)
11 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %tex, i32 15, i32 %c, <8 x i32> %rsrc, i32 0, i32 0)
15 ; Check that WQM is triggered by code calculating inputs to image samples and is disabled as soon as possible
17 ;CHECK-LABEL: {{^}}test2:
18 ;CHECK-NEXT: ; %main_body
19 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
20 ;CHECK-NEXT: s_wqm_b64 exec, exec
22 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
27 define amdgpu_ps <4 x float> @test2(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 {
29 %inst23 = extractelement <2 x float> %pos, i32 0
30 %inst24 = extractelement <2 x float> %pos, i32 1
31 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)
32 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)
33 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0)
34 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0)
35 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
39 ; ... but disabled for stores (and, in this simple case, not re-enabled) ...
41 ;CHECK-LABEL: {{^}}test3:
42 ;CHECK-NEXT: ; %main_body
43 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
44 ;CHECK-NEXT: s_wqm_b64 exec, exec
45 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
50 define amdgpu_ps <4 x float> @test3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %c) {
52 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
53 %tex.1 = bitcast <4 x float> %tex to <4 x i32>
54 %tex.2 = extractelement <4 x i32> %tex.1, i32 0
56 call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %tex, <4 x i32> undef, i32 %tex.2, i32 0, i1 0, i1 0)
61 ; ... and disabled for export.
63 ;CHECK-LABEL: {{^}}test3x:
64 ;CHECK-NEXT: ; %main_body
65 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
66 ;CHECK-NEXT: s_wqm_b64 exec, exec
67 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
72 define amdgpu_ps void @test3x(i32 inreg, i32 inreg, i32 inreg, i32 inreg %m0, <8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <2 x float> %pos) #6 {
74 %inst23 = extractelement <2 x float> %pos, i32 0
75 %inst24 = extractelement <2 x float> %pos, i32 1
76 %inst25 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 0, i32 0, i32 %m0)
77 %inst26 = tail call float @llvm.amdgcn.interp.p2(float %inst25, float %inst24, i32 0, i32 0, i32 %m0)
78 %inst28 = tail call float @llvm.amdgcn.interp.p1(float %inst23, i32 1, i32 0, i32 %m0)
79 %inst29 = tail call float @llvm.amdgcn.interp.p2(float %inst28, float %inst24, i32 1, i32 0, i32 %m0)
80 %tex = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %inst26, float %inst29, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
81 %tex.0 = extractelement <4 x float> %tex, i32 0
82 %tex.1 = extractelement <4 x float> %tex, i32 1
83 %tex.2 = extractelement <4 x float> %tex, i32 2
84 %tex.3 = extractelement <4 x float> %tex, i32 3
85 call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tex.0, float %tex.1, float %tex.2, float %tex.3, i1 true, i1 true)
89 ; Check that WQM is re-enabled when required.
91 ;CHECK-LABEL: {{^}}test4:
92 ;CHECK-NEXT: ; %main_body
93 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
94 ;CHECK-NEXT: s_wqm_b64 exec, exec
95 ;CHECK: v_mul_lo_u32 [[MUL:v[0-9]+]], v0, v1
96 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
98 ;CHECK: s_wqm_b64 exec, exec
101 define amdgpu_ps <4 x float> @test4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, i32 %c, i32 %d, float %data) {
103 %c.1 = mul i32 %c, %d
105 call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> undef, <4 x i32> undef, i32 %c.1, i32 0, i1 0, i1 0)
106 %c.1.bc = bitcast i32 %c.1 to float
107 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.1.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
108 %tex0 = extractelement <4 x float> %tex, i32 0
109 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
110 ret <4 x float> %dtex
113 ; Check that WQM is triggered by the wqm intrinsic.
115 ;CHECK-LABEL: {{^}}test5:
116 ;CHECK: s_wqm_b64 exec, exec
117 ;CHECK: buffer_load_dword
118 ;CHECK: buffer_load_dword
119 ;CHECK: v_add_f32_e32
120 define amdgpu_ps float @test5(i32 inreg %idx0, i32 inreg %idx1) {
122 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
123 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
124 %out = fadd float %src0, %src1
125 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)
129 ; Check that the wqm intrinsic works correctly for integers.
131 ;CHECK-LABEL: {{^}}test6:
132 ;CHECK: s_wqm_b64 exec, exec
133 ;CHECK: buffer_load_dword
134 ;CHECK: buffer_load_dword
135 ;CHECK: v_add_f32_e32
136 define amdgpu_ps float @test6(i32 inreg %idx0, i32 inreg %idx1) {
138 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
139 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
140 %out = fadd float %src0, %src1
141 %out.0 = bitcast float %out to i32
142 %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0)
143 %out.2 = bitcast i32 %out.1 to float
147 ; Check that WWM is triggered by the wwm intrinsic.
149 ;CHECK-LABEL: {{^}}test_wwm1:
150 ;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
151 ;CHECK: buffer_load_dword
152 ;CHECK: buffer_load_dword
153 ;CHECK: v_add_f32_e32
154 define amdgpu_ps float @test_wwm1(i32 inreg %idx0, i32 inreg %idx1) {
156 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
157 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
158 %out = fadd float %src0, %src1
159 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
163 ; Same as above, but with an integer type.
165 ;CHECK-LABEL: {{^}}test_wwm2:
166 ;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
167 ;CHECK: buffer_load_dword
168 ;CHECK: buffer_load_dword
169 ;CHECK: v_add_{{[iu]}}32_e32
170 define amdgpu_ps float @test_wwm2(i32 inreg %idx0, i32 inreg %idx1) {
172 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
173 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
174 %src0.0 = bitcast float %src0 to i32
175 %src1.0 = bitcast float %src1 to i32
176 %out = add i32 %src0.0, %src1.0
177 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out)
178 %out.1 = bitcast i32 %out.0 to float
182 ; Check that we don't leave WWM on for computations that don't require WWM,
183 ; since that will lead clobbering things that aren't supposed to be clobbered
184 ; in cases like this.
186 ;CHECK-LABEL: {{^}}test_wwm3:
187 ;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
188 ;CHECK: buffer_load_dword
189 ;CHECK: v_add_f32_e32
190 ;CHECK: s_mov_b64 exec, [[ORIG]]
191 ;CHECK: v_add_f32_e32
192 define amdgpu_ps float @test_wwm3(i32 inreg %idx) {
194 ; use mbcnt to make sure the branch is divergent
195 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
196 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
197 %cc = icmp uge i32 %hi, 32
198 br i1 %cc, label %endif, label %if
201 %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
202 %out = fadd float %src, %src
203 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
204 %out.1 = fadd float %src, %out.0
208 %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]
212 ; Check that WWM writes aren't coalesced with non-WWM writes, since the WWM
213 ; write could clobber disabled channels in the non-WWM one.
215 ;CHECK-LABEL: {{^}}test_wwm4:
216 ;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
217 ;CHECK: buffer_load_dword
218 ;CHECK: v_add_f32_e32
219 ;CHECK: s_mov_b64 exec, [[ORIG]]
220 ;CHECK-NEXT: v_mov_b32_e32
221 define amdgpu_ps float @test_wwm4(i32 inreg %idx) {
223 ; use mbcnt to make sure the branch is divergent
224 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
225 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
226 %cc = icmp uge i32 %hi, 32
227 br i1 %cc, label %endif, label %if
230 %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
231 %out = fadd float %src, %src
232 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
236 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
240 ; Make sure the transition from Exact to WWM then WQM works properly.
242 ;CHECK-LABEL: {{^}}test_wwm5:
243 ;CHECK: buffer_load_dword
244 ;CHECK: buffer_store_dword
245 ;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
246 ;CHECK: buffer_load_dword
247 ;CHECK: v_add_f32_e32
248 ;CHECK: s_mov_b64 exec, [[ORIG]]
249 ;CHECK: s_wqm_b64 exec, exec
250 define amdgpu_ps float @test_wwm5(i32 inreg %idx0, i32 inreg %idx1) {
252 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
253 call void @llvm.amdgcn.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
254 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
255 %temp = fadd float %src1, %src1
256 %temp.0 = call float @llvm.amdgcn.wwm.f32(float %temp)
257 %out = fadd float %temp.0, %temp.0
258 %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)
262 ; Check that WWM is turned on correctly across basic block boundaries.
263 ; if..then..endif version
265 ;CHECK-LABEL: {{^}}test_wwm6_then:
266 ;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
267 ;SI-CHECK: buffer_load_dword
268 ;VI-CHECK: flat_load_dword
269 ;CHECK: s_mov_b64 exec, [[ORIG]]
271 ;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1
272 ;SI-CHECK: buffer_load_dword
273 ;VI-CHECK: flat_load_dword
274 ;CHECK: v_add_f32_e32
275 ;CHECK: s_mov_b64 exec, [[ORIG2]]
276 define amdgpu_ps float @test_wwm6_then() {
278 %src0 = load volatile float, float addrspace(1)* undef
279 ; use mbcnt to make sure the branch is divergent
280 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
281 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
282 %cc = icmp uge i32 %hi, 32
283 br i1 %cc, label %endif, label %if
286 %src1 = load volatile float, float addrspace(1)* undef
287 %out = fadd float %src0, %src1
288 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
292 %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
296 ; Check that WWM is turned on correctly across basic block boundaries.
299 ;CHECK-LABEL: {{^}}test_wwm6_loop:
300 ;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
301 ;SI-CHECK: buffer_load_dword
302 ;VI-CHECK: flat_load_dword
303 ;CHECK: s_mov_b64 exec, [[ORIG]]
305 ;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1
306 ;SI-CHECK: buffer_load_dword
307 ;VI-CHECK: flat_load_dword
308 ;CHECK: s_mov_b64 exec, [[ORIG2]]
309 define amdgpu_ps float @test_wwm6_loop() {
311 %src0 = load volatile float, float addrspace(1)* undef
312 ; use mbcnt to make sure the branch is divergent
313 %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
314 %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
318 %counter = phi i32 [ %lo, %main_body ], [ %counter.1, %loop ]
319 %src1 = load volatile float, float addrspace(1)* undef
320 %out = fadd float %src0, %src1
321 %out.0 = call float @llvm.amdgcn.wwm.f32(float %out)
322 %counter.1 = sub i32 %counter, 1
323 %cc = icmp ne i32 %counter.1, 0
324 br i1 %cc, label %loop, label %endloop
330 ; Check that @llvm.amdgcn.set.inactive disables WWM.
332 ;CHECK-LABEL: {{^}}test_set_inactive1:
333 ;CHECK: buffer_load_dword
334 ;CHECK: s_not_b64 exec, exec
335 ;CHECK: v_mov_b32_e32
336 ;CHECK: s_not_b64 exec, exec
337 ;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
338 ;CHECK: v_add_{{[iu]}}32_e32
339 define amdgpu_ps void @test_set_inactive1(i32 inreg %idx) {
341 %src = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
342 %src.0 = bitcast float %src to i32
343 %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0)
344 %out = add i32 %src.1, %src.1
345 %out.0 = call i32 @llvm.amdgcn.wwm.i32(i32 %out)
346 %out.1 = bitcast i32 %out.0 to float
347 call void @llvm.amdgcn.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
351 ; Check that enabling WQM anywhere enables WQM for the set.inactive source.
353 ;CHECK-LABEL: {{^}}test_set_inactive2:
354 ;CHECK: s_wqm_b64 exec, exec
355 ;CHECK: buffer_load_dword
356 ;CHECK: buffer_load_dword
357 define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) {
359 %src1 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
360 %src1.0 = bitcast float %src1 to i32
361 %src1.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src1.0, i32 undef)
362 %src0 = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i1 0, i1 0)
363 %src0.0 = bitcast float %src0 to i32
364 %src0.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %src0.0)
365 %out = add i32 %src0.1, %src1.1
366 %out.0 = bitcast i32 %out to float
367 call void @llvm.amdgcn.buffer.store.f32(float %out.0, <4 x i32> undef, i32 %idx1, i32 0, i1 0, i1 0)
371 ; Check a case of one branch of an if-else requiring WQM, the other requiring
374 ; Note: In this particular case, the save-and-restore could be avoided if the
375 ; analysis understood that the two branches of the if-else are mutually
378 ;CHECK-LABEL: {{^}}test_control_flow_0:
379 ;CHECK-NEXT: ; %main_body
380 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
381 ;CHECK-NEXT: s_wqm_b64 exec, exec
383 ;CHECK: s_and_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]], [[ORIG]]
385 ;CHECK: s_mov_b64 exec, [[SAVED]]
389 define amdgpu_ps float @test_control_flow_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {
391 %cmp = icmp eq i32 %z, 0
392 br i1 %cmp, label %IF, label %ELSE
395 %c.bc = bitcast i32 %c to float
396 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
397 %tex0 = extractelement <4 x float> %tex, i32 0
398 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
399 %data.if = extractelement <4 x float> %dtex, i32 0
403 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i1 0, i1 0)
407 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ]
411 ; Reverse branch order compared to the previous test.
413 ;CHECK-LABEL: {{^}}test_control_flow_1:
414 ;CHECK-NEXT: ; %main_body
415 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
416 ;CHECK-NEXT: s_wqm_b64 exec, exec
421 ;CHECK-NEXT: s_or_saveexec_b64 [[SAVED:s\[[0-9]+:[0-9]+\]]],
422 ;CHECK-NEXT: s_and_b64 exec, exec, [[ORIG]]
423 ;CHECK-NEXT: s_and_b64 [[SAVED]], exec, [[SAVED]]
424 ;CHECK-NEXT: s_xor_b64 exec, exec, [[SAVED]]
425 ;CHECK-NEXT: mask branch [[END_BB:BB[0-9]+_[0-9]+]]
426 ;CHECK-NEXT: s_cbranch_execz [[END_BB]]
427 ;CHECK-NEXT: BB{{[0-9]+_[0-9]+}}: ; %ELSE
429 ;CHECK: [[END_BB]]: ; %END
430 ;CHECK: s_or_b64 exec, exec,
431 ;CHECK: v_mov_b32_e32 v0
433 define amdgpu_ps float @test_control_flow_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {
435 %cmp = icmp eq i32 %z, 0
436 br i1 %cmp, label %ELSE, label %IF
439 %c.bc = bitcast i32 %c to float
440 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
441 %tex0 = extractelement <4 x float> %tex, i32 0
442 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
443 %data.if = extractelement <4 x float> %dtex, i32 0
447 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 %c, i32 0, i1 0, i1 0)
451 %r = phi float [ %data.if, %IF ], [ %data, %ELSE ]
455 ; Check that branch conditions are properly marked as needing WQM...
457 ;CHECK-LABEL: {{^}}test_control_flow_2:
458 ;CHECK-NEXT: ; %main_body
459 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
460 ;CHECK-NEXT: s_wqm_b64 exec, exec
461 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
463 ;CHECK: s_wqm_b64 exec, exec
465 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
467 ;CHECK: s_wqm_b64 exec, exec
469 define amdgpu_ps <4 x float> @test_control_flow_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, <3 x i32> %idx, <2 x float> %data, i32 %coord) {
471 %idx.1 = extractelement <3 x i32> %idx, i32 0
472 %data.1 = extractelement <2 x float> %data, i32 0
473 call void @llvm.amdgcn.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i1 0, i1 0)
475 ; The load that determines the branch (and should therefore be WQM) is
476 ; surrounded by stores that require disabled WQM.
477 %idx.2 = extractelement <3 x i32> %idx, i32 1
478 %z = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 %idx.2, i32 0, i1 0, i1 0)
480 %idx.3 = extractelement <3 x i32> %idx, i32 2
481 %data.3 = extractelement <2 x float> %data, i32 1
482 call void @llvm.amdgcn.buffer.store.f32(float %data.3, <4 x i32> undef, i32 %idx.3, i32 0, i1 0, i1 0)
484 %cc = fcmp ogt float %z, 0.0
485 br i1 %cc, label %IF, label %ELSE
488 %coord.IF = mul i32 %coord, 3
492 %coord.ELSE = mul i32 %coord, 4
496 %coord.END = phi i32 [ %coord.IF, %IF ], [ %coord.ELSE, %ELSE ]
497 %coord.END.bc = bitcast i32 %coord.END to float
498 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord.END.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
502 ; ... but only if they really do need it.
504 ;CHECK-LABEL: {{^}}test_control_flow_3:
505 ;CHECK-NEXT: ; %main_body
506 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
507 ;CHECK-NEXT: s_wqm_b64 exec, exec
509 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
513 define amdgpu_ps float @test_control_flow_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %coord) {
515 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
516 %tex0 = extractelement <4 x float> %tex, i32 0
517 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
518 %dtex.1 = extractelement <4 x float> %dtex, i32 0
519 call void @llvm.amdgcn.buffer.store.f32(float %dtex.1, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
521 %cc = fcmp ogt float %dtex.1, 0.0
522 br i1 %cc, label %IF, label %ELSE
525 %tex.IF = fmul float %dtex.1, 3.0
529 %tex.ELSE = fmul float %dtex.1, 4.0
533 %tex.END = phi float [ %tex.IF, %IF ], [ %tex.ELSE, %ELSE ]
537 ; Another test that failed at some point because of terminator handling.
539 ;CHECK-LABEL: {{^}}test_control_flow_4:
540 ;CHECK-NEXT: ; %main_body
541 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
542 ;CHECK-NEXT: s_wqm_b64 exec, exec
544 ;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]]
547 ;CHECK: s_mov_b64 exec, [[SAVE]]
551 define amdgpu_ps <4 x float> @test_control_flow_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float %coord, i32 %y, float %z) {
553 %cond = icmp eq i32 %y, 0
554 br i1 %cond, label %IF, label %END
557 %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> undef, i32 0, i32 0, i1 0, i1 0)
558 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i1 0, i1 0)
562 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
563 %tex0 = extractelement <4 x float> %tex, i32 0
564 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
565 ret <4 x float> %dtex
568 ; Kill is performed in WQM mode so that uniform kill behaves correctly ...
570 ;CHECK-LABEL: {{^}}test_kill_0:
571 ;CHECK-NEXT: ; %main_body
572 ;CHECK-NEXT: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
573 ;CHECK-NEXT: s_wqm_b64 exec, exec
574 ;CHECK: s_and_b64 exec, exec, [[ORIG]]
576 ;CHECK: buffer_store_dword
577 ;CHECK: s_wqm_b64 exec, exec
579 ;CHECK: s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[ORIG]]
580 ;CHECK: buffer_store_dword
581 ;CHECK: s_mov_b64 exec, [[SAVE]]
583 define amdgpu_ps <4 x float> @test_kill_0(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, float addrspace(1)* inreg %ptr, <2 x i32> %idx, <2 x float> %data, float %coord, float %coord2, float %z) {
585 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
586 %idx.0 = extractelement <2 x i32> %idx, i32 0
587 %data.0 = extractelement <2 x float> %data, i32 0
588 call void @llvm.amdgcn.buffer.store.f32(float %data.0, <4 x i32> undef, i32 %idx.0, i32 0, i1 0, i1 0)
590 %z.cmp = fcmp olt float %z, 0.0
591 call void @llvm.amdgcn.kill(i1 %z.cmp)
593 %idx.1 = extractelement <2 x i32> %idx, i32 1
594 %data.1 = extractelement <2 x float> %data, i32 1
595 call void @llvm.amdgcn.buffer.store.f32(float %data.1, <4 x i32> undef, i32 %idx.1, i32 0, i1 0, i1 0)
596 %tex2 = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord2, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
597 %tex2.0 = extractelement <4 x float> %tex2, i32 0
598 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex2.0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
599 %out = fadd <4 x float> %tex, %dtex
604 ; ... but only if WQM is necessary.
606 ; CHECK-LABEL: {{^}}test_kill_1:
607 ; CHECK-NEXT: ; %main_body
608 ; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
609 ; CHECK: s_wqm_b64 exec, exec
610 ; CHECK: image_sample
611 ; CHECK: s_and_b64 exec, exec, [[ORIG]]
612 ; CHECK: image_sample
613 ; CHECK: buffer_store_dword
616 define amdgpu_ps <4 x float> @test_kill_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %idx, float %data, float %coord, float %coord2, float %z) {
618 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %coord, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
619 %tex0 = extractelement <4 x float> %tex, i32 0
620 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
622 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0)
624 %z.cmp = fcmp olt float %z, 0.0
625 call void @llvm.amdgcn.kill(i1 %z.cmp)
627 ret <4 x float> %dtex
630 ; Check prolog shaders.
632 ; CHECK-LABEL: {{^}}test_prolog_1:
633 ; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
634 ; CHECK: s_wqm_b64 exec, exec
635 ; CHECK: v_add_f32_e32 v0,
636 ; CHECK: s_and_b64 exec, exec, [[ORIG]]
637 define amdgpu_ps float @test_prolog_1(float %a, float %b) #5 {
639 %s = fadd float %a, %b
643 ; CHECK-LABEL: {{^}}test_loop_vcc:
644 ; CHECK-NEXT: ; %entry
645 ; CHECK-NEXT: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec
646 ; CHECK: s_wqm_b64 exec, exec
647 ; CHECK: s_and_b64 exec, exec, [[LIVE]]
649 ; CHECK: s_wqm_b64 exec, exec
650 ; CHECK-DAG: v_mov_b32_e32 [[CTR:v[0-9]+]], 0
651 ; CHECK-DAG: s_mov_b32 [[SEVEN:s[0-9]+]], 0x40e00000
653 ; CHECK: [[LOOPHDR:BB[0-9]+_[0-9]+]]: ; %body
654 ; CHECK: v_add_f32_e32 [[CTR]], 2.0, [[CTR]]
655 ; CHECK: v_cmp_lt_f32_e32 vcc, [[SEVEN]], [[CTR]]
656 ; CHECK: s_cbranch_vccz [[LOOPHDR]]
660 define amdgpu_ps <4 x float> @test_loop_vcc(<4 x float> %in) nounwind {
662 call void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float> %in, i32 15, i32 undef, <8 x i32> undef, i32 0, i32 0)
666 %ctr.iv = phi float [ 0.0, %entry ], [ %ctr.next, %body ]
667 %c.iv = phi <4 x float> [ %in, %entry ], [ %c.next, %body ]
668 %cc = fcmp ogt float %ctr.iv, 7.0
669 br i1 %cc, label %break, label %body
672 %c.iv0 = extractelement <4 x float> %c.iv, i32 0
673 %c.next = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.iv0, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
674 %ctr.next = fadd float %ctr.iv, 2.0
678 ret <4 x float> %c.iv
681 ; Only intrinsic stores need exact execution -- other stores do not have
682 ; externally visible effects and may require WQM for correctness.
684 ; CHECK-LABEL: {{^}}test_alloca:
685 ; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec
686 ; CHECK: s_wqm_b64 exec, exec
688 ; CHECK: s_and_b64 exec, exec, [[LIVE]]
689 ; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, 0
690 ; CHECK: s_wqm_b64 exec, exec
691 ; CHECK: buffer_store_dword {{v[0-9]+}}, off, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offset:4{{$}}
692 ; CHECK: s_and_b64 exec, exec, [[LIVE]]
693 ; CHECK: buffer_store_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, 0 idxen
694 ; CHECK: s_wqm_b64 exec, exec
695 ; CHECK: buffer_load_dword {{v[0-9]+}}, {{v[0-9]+}}, {{s\[[0-9]+:[0-9]+\]}}, {{s[0-9]+}} offen
697 ; CHECK: s_and_b64 exec, exec, [[LIVE]]
698 ; CHECK: image_sample
699 ; CHECK: buffer_store_dwordx4
700 define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind {
702 %array = alloca [32 x i32], align 4, addrspace(5)
704 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0)
706 %s.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 0
707 store volatile i32 %a, i32 addrspace(5)* %s.gep, align 4
709 call void @llvm.amdgcn.buffer.store.f32(float %data, <4 x i32> undef, i32 1, i32 0, i1 0, i1 0)
711 %c.gep = getelementptr [32 x i32], [32 x i32] addrspace(5)* %array, i32 0, i32 %idx
712 %c = load i32, i32 addrspace(5)* %c.gep, align 4
713 %c.bc = bitcast i32 %c to float
714 %t = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
715 call void @llvm.amdgcn.buffer.store.v4f32(<4 x float> %t, <4 x i32> undef, i32 0, i32 0, i1 0, i1 0)
720 ; Must return to exact at the end of a non-void returning shader,
721 ; otherwise the EXEC mask exported by the epilog will be wrong. This is true
722 ; even if the shader has no kills, because a kill could have happened in a
723 ; previous shader fragment.
725 ; CHECK-LABEL: {{^}}test_nonvoid_return:
726 ; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec
727 ; CHECK: s_wqm_b64 exec, exec
729 ; CHECK: s_and_b64 exec, exec, [[LIVE]]
731 define amdgpu_ps <4 x float> @test_nonvoid_return() nounwind {
732 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
733 %tex0 = extractelement <4 x float> %tex, i32 0
734 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
735 ret <4 x float> %dtex
738 ; CHECK-LABEL: {{^}}test_nonvoid_return_unreachable:
739 ; CHECK: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec
740 ; CHECK: s_wqm_b64 exec, exec
742 ; CHECK: s_and_b64 exec, exec, [[LIVE]]
744 define amdgpu_ps <4 x float> @test_nonvoid_return_unreachable(i32 inreg %c) nounwind {
746 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float undef, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
747 %tex0 = extractelement <4 x float> %tex, i32 0
748 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
749 %cc = icmp sgt i32 %c, 0
750 br i1 %cc, label %if, label %else
753 store volatile <4 x float> %dtex, <4 x float> addrspace(1)* undef
757 ret <4 x float> %dtex
760 ; Test awareness that s_wqm_b64 clobbers SCC.
762 ; CHECK-LABEL: {{^}}test_scc:
763 ; CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
764 ; CHECK: s_wqm_b64 exec, exec
766 ; CHECK-NEXT: s_cbranch_scc
768 ; CHECK: s_and_b64 exec, exec, [[ORIG]]
769 ; CHECK: image_sample
771 ; CHECK: s_and_b64 exec, exec, [[ORIG]]
772 ; CHECK: image_sample
774 define amdgpu_ps <4 x float> @test_scc(i32 inreg %sel, i32 %idx) #1 {
776 %cc = icmp sgt i32 %sel, 0
777 br i1 %cc, label %if, label %else
780 %r.if = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float 0.0, <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
784 %r.else = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float 0.0, float bitcast (i32 1 to float), <8 x i32> undef, <4 x i32> undef, i1 0, i32 0, i32 0) #0
788 %r = phi <4 x float> [ %r.if, %if ], [ %r.else, %else ]
789 call void @llvm.amdgcn.buffer.store.f32(float 1.0, <4 x i32> undef, i32 %idx, i32 0, i1 0, i1 0)
793 ; Check a case of a block being entirely WQM except for a bit of WWM.
794 ; There was a bug where it forgot to enter and leave WWM.
796 ;CHECK-LABEL: {{^}}test_wwm_within_wqm:
798 ;CHECK: s_or_saveexec_b64 {{.*}}, -1
801 define amdgpu_ps float @test_wwm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {
803 %c.bc = bitcast i32 %c to float
804 %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
805 %tex0 = extractelement <4 x float> %tex, i32 0
806 %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 0, i32 0, i32 0) #0
807 %cmp = icmp eq i32 %z, 0
808 br i1 %cmp, label %IF, label %ENDIF
811 %dataf = extractelement <4 x float> %dtex, i32 0
812 %data1 = fptosi float %dataf to i32
813 %data2 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %data1, i32 0)
814 %data3 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data2, i32 2079)
815 %data4 = call i32 @llvm.amdgcn.wwm.i32(i32 %data3)
816 %data4f = sitofp i32 %data4 to float
820 %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ]
824 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1
825 declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1
826 declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1) #2
827 declare void @llvm.amdgcn.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i1, i1) #2
828 declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #3
829 declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #3
830 declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
831 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
832 declare void @llvm.amdgcn.kill(i1) #1
833 declare float @llvm.amdgcn.wqm.f32(float) #3
834 declare i32 @llvm.amdgcn.wqm.i32(i32) #3
835 declare float @llvm.amdgcn.wwm.f32(float) #3
836 declare i32 @llvm.amdgcn.wwm.i32(i32) #3
837 declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4
838 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3
839 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3
840 declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3
841 declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1
842 declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2
843 declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2
844 declare i32 @llvm.amdgcn.ds.swizzle(i32, i32)
846 attributes #1 = { nounwind }
847 attributes #2 = { nounwind readonly }
848 attributes #3 = { nounwind readnone }
849 attributes #4 = { nounwind readnone convergent }
850 attributes #5 = { "amdgpu-ps-wqm-outputs" }
851 attributes #6 = { nounwind "InitialPSInputAddr"="2" }