?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[Codegen] Alter the default promotion for saturating adds and subs
[llvm-complete.git] / lib / Target / AMDGPU / AMDGPUISelLowering.cpp
blob3027b21fb05376f11da9d2526f31fb0924964bab
1 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This is the parent TargetLowering class for hardware code gen
11 /// targets.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "AMDGPUISelLowering.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUCallLowering.h"
18 #include "AMDGPUFrameLowering.h"
19 #include "AMDGPURegisterInfo.h"
20 #include "AMDGPUSubtarget.h"
21 #include "AMDGPUTargetMachine.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "R600MachineFunctionInfo.h"
24 #include "SIInstrInfo.h"
25 #include "SIMachineFunctionInfo.h"
26 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
27 #include "llvm/CodeGen/Analysis.h"
28 #include "llvm/CodeGen/CallingConvLower.h"
29 #include "llvm/CodeGen/MachineFunction.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/CodeGen/SelectionDAG.h"
32 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
33 #include "llvm/IR/DataLayout.h"
34 #include "llvm/IR/DiagnosticInfo.h"
35 #include "llvm/Support/KnownBits.h"
36 #include "llvm/Support/MathExtras.h"
37 using namespace llvm;
38
39 #include "AMDGPUGenCallingConv.inc"
40
41 // Find a larger type to do a load / store of a vector with.
42 EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
43 unsigned StoreSize = VT.getStoreSizeInBits();
44 if (StoreSize <= 32)
45 return EVT::getIntegerVT(Ctx, StoreSize);
46
47 assert(StoreSize % 32 == 0 && "Store size not a multiple of 32");
48 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32);
49 }
50
51 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
52 EVT VT = Op.getValueType();
53 KnownBits Known = DAG.computeKnownBits(Op);
54 return VT.getSizeInBits() - Known.countMinLeadingZeros();
55 }
56
57 unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
58 EVT VT = Op.getValueType();
59
60 // In order for this to be a signed 24-bit value, bit 23, must
61 // be a sign bit.
62 return VT.getSizeInBits() - DAG.ComputeNumSignBits(Op);
63 }
64
65 AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
66 const AMDGPUSubtarget &STI)
67 : TargetLowering(TM), Subtarget(&STI) {
68 // Lower floating point store/load to integer store/load to reduce the number
69 // of patterns in tablegen.
70 setOperationAction(ISD::LOAD, MVT::f32, Promote);
71 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32);
72
73 setOperationAction(ISD::LOAD, MVT::v2f32, Promote);
74 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32);
75
76 setOperationAction(ISD::LOAD, MVT::v3f32, Promote);
77 AddPromotedToType(ISD::LOAD, MVT::v3f32, MVT::v3i32);
78
79 setOperationAction(ISD::LOAD, MVT::v4f32, Promote);
80 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32);
81
82 setOperationAction(ISD::LOAD, MVT::v5f32, Promote);
83 AddPromotedToType(ISD::LOAD, MVT::v5f32, MVT::v5i32);
84
85 setOperationAction(ISD::LOAD, MVT::v8f32, Promote);
86 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32);
87
88 setOperationAction(ISD::LOAD, MVT::v16f32, Promote);
89 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32);
90
91 setOperationAction(ISD::LOAD, MVT::v32f32, Promote);
92 AddPromotedToType(ISD::LOAD, MVT::v32f32, MVT::v32i32);
93
94 setOperationAction(ISD::LOAD, MVT::i64, Promote);
95 AddPromotedToType(ISD::LOAD, MVT::i64, MVT::v2i32);
96
97 setOperationAction(ISD::LOAD, MVT::v2i64, Promote);
98 AddPromotedToType(ISD::LOAD, MVT::v2i64, MVT::v4i32);
99
100 setOperationAction(ISD::LOAD, MVT::f64, Promote);
101 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::v2i32);
102
103 setOperationAction(ISD::LOAD, MVT::v2f64, Promote);
104 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v4i32);
105
106 // There are no 64-bit extloads. These should be done as a 32-bit extload and
107 // an extension to 64-bit.
108 for (MVT VT : MVT::integer_valuetypes()) {
109 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand);
110 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand);
111 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand);
112 }
113
114 for (MVT VT : MVT::integer_valuetypes()) {
115 if (VT == MVT::i64)
116 continue;
117
118 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
119 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Legal);
120 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i16, Legal);
121 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i32, Expand);
122
123 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote);
124 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i8, Legal);
125 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i16, Legal);
126 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i32, Expand);
127
128 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i1, Promote);
129 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i8, Legal);
130 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i16, Legal);
131 setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
132 }
133
134 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
135 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand);
136 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand);
137 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand);
138 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand);
139 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand);
140 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand);
141 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand);
142 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand);
143 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand);
144 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v3i16, Expand);
145 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v3i16, Expand);
146 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v3i16, Expand);
147 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand);
148 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand);
149 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand);
150 }
151
152 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
153 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand);
154 setLoadExtAction(ISD::EXTLOAD, MVT::v3f32, MVT::v3f16, Expand);
155 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand);
156 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand);
157 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Expand);
158 setLoadExtAction(ISD::EXTLOAD, MVT::v32f32, MVT::v32f16, Expand);
159
160 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
161 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Expand);
162 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Expand);
163 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f32, Expand);
164
165 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
166 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand);
167 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand);
168 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand);
169
170 setOperationAction(ISD::STORE, MVT::f32, Promote);
171 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32);
172
173 setOperationAction(ISD::STORE, MVT::v2f32, Promote);
174 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32);
175
176 setOperationAction(ISD::STORE, MVT::v3f32, Promote);
177 AddPromotedToType(ISD::STORE, MVT::v3f32, MVT::v3i32);
178
179 setOperationAction(ISD::STORE, MVT::v4f32, Promote);
180 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32);
181
182 setOperationAction(ISD::STORE, MVT::v5f32, Promote);
183 AddPromotedToType(ISD::STORE, MVT::v5f32, MVT::v5i32);
184
185 setOperationAction(ISD::STORE, MVT::v8f32, Promote);
186 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32);
187
188 setOperationAction(ISD::STORE, MVT::v16f32, Promote);
189 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32);
190
191 setOperationAction(ISD::STORE, MVT::v32f32, Promote);
192 AddPromotedToType(ISD::STORE, MVT::v32f32, MVT::v32i32);
193
194 setOperationAction(ISD::STORE, MVT::i64, Promote);
195 AddPromotedToType(ISD::STORE, MVT::i64, MVT::v2i32);
196
197 setOperationAction(ISD::STORE, MVT::v2i64, Promote);
198 AddPromotedToType(ISD::STORE, MVT::v2i64, MVT::v4i32);
199
200 setOperationAction(ISD::STORE, MVT::f64, Promote);
201 AddPromotedToType(ISD::STORE, MVT::f64, MVT::v2i32);
202
203 setOperationAction(ISD::STORE, MVT::v2f64, Promote);
204 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v4i32);
205
206 setTruncStoreAction(MVT::i64, MVT::i1, Expand);
207 setTruncStoreAction(MVT::i64, MVT::i8, Expand);
208 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
209 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
210
211 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand);
212 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Expand);
213 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Expand);
214 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Expand);
215
216 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
217 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand);
218 setTruncStoreAction(MVT::v3f32, MVT::v3f16, Expand);
219 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand);
220 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand);
221 setTruncStoreAction(MVT::v16f32, MVT::v16f16, Expand);
222 setTruncStoreAction(MVT::v32f32, MVT::v32f16, Expand);
223
224 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
225 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
226
227 setTruncStoreAction(MVT::v2f64, MVT::v2f32, Expand);
228 setTruncStoreAction(MVT::v2f64, MVT::v2f16, Expand);
229
230 setTruncStoreAction(MVT::v4f64, MVT::v4f32, Expand);
231 setTruncStoreAction(MVT::v4f64, MVT::v4f16, Expand);
232
233 setTruncStoreAction(MVT::v8f64, MVT::v8f32, Expand);
234 setTruncStoreAction(MVT::v8f64, MVT::v8f16, Expand);
235
236
237 setOperationAction(ISD::Constant, MVT::i32, Legal);
238 setOperationAction(ISD::Constant, MVT::i64, Legal);
239 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
240 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
241
242 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
243 setOperationAction(ISD::BRIND, MVT::Other, Expand);
244
245 // This is totally unsupported, just custom lower to produce an error.
246 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
247
248 // Library functions. These default to Expand, but we have instructions
249 // for them.
250 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
251 setOperationAction(ISD::FEXP2, MVT::f32, Legal);
252 setOperationAction(ISD::FPOW, MVT::f32, Legal);
253 setOperationAction(ISD::FLOG2, MVT::f32, Legal);
254 setOperationAction(ISD::FABS, MVT::f32, Legal);
255 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
256 setOperationAction(ISD::FRINT, MVT::f32, Legal);
257 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
258 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
259 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
260
261 setOperationAction(ISD::FROUND, MVT::f32, Custom);
262 setOperationAction(ISD::FROUND, MVT::f64, Custom);
263
264 setOperationAction(ISD::FLOG, MVT::f32, Custom);
265 setOperationAction(ISD::FLOG10, MVT::f32, Custom);
266 setOperationAction(ISD::FEXP, MVT::f32, Custom);
267
268
269 setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom);
270 setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom);
271
272 setOperationAction(ISD::FREM, MVT::f32, Custom);
273 setOperationAction(ISD::FREM, MVT::f64, Custom);
274
275 // Expand to fneg + fadd.
276 setOperationAction(ISD::FSUB, MVT::f64, Expand);
277
278 setOperationAction(ISD::CONCAT_VECTORS, MVT::v3i32, Custom);
279 setOperationAction(ISD::CONCAT_VECTORS, MVT::v3f32, Custom);
280 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom);
281 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom);
282 setOperationAction(ISD::CONCAT_VECTORS, MVT::v5i32, Custom);
283 setOperationAction(ISD::CONCAT_VECTORS, MVT::v5f32, Custom);
284 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom);
285 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom);
286 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom);
287 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom);
288 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v3f32, Custom);
289 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v3i32, Custom);
290 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom);
291 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom);
292 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v5f32, Custom);
293 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v5i32, Custom);
294 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom);
295 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom);
296 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f32, Custom);
297 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16i32, Custom);
298 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v32f32, Custom);
299 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v32i32, Custom);
300
301 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
302 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
303 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
304
305 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
306 for (MVT VT : ScalarIntVTs) {
307 // These should use [SU]DIVREM, so set them to expand
308 setOperationAction(ISD::SDIV, VT, Expand);
309 setOperationAction(ISD::UDIV, VT, Expand);
310 setOperationAction(ISD::SREM, VT, Expand);
311 setOperationAction(ISD::UREM, VT, Expand);
312
313 // GPU does not have divrem function for signed or unsigned.
314 setOperationAction(ISD::SDIVREM, VT, Custom);
315 setOperationAction(ISD::UDIVREM, VT, Custom);
316
317 // GPU does not have [S|U]MUL_LOHI functions as a single instruction.
318 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
319 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
320
321 setOperationAction(ISD::BSWAP, VT, Expand);
322 setOperationAction(ISD::CTTZ, VT, Expand);
323 setOperationAction(ISD::CTLZ, VT, Expand);
324
325 // AMDGPU uses ADDC/SUBC/ADDE/SUBE
326 setOperationAction(ISD::ADDC, VT, Legal);
327 setOperationAction(ISD::SUBC, VT, Legal);
328 setOperationAction(ISD::ADDE, VT, Legal);
329 setOperationAction(ISD::SUBE, VT, Legal);
330 }
331
332 // The hardware supports 32-bit ROTR, but not ROTL.
333 setOperationAction(ISD::ROTL, MVT::i32, Expand);
334 setOperationAction(ISD::ROTL, MVT::i64, Expand);
335 setOperationAction(ISD::ROTR, MVT::i64, Expand);
336
337 setOperationAction(ISD::MUL, MVT::i64, Expand);
338 setOperationAction(ISD::MULHU, MVT::i64, Expand);
339 setOperationAction(ISD::MULHS, MVT::i64, Expand);
340 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
341 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
342 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
343 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
344 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
345
346 setOperationAction(ISD::SMIN, MVT::i32, Legal);
347 setOperationAction(ISD::UMIN, MVT::i32, Legal);
348 setOperationAction(ISD::SMAX, MVT::i32, Legal);
349 setOperationAction(ISD::UMAX, MVT::i32, Legal);
350
351 setOperationAction(ISD::CTTZ, MVT::i64, Custom);
352 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Custom);
353 setOperationAction(ISD::CTLZ, MVT::i64, Custom);
354 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);
355
356 static const MVT::SimpleValueType VectorIntTypes[] = {
357 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32
358 };
359
360 for (MVT VT : VectorIntTypes) {
361 // Expand the following operations for the current type by default.
362 setOperationAction(ISD::ADD, VT, Expand);
363 setOperationAction(ISD::AND, VT, Expand);
364 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
365 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
366 setOperationAction(ISD::MUL, VT, Expand);
367 setOperationAction(ISD::MULHU, VT, Expand);
368 setOperationAction(ISD::MULHS, VT, Expand);
369 setOperationAction(ISD::OR, VT, Expand);
370 setOperationAction(ISD::SHL, VT, Expand);
371 setOperationAction(ISD::SRA, VT, Expand);
372 setOperationAction(ISD::SRL, VT, Expand);
373 setOperationAction(ISD::ROTL, VT, Expand);
374 setOperationAction(ISD::ROTR, VT, Expand);
375 setOperationAction(ISD::SUB, VT, Expand);
376 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
377 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
378 setOperationAction(ISD::SDIV, VT, Expand);
379 setOperationAction(ISD::UDIV, VT, Expand);
380 setOperationAction(ISD::SREM, VT, Expand);
381 setOperationAction(ISD::UREM, VT, Expand);
382 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
383 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
384 setOperationAction(ISD::SDIVREM, VT, Custom);
385 setOperationAction(ISD::UDIVREM, VT, Expand);
386 setOperationAction(ISD::SELECT, VT, Expand);
387 setOperationAction(ISD::VSELECT, VT, Expand);
388 setOperationAction(ISD::SELECT_CC, VT, Expand);
389 setOperationAction(ISD::XOR, VT, Expand);
390 setOperationAction(ISD::BSWAP, VT, Expand);
391 setOperationAction(ISD::CTPOP, VT, Expand);
392 setOperationAction(ISD::CTTZ, VT, Expand);
393 setOperationAction(ISD::CTLZ, VT, Expand);
394 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
395 setOperationAction(ISD::SETCC, VT, Expand);
396 }
397
398 static const MVT::SimpleValueType FloatVectorTypes[] = {
399 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32
400 };
401
402 for (MVT VT : FloatVectorTypes) {
403 setOperationAction(ISD::FABS, VT, Expand);
404 setOperationAction(ISD::FMINNUM, VT, Expand);
405 setOperationAction(ISD::FMAXNUM, VT, Expand);
406 setOperationAction(ISD::FADD, VT, Expand);
407 setOperationAction(ISD::FCEIL, VT, Expand);
408 setOperationAction(ISD::FCOS, VT, Expand);
409 setOperationAction(ISD::FDIV, VT, Expand);
410 setOperationAction(ISD::FEXP2, VT, Expand);
411 setOperationAction(ISD::FEXP, VT, Expand);
412 setOperationAction(ISD::FLOG2, VT, Expand);
413 setOperationAction(ISD::FREM, VT, Expand);
414 setOperationAction(ISD::FLOG, VT, Expand);
415 setOperationAction(ISD::FLOG10, VT, Expand);
416 setOperationAction(ISD::FPOW, VT, Expand);
417 setOperationAction(ISD::FFLOOR, VT, Expand);
418 setOperationAction(ISD::FTRUNC, VT, Expand);
419 setOperationAction(ISD::FMUL, VT, Expand);
420 setOperationAction(ISD::FMA, VT, Expand);
421 setOperationAction(ISD::FRINT, VT, Expand);
422 setOperationAction(ISD::FNEARBYINT, VT, Expand);
423 setOperationAction(ISD::FSQRT, VT, Expand);
424 setOperationAction(ISD::FSIN, VT, Expand);
425 setOperationAction(ISD::FSUB, VT, Expand);
426 setOperationAction(ISD::FNEG, VT, Expand);
427 setOperationAction(ISD::VSELECT, VT, Expand);
428 setOperationAction(ISD::SELECT_CC, VT, Expand);
429 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
430 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand);
431 setOperationAction(ISD::SETCC, VT, Expand);
432 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
433 }
434
435 // This causes using an unrolled select operation rather than expansion with
436 // bit operations. This is in general better, but the alternative using BFI
437 // instructions may be better if the select sources are SGPRs.
438 setOperationAction(ISD::SELECT, MVT::v2f32, Promote);
439 AddPromotedToType(ISD::SELECT, MVT::v2f32, MVT::v2i32);
440
441 setOperationAction(ISD::SELECT, MVT::v3f32, Promote);
442 AddPromotedToType(ISD::SELECT, MVT::v3f32, MVT::v3i32);
443
444 setOperationAction(ISD::SELECT, MVT::v4f32, Promote);
445 AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32);
446
447 setOperationAction(ISD::SELECT, MVT::v5f32, Promote);
448 AddPromotedToType(ISD::SELECT, MVT::v5f32, MVT::v5i32);
449
450 // There are no libcalls of any kind.
451 for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I)
452 setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr);
453
454 setBooleanContents(ZeroOrNegativeOneBooleanContent);
455 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
456
457 setSchedulingPreference(Sched::RegPressure);
458 setJumpIsExpensive(true);
459
460 // FIXME: This is only partially true. If we have to do vector compares, any
461 // SGPR pair can be a condition register. If we have a uniform condition, we
462 // are better off doing SALU operations, where there is only one SCC. For now,
463 // we don't have a way of knowing during instruction selection if a condition
464 // will be uniform and we always use vector compares. Assume we are using
465 // vector compares until that is fixed.
466 setHasMultipleConditionRegisters(true);
467
468 setMinCmpXchgSizeInBits(32);
469 setSupportsUnalignedAtomics(false);
470
471 PredictableSelectIsExpensive = false;
472
473 // We want to find all load dependencies for long chains of stores to enable
474 // merging into very wide vectors. The problem is with vectors with > 4
475 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
476 // vectors are a legal type, even though we have to split the loads
477 // usually. When we can more precisely specify load legality per address
478 // space, we should be able to make FindBetterChain/MergeConsecutiveStores
479 // smarter so that they can figure out what to do in 2 iterations without all
480 // N > 4 stores on the same chain.
481 GatherAllAliasesMaxDepth = 16;
482
483 // memcpy/memmove/memset are expanded in the IR, so we shouldn't need to worry
484 // about these during lowering.
485 MaxStoresPerMemcpy = 0xffffffff;
486 MaxStoresPerMemmove = 0xffffffff;
487 MaxStoresPerMemset = 0xffffffff;
488
489 setTargetDAGCombine(ISD::BITCAST);
490 setTargetDAGCombine(ISD::SHL);
491 setTargetDAGCombine(ISD::SRA);
492 setTargetDAGCombine(ISD::SRL);
493 setTargetDAGCombine(ISD::TRUNCATE);
494 setTargetDAGCombine(ISD::MUL);
495 setTargetDAGCombine(ISD::MULHU);
496 setTargetDAGCombine(ISD::MULHS);
497 setTargetDAGCombine(ISD::SELECT);
498 setTargetDAGCombine(ISD::SELECT_CC);
499 setTargetDAGCombine(ISD::STORE);
500 setTargetDAGCombine(ISD::FADD);
501 setTargetDAGCombine(ISD::FSUB);
502 setTargetDAGCombine(ISD::FNEG);
503 setTargetDAGCombine(ISD::FABS);
504 setTargetDAGCombine(ISD::AssertZext);
505 setTargetDAGCombine(ISD::AssertSext);
506 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
507 }
508
509 //===----------------------------------------------------------------------===//
510 // Target Information
511 //===----------------------------------------------------------------------===//
512
513 LLVM_READNONE
514 static bool fnegFoldsIntoOp(unsigned Opc) {
515 switch (Opc) {
516 case ISD::FADD:
517 case ISD::FSUB:
518 case ISD::FMUL:
519 case ISD::FMA:
520 case ISD::FMAD:
521 case ISD::FMINNUM:
522 case ISD::FMAXNUM:
523 case ISD::FMINNUM_IEEE:
524 case ISD::FMAXNUM_IEEE:
525 case ISD::FSIN:
526 case ISD::FTRUNC:
527 case ISD::FRINT:
528 case ISD::FNEARBYINT:
529 case ISD::FCANONICALIZE:
530 case AMDGPUISD::RCP:
531 case AMDGPUISD::RCP_LEGACY:
532 case AMDGPUISD::RCP_IFLAG:
533 case AMDGPUISD::SIN_HW:
534 case AMDGPUISD::FMUL_LEGACY:
535 case AMDGPUISD::FMIN_LEGACY:
536 case AMDGPUISD::FMAX_LEGACY:
537 case AMDGPUISD::FMED3:
538 return true;
539 default:
540 return false;
541 }
542 }
543
544 /// \p returns true if the operation will definitely need to use a 64-bit
545 /// encoding, and thus will use a VOP3 encoding regardless of the source
546 /// modifiers.
547 LLVM_READONLY
548 static bool opMustUseVOP3Encoding(const SDNode *N, MVT VT) {
549 return N->getNumOperands() > 2 || VT == MVT::f64;
550 }
551
552 // Most FP instructions support source modifiers, but this could be refined
553 // slightly.
554 LLVM_READONLY
555 static bool hasSourceMods(const SDNode *N) {
556 if (isa<MemSDNode>(N))
557 return false;
558
559 switch (N->getOpcode()) {
560 case ISD::CopyToReg:
561 case ISD::SELECT:
562 case ISD::FDIV:
563 case ISD::FREM:
564 case ISD::INLINEASM:
565 case ISD::INLINEASM_BR:
566 case AMDGPUISD::INTERP_P1:
567 case AMDGPUISD::INTERP_P2:
568 case AMDGPUISD::DIV_SCALE:
569
570 // TODO: Should really be looking at the users of the bitcast. These are
571 // problematic because bitcasts are used to legalize all stores to integer
572 // types.
573 case ISD::BITCAST:
574 return false;
575 default:
576 return true;
577 }
578 }
579
580 bool AMDGPUTargetLowering::allUsesHaveSourceMods(const SDNode *N,
581 unsigned CostThreshold) {
582 // Some users (such as 3-operand FMA/MAD) must use a VOP3 encoding, and thus
583 // it is truly free to use a source modifier in all cases. If there are
584 // multiple users but for each one will necessitate using VOP3, there will be
585 // a code size increase. Try to avoid increasing code size unless we know it
586 // will save on the instruction count.
587 unsigned NumMayIncreaseSize = 0;
588 MVT VT = N->getValueType(0).getScalarType().getSimpleVT();
589
590 // XXX - Should this limit number of uses to check?
591 for (const SDNode *U : N->uses()) {
592 if (!hasSourceMods(U))
593 return false;
594
595 if (!opMustUseVOP3Encoding(U, VT)) {
596 if (++NumMayIncreaseSize > CostThreshold)
597 return false;
598 }
599 }
600
601 return true;
602 }
603
604 MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const {
605 return MVT::i32;
606 }
607
608 bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
609 return true;
610 }
611
612 // The backend supports 32 and 64 bit floating point immediates.
613 // FIXME: Why are we reporting vectors of FP immediates as legal?
614 bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
615 bool ForCodeSize) const {
616 EVT ScalarVT = VT.getScalarType();
617 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 ||
618 (ScalarVT == MVT::f16 && Subtarget->has16BitInsts()));
619 }
620
621 // We don't want to shrink f64 / f32 constants.
622 bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
623 EVT ScalarVT = VT.getScalarType();
624 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
625 }
626
627 bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N,
628 ISD::LoadExtType ExtTy,
629 EVT NewVT) const {
630 // TODO: This may be worth removing. Check regression tests for diffs.
631 if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT))
632 return false;
633
634 unsigned NewSize = NewVT.getStoreSizeInBits();
635
636 // If we are reducing to a 32-bit load, this is always better.
637 if (NewSize == 32)
638 return true;
639
640 EVT OldVT = N->getValueType(0);
641 unsigned OldSize = OldVT.getStoreSizeInBits();
642
643 MemSDNode *MN = cast<MemSDNode>(N);
644 unsigned AS = MN->getAddressSpace();
645 // Do not shrink an aligned scalar load to sub-dword.
646 // Scalar engine cannot do sub-dword loads.
647 if (OldSize >= 32 && NewSize < 32 && MN->getAlignment() >= 4 &&
648 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
649 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
650 (isa<LoadSDNode>(N) &&
651 AS == AMDGPUAS::GLOBAL_ADDRESS && MN->isInvariant())) &&
652 AMDGPUInstrInfo::isUniformMMO(MN->getMemOperand()))
653 return false;
654
655 // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
656 // extloads, so doing one requires using a buffer_load. In cases where we
657 // still couldn't use a scalar load, using the wider load shouldn't really
658 // hurt anything.
659
660 // If the old size already had to be an extload, there's no harm in continuing
661 // to reduce the width.
662 return (OldSize < 32);
663 }
664
665 bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
666 const SelectionDAG &DAG,
667 const MachineMemOperand &MMO) const {
668
669 assert(LoadTy.getSizeInBits() == CastTy.getSizeInBits());
670
671 if (LoadTy.getScalarType() == MVT::i32)
672 return false;
673
674 unsigned LScalarSize = LoadTy.getScalarSizeInBits();
675 unsigned CastScalarSize = CastTy.getScalarSizeInBits();
676
677 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
678 return false;
679
680 bool Fast = false;
681 return allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
682 CastTy, MMO, &Fast) &&
683 Fast;
684 }
685
686 // SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
687 // profitable with the expansion for 64-bit since it's generally good to
688 // speculate things.
689 // FIXME: These should really have the size as a parameter.
690 bool AMDGPUTargetLowering::isCheapToSpeculateCttz() const {
691 return true;
692 }
693
694 bool AMDGPUTargetLowering::isCheapToSpeculateCtlz() const {
695 return true;
696 }
697
698 bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode * N) const {
699 switch (N->getOpcode()) {
700 default:
701 return false;
702 case ISD::EntryToken:
703 case ISD::TokenFactor:
704 return true;
705 case ISD::INTRINSIC_WO_CHAIN:
706 {
707 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
708 switch (IntrID) {
709 default:
710 return false;
711 case Intrinsic::amdgcn_readfirstlane:
712 case Intrinsic::amdgcn_readlane:
713 return true;
714 }
715 }
716 break;
717 case ISD::LOAD:
718 {
719 if (cast<LoadSDNode>(N)->getMemOperand()->getAddrSpace() ==
720 AMDGPUAS::CONSTANT_ADDRESS_32BIT)
721 return true;
722 return false;
723 }
724 break;
725 }
726 }
727
728 //===---------------------------------------------------------------------===//
729 // Target Properties
730 //===---------------------------------------------------------------------===//
731
732 bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const {
733 assert(VT.isFloatingPoint());
734
735 // Packed operations do not have a fabs modifier.
736 return VT == MVT::f32 || VT == MVT::f64 ||
737 (Subtarget->has16BitInsts() && VT == MVT::f16);
738 }
739
740 bool AMDGPUTargetLowering::isFNegFree(EVT VT) const {
741 assert(VT.isFloatingPoint());
742 return VT == MVT::f32 || VT == MVT::f64 ||
743 (Subtarget->has16BitInsts() && VT == MVT::f16) ||
744 (Subtarget->hasVOP3PInsts() && VT == MVT::v2f16);
745 }
746
747 bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(EVT MemVT,
748 unsigned NumElem,
749 unsigned AS) const {
750 return true;
751 }
752
753 bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const {
754 // There are few operations which truly have vector input operands. Any vector
755 // operation is going to involve operations on each component, and a
756 // build_vector will be a copy per element, so it always makes sense to use a
757 // build_vector input in place of the extracted element to avoid a copy into a
758 // super register.
759 //
760 // We should probably only do this if all users are extracts only, but this
761 // should be the common case.
762 return true;
763 }
764
765 bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const {
766 // Truncate is just accessing a subregister.
767
768 unsigned SrcSize = Source.getSizeInBits();
769 unsigned DestSize = Dest.getSizeInBits();
770
771 return DestSize < SrcSize && DestSize % 32 == 0 ;
772 }
773
774 bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const {
775 // Truncate is just accessing a subregister.
776
777 unsigned SrcSize = Source->getScalarSizeInBits();
778 unsigned DestSize = Dest->getScalarSizeInBits();
779
780 if (DestSize== 16 && Subtarget->has16BitInsts())
781 return SrcSize >= 32;
782
783 return DestSize < SrcSize && DestSize % 32 == 0;
784 }
785
786 bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const {
787 unsigned SrcSize = Src->getScalarSizeInBits();
788 unsigned DestSize = Dest->getScalarSizeInBits();
789
790 if (SrcSize == 16 && Subtarget->has16BitInsts())
791 return DestSize >= 32;
792
793 return SrcSize == 32 && DestSize == 64;
794 }
795
796 bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const {
797 // Any register load of a 64-bit value really requires 2 32-bit moves. For all
798 // practical purposes, the extra mov 0 to load a 64-bit is free. As used,
799 // this will enable reducing 64-bit operations the 32-bit, which is always
800 // good.
801
802 if (Src == MVT::i16)
803 return Dest == MVT::i32 ||Dest == MVT::i64 ;
804
805 return Src == MVT::i32 && Dest == MVT::i64;
806 }
807
808 bool AMDGPUTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
809 return isZExtFree(Val.getValueType(), VT2);
810 }
811
812 bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const {
813 // There aren't really 64-bit registers, but pairs of 32-bit ones and only a
814 // limited number of native 64-bit operations. Shrinking an operation to fit
815 // in a single 32-bit register should always be helpful. As currently used,
816 // this is much less general than the name suggests, and is only used in
817 // places trying to reduce the sizes of loads. Shrinking loads to < 32-bits is
818 // not profitable, and may actually be harmful.
819 return SrcVT.getSizeInBits() > 32 && DestVT.getSizeInBits() == 32;
820 }
821
822 //===---------------------------------------------------------------------===//
823 // TargetLowering Callbacks
824 //===---------------------------------------------------------------------===//
825
826 CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
827 bool IsVarArg) {
828 switch (CC) {
829 case CallingConv::AMDGPU_VS:
830 case CallingConv::AMDGPU_GS:
831 case CallingConv::AMDGPU_PS:
832 case CallingConv::AMDGPU_CS:
833 case CallingConv::AMDGPU_HS:
834 case CallingConv::AMDGPU_ES:
835 case CallingConv::AMDGPU_LS:
836 return CC_AMDGPU;
837 case CallingConv::C:
838 case CallingConv::Fast:
839 case CallingConv::Cold:
840 return CC_AMDGPU_Func;
841 case CallingConv::AMDGPU_KERNEL:
842 case CallingConv::SPIR_KERNEL:
843 default:
844 report_fatal_error("Unsupported calling convention for call");
845 }
846 }
847
848 CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
849 bool IsVarArg) {
850 switch (CC) {
851 case CallingConv::AMDGPU_KERNEL:
852 case CallingConv::SPIR_KERNEL:
853 llvm_unreachable("kernels should not be handled here");
854 case CallingConv::AMDGPU_VS:
855 case CallingConv::AMDGPU_GS:
856 case CallingConv::AMDGPU_PS:
857 case CallingConv::AMDGPU_CS:
858 case CallingConv::AMDGPU_HS:
859 case CallingConv::AMDGPU_ES:
860 case CallingConv::AMDGPU_LS:
861 return RetCC_SI_Shader;
862 case CallingConv::C:
863 case CallingConv::Fast:
864 case CallingConv::Cold:
865 return RetCC_AMDGPU_Func;
866 default:
867 report_fatal_error("Unsupported calling convention.");
868 }
869 }
870
871 /// The SelectionDAGBuilder will automatically promote function arguments
872 /// with illegal types. However, this does not work for the AMDGPU targets
873 /// since the function arguments are stored in memory as these illegal types.
874 /// In order to handle this properly we need to get the original types sizes
875 /// from the LLVM IR Function and fixup the ISD:InputArg values before
876 /// passing them to AnalyzeFormalArguments()
877
878 /// When the SelectionDAGBuilder computes the Ins, it takes care of splitting
879 /// input values across multiple registers. Each item in the Ins array
880 /// represents a single value that will be stored in registers. Ins[x].VT is
881 /// the value type of the value that will be stored in the register, so
882 /// whatever SDNode we lower the argument to needs to be this type.
883 ///
884 /// In order to correctly lower the arguments we need to know the size of each
885 /// argument. Since Ins[x].VT gives us the size of the register that will
886 /// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type
887 /// for the orignal function argument so that we can deduce the correct memory
888 /// type to use for Ins[x]. In most cases the correct memory type will be
889 /// Ins[x].ArgVT. However, this will not always be the case. If, for example,
890 /// we have a kernel argument of type v8i8, this argument will be split into
891 /// 8 parts and each part will be represented by its own item in the Ins array.
892 /// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of
893 /// the argument before it was split. From this, we deduce that the memory type
894 /// for each individual part is i8. We pass the memory type as LocVT to the
895 /// calling convention analysis function and the register type (Ins[x].VT) as
896 /// the ValVT.
897 void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(
898 CCState &State,
899 const SmallVectorImpl<ISD::InputArg> &Ins) const {
900 const MachineFunction &MF = State.getMachineFunction();
901 const Function &Fn = MF.getFunction();
902 LLVMContext &Ctx = Fn.getParent()->getContext();
903 const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(MF);
904 const unsigned ExplicitOffset = ST.getExplicitKernelArgOffset(Fn);
905 CallingConv::ID CC = Fn.getCallingConv();
906
907 unsigned MaxAlign = 1;
908 uint64_t ExplicitArgOffset = 0;
909 const DataLayout &DL = Fn.getParent()->getDataLayout();
910
911 unsigned InIndex = 0;
912
913 for (const Argument &Arg : Fn.args()) {
914 Type *BaseArgTy = Arg.getType();
915 unsigned Align = DL.getABITypeAlignment(BaseArgTy);
916 MaxAlign = std::max(Align, MaxAlign);
917 unsigned AllocSize = DL.getTypeAllocSize(BaseArgTy);
918
919 uint64_t ArgOffset = alignTo(ExplicitArgOffset, Align) + ExplicitOffset;
920 ExplicitArgOffset = alignTo(ExplicitArgOffset, Align) + AllocSize;
921
922 // We're basically throwing away everything passed into us and starting over
923 // to get accurate in-memory offsets. The "PartOffset" is completely useless
924 // to us as computed in Ins.
925 //
926 // We also need to figure out what type legalization is trying to do to get
927 // the correct memory offsets.
928
929 SmallVector<EVT, 16> ValueVTs;
930 SmallVector<uint64_t, 16> Offsets;
931 ComputeValueVTs(*this, DL, BaseArgTy, ValueVTs, &Offsets, ArgOffset);
932
933 for (unsigned Value = 0, NumValues = ValueVTs.size();
934 Value != NumValues; ++Value) {
935 uint64_t BasePartOffset = Offsets[Value];
936
937 EVT ArgVT = ValueVTs[Value];
938 EVT MemVT = ArgVT;
939 MVT RegisterVT = getRegisterTypeForCallingConv(Ctx, CC, ArgVT);
940 unsigned NumRegs = getNumRegistersForCallingConv(Ctx, CC, ArgVT);
941
942 if (NumRegs == 1) {
943 // This argument is not split, so the IR type is the memory type.
944 if (ArgVT.isExtended()) {
945 // We have an extended type, like i24, so we should just use the
946 // register type.
947 MemVT = RegisterVT;
948 } else {
949 MemVT = ArgVT;
950 }
951 } else if (ArgVT.isVector() && RegisterVT.isVector() &&
952 ArgVT.getScalarType() == RegisterVT.getScalarType()) {
953 assert(ArgVT.getVectorNumElements() > RegisterVT.getVectorNumElements());
954 // We have a vector value which has been split into a vector with
955 // the same scalar type, but fewer elements. This should handle
956 // all the floating-point vector types.
957 MemVT = RegisterVT;
958 } else if (ArgVT.isVector() &&
959 ArgVT.getVectorNumElements() == NumRegs) {
960 // This arg has been split so that each element is stored in a separate
961 // register.
962 MemVT = ArgVT.getScalarType();
963 } else if (ArgVT.isExtended()) {
964 // We have an extended type, like i65.
965 MemVT = RegisterVT;
966 } else {
967 unsigned MemoryBits = ArgVT.getStoreSizeInBits() / NumRegs;
968 assert(ArgVT.getStoreSizeInBits() % NumRegs == 0);
969 if (RegisterVT.isInteger()) {
970 MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits);
971 } else if (RegisterVT.isVector()) {
972 assert(!RegisterVT.getScalarType().isFloatingPoint());
973 unsigned NumElements = RegisterVT.getVectorNumElements();
974 assert(MemoryBits % NumElements == 0);
975 // This vector type has been split into another vector type with
976 // a different elements size.
977 EVT ScalarVT = EVT::getIntegerVT(State.getContext(),
978 MemoryBits / NumElements);
979 MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements);
980 } else {
981 llvm_unreachable("cannot deduce memory type.");
982 }
983 }
984
985 // Convert one element vectors to scalar.
986 if (MemVT.isVector() && MemVT.getVectorNumElements() == 1)
987 MemVT = MemVT.getScalarType();
988
989 // Round up vec3/vec5 argument.
990 if (MemVT.isVector() && !MemVT.isPow2VectorType()) {
991 assert(MemVT.getVectorNumElements() == 3 ||
992 MemVT.getVectorNumElements() == 5);
993 MemVT = MemVT.getPow2VectorType(State.getContext());
994 }
995
996 unsigned PartOffset = 0;
997 for (unsigned i = 0; i != NumRegs; ++i) {
998 State.addLoc(CCValAssign::getCustomMem(InIndex++, RegisterVT,
999 BasePartOffset + PartOffset,
1000 MemVT.getSimpleVT(),
1001 CCValAssign::Full));
1002 PartOffset += MemVT.getStoreSize();
1003 }
1004 }
1005 }
1006 }
1007
1008 SDValue AMDGPUTargetLowering::LowerReturn(
1009 SDValue Chain, CallingConv::ID CallConv,
1010 bool isVarArg,
1011 const SmallVectorImpl<ISD::OutputArg> &Outs,
1012 const SmallVectorImpl<SDValue> &OutVals,
1013 const SDLoc &DL, SelectionDAG &DAG) const {
1014 // FIXME: Fails for r600 tests
1015 //assert(!isVarArg && Outs.empty() && OutVals.empty() &&
1016 // "wave terminate should not have return values");
1017 return DAG.getNode(AMDGPUISD::ENDPGM, DL, MVT::Other, Chain);
1018 }
1019
1020 //===---------------------------------------------------------------------===//
1021 // Target specific lowering
1022 //===---------------------------------------------------------------------===//
1023
1024 /// Selects the correct CCAssignFn for a given CallingConvention value.
1025 CCAssignFn *AMDGPUTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
1026 bool IsVarArg) {
1027 return AMDGPUCallLowering::CCAssignFnForCall(CC, IsVarArg);
1028 }
1029
1030 CCAssignFn *AMDGPUTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
1031 bool IsVarArg) {
1032 return AMDGPUCallLowering::CCAssignFnForReturn(CC, IsVarArg);
1033 }
1034
1035 SDValue AMDGPUTargetLowering::addTokenForArgument(SDValue Chain,
1036 SelectionDAG &DAG,
1037 MachineFrameInfo &MFI,
1038 int ClobberedFI) const {
1039 SmallVector<SDValue, 8> ArgChains;
1040 int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
1041 int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
1042
1043 // Include the original chain at the beginning of the list. When this is
1044 // used by target LowerCall hooks, this helps legalize find the
1045 // CALLSEQ_BEGIN node.
1046 ArgChains.push_back(Chain);
1047
1048 // Add a chain value for each stack argument corresponding
1049 for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
1050 UE = DAG.getEntryNode().getNode()->use_end();
1051 U != UE; ++U) {
1052 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) {
1053 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) {
1054 if (FI->getIndex() < 0) {
1055 int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
1056 int64_t InLastByte = InFirstByte;
1057 InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
1058
1059 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1060 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1061 ArgChains.push_back(SDValue(L, 1));
1062 }
1063 }
1064 }
1065 }
1066
1067 // Build a tokenfactor for all the chains.
1068 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
1069 }
1070
1071 SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
1072 SmallVectorImpl<SDValue> &InVals,
1073 StringRef Reason) const {
1074 SDValue Callee = CLI.Callee;
1075 SelectionDAG &DAG = CLI.DAG;
1076
1077 const Function &Fn = DAG.getMachineFunction().getFunction();
1078
1079 StringRef FuncName("<unknown>");
1080
1081 if (const ExternalSymbolSDNode *G = dyn_cast<ExternalSymbolSDNode>(Callee))
1082 FuncName = G->getSymbol();
1083 else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
1084 FuncName = G->getGlobal()->getName();
1085
1086 DiagnosticInfoUnsupported NoCalls(
1087 Fn, Reason + FuncName, CLI.DL.getDebugLoc());
1088 DAG.getContext()->diagnose(NoCalls);
1089
1090 if (!CLI.IsTailCall) {
1091 for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I)
1092 InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT));
1093 }
1094
1095 return DAG.getEntryNode();
1096 }
1097
1098 SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI,
1099 SmallVectorImpl<SDValue> &InVals) const {
1100 return lowerUnhandledCall(CLI, InVals, "unsupported call to function ");
1101 }
1102
1103 SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
1104 SelectionDAG &DAG) const {
1105 const Function &Fn = DAG.getMachineFunction().getFunction();
1106
1107 DiagnosticInfoUnsupported NoDynamicAlloca(Fn, "unsupported dynamic alloca",
1108 SDLoc(Op).getDebugLoc());
1109 DAG.getContext()->diagnose(NoDynamicAlloca);
1110 auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
1111 return DAG.getMergeValues(Ops, SDLoc());
1112 }
1113
1114 SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op,
1115 SelectionDAG &DAG) const {
1116 switch (Op.getOpcode()) {
1117 default:
1118 Op->print(errs(), &DAG);
1119 llvm_unreachable("Custom lowering code for this"
1120 "instruction is not implemented yet!");
1121 break;
1122 case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG);
1123 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
1124 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG);
1125 case ISD::UDIVREM: return LowerUDIVREM(Op, DAG);
1126 case ISD::SDIVREM: return LowerSDIVREM(Op, DAG);
1127 case ISD::FREM: return LowerFREM(Op, DAG);
1128 case ISD::FCEIL: return LowerFCEIL(Op, DAG);
1129 case ISD::FTRUNC: return LowerFTRUNC(Op, DAG);
1130 case ISD::FRINT: return LowerFRINT(Op, DAG);
1131 case ISD::FNEARBYINT: return LowerFNEARBYINT(Op, DAG);
1132 case ISD::FROUND: return LowerFROUND(Op, DAG);
1133 case ISD::FFLOOR: return LowerFFLOOR(Op, DAG);
1134 case ISD::FLOG:
1135 return LowerFLOG(Op, DAG, 1.0F / numbers::log2ef);
1136 case ISD::FLOG10:
1137 return LowerFLOG(Op, DAG, numbers::ln2f / numbers::ln10f);
1138 case ISD::FEXP:
1139 return lowerFEXP(Op, DAG);
1140 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
1141 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
1142 case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG);
1143 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG);
1144 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG);
1145 case ISD::CTTZ:
1146 case ISD::CTTZ_ZERO_UNDEF:
1147 case ISD::CTLZ:
1148 case ISD::CTLZ_ZERO_UNDEF:
1149 return LowerCTLZ_CTTZ(Op, DAG);
1150 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
1151 }
1152 return Op;
1153 }
1154
1155 void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N,
1156 SmallVectorImpl<SDValue> &Results,
1157 SelectionDAG &DAG) const {
1158 switch (N->getOpcode()) {
1159 case ISD::SIGN_EXTEND_INREG:
1160 // Different parts of legalization seem to interpret which type of
1161 // sign_extend_inreg is the one to check for custom lowering. The extended
1162 // from type is what really matters, but some places check for custom
1163 // lowering of the result type. This results in trying to use
1164 // ReplaceNodeResults to sext_in_reg to an illegal type, so we'll just do
1165 // nothing here and let the illegal result integer be handled normally.
1166 return;
1167 default:
1168 return;
1169 }
1170 }
1171
1172 bool AMDGPUTargetLowering::hasDefinedInitializer(const GlobalValue *GV) {
1173 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
1174 if (!GVar || !GVar->hasInitializer())
1175 return false;
1176
1177 return !isa<UndefValue>(GVar->getInitializer());
1178 }
1179
1180 SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
1181 SDValue Op,
1182 SelectionDAG &DAG) const {
1183
1184 const DataLayout &DL = DAG.getDataLayout();
1185 GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op);
1186 const GlobalValue *GV = G->getGlobal();
1187
1188 if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
1189 G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
1190 if (!MFI->isEntryFunction()) {
1191 const Function &Fn = DAG.getMachineFunction().getFunction();
1192 DiagnosticInfoUnsupported BadLDSDecl(
1193 Fn, "local memory global used by non-kernel function", SDLoc(Op).getDebugLoc());
1194 DAG.getContext()->diagnose(BadLDSDecl);
1195 }
1196
1197 // XXX: What does the value of G->getOffset() mean?
1198 assert(G->getOffset() == 0 &&
1199 "Do not know what to do with an non-zero offset");
1200
1201 // TODO: We could emit code to handle the initialization somewhere.
1202 if (!hasDefinedInitializer(GV)) {
1203 unsigned Offset = MFI->allocateLDSGlobal(DL, *GV);
1204 return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType());
1205 }
1206 }
1207
1208 const Function &Fn = DAG.getMachineFunction().getFunction();
1209 DiagnosticInfoUnsupported BadInit(
1210 Fn, "unsupported initializer for address space", SDLoc(Op).getDebugLoc());
1211 DAG.getContext()->diagnose(BadInit);
1212 return SDValue();
1213 }
1214
1215 SDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op,
1216 SelectionDAG &DAG) const {
1217 SmallVector<SDValue, 8> Args;
1218
1219 EVT VT = Op.getValueType();
1220 if (VT == MVT::v4i16 || VT == MVT::v4f16) {
1221 SDLoc SL(Op);
1222 SDValue Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(0));
1223 SDValue Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Op.getOperand(1));
1224
1225 SDValue BV = DAG.getBuildVector(MVT::v2i32, SL, { Lo, Hi });
1226 return DAG.getNode(ISD::BITCAST, SL, VT, BV);
1227 }
1228
1229 for (const SDUse &U : Op->ops())
1230 DAG.ExtractVectorElements(U.get(), Args);
1231
1232 return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
1233 }
1234
1235 SDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
1236 SelectionDAG &DAG) const {
1237
1238 SmallVector<SDValue, 8> Args;
1239 unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1240 EVT VT = Op.getValueType();
1241 DAG.ExtractVectorElements(Op.getOperand(0), Args, Start,
1242 VT.getVectorNumElements());
1243
1244 return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Args);
1245 }
1246
1247 /// Generate Min/Max node
1248 SDValue AMDGPUTargetLowering::combineFMinMaxLegacy(const SDLoc &DL, EVT VT,
1249 SDValue LHS, SDValue RHS,
1250 SDValue True, SDValue False,
1251 SDValue CC,
1252 DAGCombinerInfo &DCI) const {
1253 if (!(LHS == True && RHS == False) && !(LHS == False && RHS == True))
1254 return SDValue();
1255
1256 SelectionDAG &DAG = DCI.DAG;
1257 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get();
1258 switch (CCOpcode) {
1259 case ISD::SETOEQ:
1260 case ISD::SETONE:
1261 case ISD::SETUNE:
1262 case ISD::SETNE:
1263 case ISD::SETUEQ:
1264 case ISD::SETEQ:
1265 case ISD::SETFALSE:
1266 case ISD::SETFALSE2:
1267 case ISD::SETTRUE:
1268 case ISD::SETTRUE2:
1269 case ISD::SETUO:
1270 case ISD::SETO:
1271 break;
1272 case ISD::SETULE:
1273 case ISD::SETULT: {
1274 if (LHS == True)
1275 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1276 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1277 }
1278 case ISD::SETOLE:
1279 case ISD::SETOLT:
1280 case ISD::SETLE:
1281 case ISD::SETLT: {
1282 // Ordered. Assume ordered for undefined.
1283
1284 // Only do this after legalization to avoid interfering with other combines
1285 // which might occur.
1286 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1287 !DCI.isCalledByLegalizer())
1288 return SDValue();
1289
1290 // We need to permute the operands to get the correct NaN behavior. The
1291 // selected operand is the second one based on the failing compare with NaN,
1292 // so permute it based on the compare type the hardware uses.
1293 if (LHS == True)
1294 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1295 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1296 }
1297 case ISD::SETUGE:
1298 case ISD::SETUGT: {
1299 if (LHS == True)
1300 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, RHS, LHS);
1301 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, LHS, RHS);
1302 }
1303 case ISD::SETGT:
1304 case ISD::SETGE:
1305 case ISD::SETOGE:
1306 case ISD::SETOGT: {
1307 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG &&
1308 !DCI.isCalledByLegalizer())
1309 return SDValue();
1310
1311 if (LHS == True)
1312 return DAG.getNode(AMDGPUISD::FMAX_LEGACY, DL, VT, LHS, RHS);
1313 return DAG.getNode(AMDGPUISD::FMIN_LEGACY, DL, VT, RHS, LHS);
1314 }
1315 case ISD::SETCC_INVALID:
1316 llvm_unreachable("Invalid setcc condcode!");
1317 }
1318 return SDValue();
1319 }
1320
1321 std::pair<SDValue, SDValue>
1322 AMDGPUTargetLowering::split64BitValue(SDValue Op, SelectionDAG &DAG) const {
1323 SDLoc SL(Op);
1324
1325 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1326
1327 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1328 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1329
1330 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1331 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1332
1333 return std::make_pair(Lo, Hi);
1334 }
1335
1336 SDValue AMDGPUTargetLowering::getLoHalf64(SDValue Op, SelectionDAG &DAG) const {
1337 SDLoc SL(Op);
1338
1339 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1340 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
1341 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
1342 }
1343
1344 SDValue AMDGPUTargetLowering::getHiHalf64(SDValue Op, SelectionDAG &DAG) const {
1345 SDLoc SL(Op);
1346
1347 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Op);
1348 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
1349 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
1350 }
1351
1352 // Split a vector type into two parts. The first part is a power of two vector.
1353 // The second part is whatever is left over, and is a scalar if it would
1354 // otherwise be a 1-vector.
1355 std::pair<EVT, EVT>
1356 AMDGPUTargetLowering::getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const {
1357 EVT LoVT, HiVT;
1358 EVT EltVT = VT.getVectorElementType();
1359 unsigned NumElts = VT.getVectorNumElements();
1360 unsigned LoNumElts = PowerOf2Ceil((NumElts + 1) / 2);
1361 LoVT = EVT::getVectorVT(*DAG.getContext(), EltVT, LoNumElts);
1362 HiVT = NumElts - LoNumElts == 1
1363 ? EltVT
1364 : EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts - LoNumElts);
1365 return std::make_pair(LoVT, HiVT);
1366 }
1367
1368 // Split a vector value into two parts of types LoVT and HiVT. HiVT could be
1369 // scalar.
1370 std::pair<SDValue, SDValue>
1371 AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
1372 const EVT &LoVT, const EVT &HiVT,
1373 SelectionDAG &DAG) const {
1374 assert(LoVT.getVectorNumElements() +
1375 (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
1376 N.getValueType().getVectorNumElements() &&
1377 "More vector elements requested than available!");
1378 auto IdxTy = getVectorIdxTy(DAG.getDataLayout());
1379 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
1380 DAG.getConstant(0, DL, IdxTy));
1381 SDValue Hi = DAG.getNode(
1382 HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL,
1383 HiVT, N, DAG.getConstant(LoVT.getVectorNumElements(), DL, IdxTy));
1384 return std::make_pair(Lo, Hi);
1385 }
1386
1387 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
1388 SelectionDAG &DAG) const {
1389 LoadSDNode *Load = cast<LoadSDNode>(Op);
1390 EVT VT = Op.getValueType();
1391
1392
1393 // If this is a 2 element vector, we really want to scalarize and not create
1394 // weird 1 element vectors.
1395 if (VT.getVectorNumElements() == 2)
1396 return scalarizeVectorLoad(Load, DAG);
1397
1398 SDValue BasePtr = Load->getBasePtr();
1399 EVT MemVT = Load->getMemoryVT();
1400 SDLoc SL(Op);
1401
1402 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1403
1404 EVT LoVT, HiVT;
1405 EVT LoMemVT, HiMemVT;
1406 SDValue Lo, Hi;
1407
1408 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1409 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
1410 std::tie(Lo, Hi) = splitVector(Op, SL, LoVT, HiVT, DAG);
1411
1412 unsigned Size = LoMemVT.getStoreSize();
1413 unsigned BaseAlign = Load->getAlignment();
1414 unsigned HiAlign = MinAlign(BaseAlign, Size);
1415
1416 SDValue LoLoad = DAG.getExtLoad(Load->getExtensionType(), SL, LoVT,
1417 Load->getChain(), BasePtr, SrcValue, LoMemVT,
1418 BaseAlign, Load->getMemOperand()->getFlags());
1419 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, Size);
1420 SDValue HiLoad =
1421 DAG.getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
1422 HiPtr, SrcValue.getWithOffset(LoMemVT.getStoreSize()),
1423 HiMemVT, HiAlign, Load->getMemOperand()->getFlags());
1424
1425 auto IdxTy = getVectorIdxTy(DAG.getDataLayout());
1426 SDValue Join;
1427 if (LoVT == HiVT) {
1428 // This is the case that the vector is power of two so was evenly split.
1429 Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
1430 } else {
1431 Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad,
1432 DAG.getConstant(0, SL, IdxTy));
1433 Join = DAG.getNode(HiVT.isVector() ? ISD::INSERT_SUBVECTOR
1434 : ISD::INSERT_VECTOR_ELT,
1435 SL, VT, Join, HiLoad,
1436 DAG.getConstant(LoVT.getVectorNumElements(), SL, IdxTy));
1437 }
1438
1439 SDValue Ops[] = {Join, DAG.getNode(ISD::TokenFactor, SL, MVT::Other,
1440 LoLoad.getValue(1), HiLoad.getValue(1))};
1441
1442 return DAG.getMergeValues(Ops, SL);
1443 }
1444
1445 // Widen a vector load from vec3 to vec4.
1446 SDValue AMDGPUTargetLowering::WidenVectorLoad(SDValue Op,
1447 SelectionDAG &DAG) const {
1448 LoadSDNode *Load = cast<LoadSDNode>(Op);
1449 EVT VT = Op.getValueType();
1450 assert(VT.getVectorNumElements() == 3);
1451 SDValue BasePtr = Load->getBasePtr();
1452 EVT MemVT = Load->getMemoryVT();
1453 SDLoc SL(Op);
1454 const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo();
1455 unsigned BaseAlign = Load->getAlignment();
1456
1457 EVT WideVT =
1458 EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4);
1459 EVT WideMemVT =
1460 EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(), 4);
1461 SDValue WideLoad = DAG.getExtLoad(
1462 Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue,
1463 WideMemVT, BaseAlign, Load->getMemOperand()->getFlags());
1464 return DAG.getMergeValues(
1465 {DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, VT, WideLoad,
1466 DAG.getConstant(0, SL, getVectorIdxTy(DAG.getDataLayout()))),
1467 WideLoad.getValue(1)},
1468 SL);
1469 }
1470
1471 SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op,
1472 SelectionDAG &DAG) const {
1473 StoreSDNode *Store = cast<StoreSDNode>(Op);
1474 SDValue Val = Store->getValue();
1475 EVT VT = Val.getValueType();
1476
1477 // If this is a 2 element vector, we really want to scalarize and not create
1478 // weird 1 element vectors.
1479 if (VT.getVectorNumElements() == 2)
1480 return scalarizeVectorStore(Store, DAG);
1481
1482 EVT MemVT = Store->getMemoryVT();
1483 SDValue Chain = Store->getChain();
1484 SDValue BasePtr = Store->getBasePtr();
1485 SDLoc SL(Op);
1486
1487 EVT LoVT, HiVT;
1488 EVT LoMemVT, HiMemVT;
1489 SDValue Lo, Hi;
1490
1491 std::tie(LoVT, HiVT) = getSplitDestVTs(VT, DAG);
1492 std::tie(LoMemVT, HiMemVT) = getSplitDestVTs(MemVT, DAG);
1493 std::tie(Lo, Hi) = splitVector(Val, SL, LoVT, HiVT, DAG);
1494
1495 SDValue HiPtr = DAG.getObjectPtrOffset(SL, BasePtr, LoMemVT.getStoreSize());
1496
1497 const MachinePointerInfo &SrcValue = Store->getMemOperand()->getPointerInfo();
1498 unsigned BaseAlign = Store->getAlignment();
1499 unsigned Size = LoMemVT.getStoreSize();
1500 unsigned HiAlign = MinAlign(BaseAlign, Size);
1501
1502 SDValue LoStore =
1503 DAG.getTruncStore(Chain, SL, Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1504 Store->getMemOperand()->getFlags());
1505 SDValue HiStore =
1506 DAG.getTruncStore(Chain, SL, Hi, HiPtr, SrcValue.getWithOffset(Size),
1507 HiMemVT, HiAlign, Store->getMemOperand()->getFlags());
1508
1509 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore);
1510 }
1511
1512 // This is a shortcut for integer division because we have fast i32<->f32
1513 // conversions, and fast f32 reciprocal instructions. The fractional part of a
1514 // float is enough to accurately represent up to a 24-bit signed integer.
1515 SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG,
1516 bool Sign) const {
1517 SDLoc DL(Op);
1518 EVT VT = Op.getValueType();
1519 SDValue LHS = Op.getOperand(0);
1520 SDValue RHS = Op.getOperand(1);
1521 MVT IntVT = MVT::i32;
1522 MVT FltVT = MVT::f32;
1523
1524 unsigned LHSSignBits = DAG.ComputeNumSignBits(LHS);
1525 if (LHSSignBits < 9)
1526 return SDValue();
1527
1528 unsigned RHSSignBits = DAG.ComputeNumSignBits(RHS);
1529 if (RHSSignBits < 9)
1530 return SDValue();
1531
1532 unsigned BitSize = VT.getSizeInBits();
1533 unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
1534 unsigned DivBits = BitSize - SignBits;
1535 if (Sign)
1536 ++DivBits;
1537
1538 ISD::NodeType ToFp = Sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
1539 ISD::NodeType ToInt = Sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
1540
1541 SDValue jq = DAG.getConstant(1, DL, IntVT);
1542
1543 if (Sign) {
1544 // char|short jq = ia ^ ib;
1545 jq = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS);
1546
1547 // jq = jq >> (bitsize - 2)
1548 jq = DAG.getNode(ISD::SRA, DL, VT, jq,
1549 DAG.getConstant(BitSize - 2, DL, VT));
1550
1551 // jq = jq | 0x1
1552 jq = DAG.getNode(ISD::OR, DL, VT, jq, DAG.getConstant(1, DL, VT));
1553 }
1554
1555 // int ia = (int)LHS;
1556 SDValue ia = LHS;
1557
1558 // int ib, (int)RHS;
1559 SDValue ib = RHS;
1560
1561 // float fa = (float)ia;
1562 SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
1563
1564 // float fb = (float)ib;
1565 SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
1566
1567 SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
1568 fa, DAG.getNode(AMDGPUISD::RCP, DL, FltVT, fb));
1569
1570 // fq = trunc(fq);
1571 fq = DAG.getNode(ISD::FTRUNC, DL, FltVT, fq);
1572
1573 // float fqneg = -fq;
1574 SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq);
1575
1576 // float fr = mad(fqneg, fb, fa);
1577 unsigned OpCode = Subtarget->hasFP32Denormals() ?
1578 (unsigned)AMDGPUISD::FMAD_FTZ :
1579 (unsigned)ISD::FMAD;
1580 SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa);
1581
1582 // int iq = (int)fq;
1583 SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
1584
1585 // fr = fabs(fr);
1586 fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
1587
1588 // fb = fabs(fb);
1589 fb = DAG.getNode(ISD::FABS, DL, FltVT, fb);
1590
1591 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
1592
1593 // int cv = fr >= fb;
1594 SDValue cv = DAG.getSetCC(DL, SetCCVT, fr, fb, ISD::SETOGE);
1595
1596 // jq = (cv ? jq : 0);
1597 jq = DAG.getNode(ISD::SELECT, DL, VT, cv, jq, DAG.getConstant(0, DL, VT));
1598
1599 // dst = iq + jq;
1600 SDValue Div = DAG.getNode(ISD::ADD, DL, VT, iq, jq);
1601
1602 // Rem needs compensation, it's easier to recompute it
1603 SDValue Rem = DAG.getNode(ISD::MUL, DL, VT, Div, RHS);
1604 Rem = DAG.getNode(ISD::SUB, DL, VT, LHS, Rem);
1605
1606 // Truncate to number of bits this divide really is.
1607 if (Sign) {
1608 SDValue InRegSize
1609 = DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), DivBits));
1610 Div = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Div, InRegSize);
1611 Rem = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Rem, InRegSize);
1612 } else {
1613 SDValue TruncMask = DAG.getConstant((UINT64_C(1) << DivBits) - 1, DL, VT);
1614 Div = DAG.getNode(ISD::AND, DL, VT, Div, TruncMask);
1615 Rem = DAG.getNode(ISD::AND, DL, VT, Rem, TruncMask);
1616 }
1617
1618 return DAG.getMergeValues({ Div, Rem }, DL);
1619 }
1620
1621 void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op,
1622 SelectionDAG &DAG,
1623 SmallVectorImpl<SDValue> &Results) const {
1624 SDLoc DL(Op);
1625 EVT VT = Op.getValueType();
1626
1627 assert(VT == MVT::i64 && "LowerUDIVREM64 expects an i64");
1628
1629 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
1630
1631 SDValue One = DAG.getConstant(1, DL, HalfVT);
1632 SDValue Zero = DAG.getConstant(0, DL, HalfVT);
1633
1634 //HiLo split
1635 SDValue LHS = Op.getOperand(0);
1636 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1637 SDValue LHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, One);
1638
1639 SDValue RHS = Op.getOperand(1);
1640 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1641 SDValue RHS_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, One);
1642
1643 if (DAG.MaskedValueIsZero(RHS, APInt::getHighBitsSet(64, 32)) &&
1644 DAG.MaskedValueIsZero(LHS, APInt::getHighBitsSet(64, 32))) {
1645
1646 SDValue Res = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1647 LHS_Lo, RHS_Lo);
1648
1649 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(0), Zero});
1650 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {Res.getValue(1), Zero});
1651
1652 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV));
1653 Results.push_back(DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM));
1654 return;
1655 }
1656
1657 if (isTypeLegal(MVT::i64)) {
1658 // Compute denominator reciprocal.
1659 unsigned FMAD = Subtarget->hasFP32Denormals() ?
1660 (unsigned)AMDGPUISD::FMAD_FTZ :
1661 (unsigned)ISD::FMAD;
1662
1663 SDValue Cvt_Lo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Lo);
1664 SDValue Cvt_Hi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, RHS_Hi);
1665 SDValue Mad1 = DAG.getNode(FMAD, DL, MVT::f32, Cvt_Hi,
1666 DAG.getConstantFP(APInt(32, 0x4f800000).bitsToFloat(), DL, MVT::f32),
1667 Cvt_Lo);
1668 SDValue Rcp = DAG.getNode(AMDGPUISD::RCP, DL, MVT::f32, Mad1);
1669 SDValue Mul1 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Rcp,
1670 DAG.getConstantFP(APInt(32, 0x5f7ffffc).bitsToFloat(), DL, MVT::f32));
1671 SDValue Mul2 = DAG.getNode(ISD::FMUL, DL, MVT::f32, Mul1,
1672 DAG.getConstantFP(APInt(32, 0x2f800000).bitsToFloat(), DL, MVT::f32));
1673 SDValue Trunc = DAG.getNode(ISD::FTRUNC, DL, MVT::f32, Mul2);
1674 SDValue Mad2 = DAG.getNode(FMAD, DL, MVT::f32, Trunc,
1675 DAG.getConstantFP(APInt(32, 0xcf800000).bitsToFloat(), DL, MVT::f32),
1676 Mul1);
1677 SDValue Rcp_Lo = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Mad2);
1678 SDValue Rcp_Hi = DAG.getNode(ISD::FP_TO_UINT, DL, HalfVT, Trunc);
1679 SDValue Rcp64 = DAG.getBitcast(VT,
1680 DAG.getBuildVector(MVT::v2i32, DL, {Rcp_Lo, Rcp_Hi}));
1681
1682 SDValue Zero64 = DAG.getConstant(0, DL, VT);
1683 SDValue One64 = DAG.getConstant(1, DL, VT);
1684 SDValue Zero1 = DAG.getConstant(0, DL, MVT::i1);
1685 SDVTList HalfCarryVT = DAG.getVTList(HalfVT, MVT::i1);
1686
1687 SDValue Neg_RHS = DAG.getNode(ISD::SUB, DL, VT, Zero64, RHS);
1688 SDValue Mullo1 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Rcp64);
1689 SDValue Mulhi1 = DAG.getNode(ISD::MULHU, DL, VT, Rcp64, Mullo1);
1690 SDValue Mulhi1_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1691 Zero);
1692 SDValue Mulhi1_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi1,
1693 One);
1694
1695 SDValue Add1_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Lo,
1696 Mulhi1_Lo, Zero1);
1697 SDValue Add1_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Rcp_Hi,
1698 Mulhi1_Hi, Add1_Lo.getValue(1));
1699 SDValue Add1_HiNc = DAG.getNode(ISD::ADD, DL, HalfVT, Rcp_Hi, Mulhi1_Hi);
1700 SDValue Add1 = DAG.getBitcast(VT,
1701 DAG.getBuildVector(MVT::v2i32, DL, {Add1_Lo, Add1_Hi}));
1702
1703 SDValue Mullo2 = DAG.getNode(ISD::MUL, DL, VT, Neg_RHS, Add1);
1704 SDValue Mulhi2 = DAG.getNode(ISD::MULHU, DL, VT, Add1, Mullo2);
1705 SDValue Mulhi2_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1706 Zero);
1707 SDValue Mulhi2_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mulhi2,
1708 One);
1709
1710 SDValue Add2_Lo = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_Lo,
1711 Mulhi2_Lo, Zero1);
1712 SDValue Add2_HiC = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add1_HiNc,
1713 Mulhi2_Hi, Add1_Lo.getValue(1));
1714 SDValue Add2_Hi = DAG.getNode(ISD::ADDCARRY, DL, HalfCarryVT, Add2_HiC,
1715 Zero, Add2_Lo.getValue(1));
1716 SDValue Add2 = DAG.getBitcast(VT,
1717 DAG.getBuildVector(MVT::v2i32, DL, {Add2_Lo, Add2_Hi}));
1718 SDValue Mulhi3 = DAG.getNode(ISD::MULHU, DL, VT, LHS, Add2);
1719
1720 SDValue Mul3 = DAG.getNode(ISD::MUL, DL, VT, RHS, Mulhi3);
1721
1722 SDValue Mul3_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, Zero);
1723 SDValue Mul3_Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, Mul3, One);
1724 SDValue Sub1_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Lo,
1725 Mul3_Lo, Zero1);
1726 SDValue Sub1_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, LHS_Hi,
1727 Mul3_Hi, Sub1_Lo.getValue(1));
1728 SDValue Sub1_Mi = DAG.getNode(ISD::SUB, DL, HalfVT, LHS_Hi, Mul3_Hi);
1729 SDValue Sub1 = DAG.getBitcast(VT,
1730 DAG.getBuildVector(MVT::v2i32, DL, {Sub1_Lo, Sub1_Hi}));
1731
1732 SDValue MinusOne = DAG.getConstant(0xffffffffu, DL, HalfVT);
1733 SDValue C1 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, MinusOne, Zero,
1734 ISD::SETUGE);
1735 SDValue C2 = DAG.getSelectCC(DL, Sub1_Lo, RHS_Lo, MinusOne, Zero,
1736 ISD::SETUGE);
1737 SDValue C3 = DAG.getSelectCC(DL, Sub1_Hi, RHS_Hi, C2, C1, ISD::SETEQ);
1738
1739 // TODO: Here and below portions of the code can be enclosed into if/endif.
1740 // Currently control flow is unconditional and we have 4 selects after
1741 // potential endif to substitute PHIs.
1742
1743 // if C3 != 0 ...
1744 SDValue Sub2_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Lo,
1745 RHS_Lo, Zero1);
1746 SDValue Sub2_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub1_Mi,
1747 RHS_Hi, Sub1_Lo.getValue(1));
1748 SDValue Sub2_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1749 Zero, Sub2_Lo.getValue(1));
1750 SDValue Sub2 = DAG.getBitcast(VT,
1751 DAG.getBuildVector(MVT::v2i32, DL, {Sub2_Lo, Sub2_Hi}));
1752
1753 SDValue Add3 = DAG.getNode(ISD::ADD, DL, VT, Mulhi3, One64);
1754
1755 SDValue C4 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, MinusOne, Zero,
1756 ISD::SETUGE);
1757 SDValue C5 = DAG.getSelectCC(DL, Sub2_Lo, RHS_Lo, MinusOne, Zero,
1758 ISD::SETUGE);
1759 SDValue C6 = DAG.getSelectCC(DL, Sub2_Hi, RHS_Hi, C5, C4, ISD::SETEQ);
1760
1761 // if (C6 != 0)
1762 SDValue Add4 = DAG.getNode(ISD::ADD, DL, VT, Add3, One64);
1763
1764 SDValue Sub3_Lo = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Lo,
1765 RHS_Lo, Zero1);
1766 SDValue Sub3_Mi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub2_Mi,
1767 RHS_Hi, Sub2_Lo.getValue(1));
1768 SDValue Sub3_Hi = DAG.getNode(ISD::SUBCARRY, DL, HalfCarryVT, Sub3_Mi,
1769 Zero, Sub3_Lo.getValue(1));
1770 SDValue Sub3 = DAG.getBitcast(VT,
1771 DAG.getBuildVector(MVT::v2i32, DL, {Sub3_Lo, Sub3_Hi}));
1772
1773 // endif C6
1774 // endif C3
1775
1776 SDValue Sel1 = DAG.getSelectCC(DL, C6, Zero, Add4, Add3, ISD::SETNE);
1777 SDValue Div = DAG.getSelectCC(DL, C3, Zero, Sel1, Mulhi3, ISD::SETNE);
1778
1779 SDValue Sel2 = DAG.getSelectCC(DL, C6, Zero, Sub3, Sub2, ISD::SETNE);
1780 SDValue Rem = DAG.getSelectCC(DL, C3, Zero, Sel2, Sub1, ISD::SETNE);
1781
1782 Results.push_back(Div);
1783 Results.push_back(Rem);
1784
1785 return;
1786 }
1787
1788 // r600 expandion.
1789 // Get Speculative values
1790 SDValue DIV_Part = DAG.getNode(ISD::UDIV, DL, HalfVT, LHS_Hi, RHS_Lo);
1791 SDValue REM_Part = DAG.getNode(ISD::UREM, DL, HalfVT, LHS_Hi, RHS_Lo);
1792
1793 SDValue REM_Lo = DAG.getSelectCC(DL, RHS_Hi, Zero, REM_Part, LHS_Hi, ISD::SETEQ);
1794 SDValue REM = DAG.getBuildVector(MVT::v2i32, DL, {REM_Lo, Zero});
1795 REM = DAG.getNode(ISD::BITCAST, DL, MVT::i64, REM);
1796
1797 SDValue DIV_Hi = DAG.getSelectCC(DL, RHS_Hi, Zero, DIV_Part, Zero, ISD::SETEQ);
1798 SDValue DIV_Lo = Zero;
1799
1800 const unsigned halfBitWidth = HalfVT.getSizeInBits();
1801
1802 for (unsigned i = 0; i < halfBitWidth; ++i) {
1803 const unsigned bitPos = halfBitWidth - i - 1;
1804 SDValue POS = DAG.getConstant(bitPos, DL, HalfVT);
1805 // Get value of high bit
1806 SDValue HBit = DAG.getNode(ISD::SRL, DL, HalfVT, LHS_Lo, POS);
1807 HBit = DAG.getNode(ISD::AND, DL, HalfVT, HBit, One);
1808 HBit = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, HBit);
1809
1810 // Shift
1811 REM = DAG.getNode(ISD::SHL, DL, VT, REM, DAG.getConstant(1, DL, VT));
1812 // Add LHS high bit
1813 REM = DAG.getNode(ISD::OR, DL, VT, REM, HBit);
1814
1815 SDValue BIT = DAG.getConstant(1ULL << bitPos, DL, HalfVT);
1816 SDValue realBIT = DAG.getSelectCC(DL, REM, RHS, BIT, Zero, ISD::SETUGE);
1817
1818 DIV_Lo = DAG.getNode(ISD::OR, DL, HalfVT, DIV_Lo, realBIT);
1819
1820 // Update REM
1821 SDValue REM_sub = DAG.getNode(ISD::SUB, DL, VT, REM, RHS);
1822 REM = DAG.getSelectCC(DL, REM, RHS, REM_sub, REM, ISD::SETUGE);
1823 }
1824
1825 SDValue DIV = DAG.getBuildVector(MVT::v2i32, DL, {DIV_Lo, DIV_Hi});
1826 DIV = DAG.getNode(ISD::BITCAST, DL, MVT::i64, DIV);
1827 Results.push_back(DIV);
1828 Results.push_back(REM);
1829 }
1830
1831 SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
1832 SelectionDAG &DAG) const {
1833 SDLoc DL(Op);
1834 EVT VT = Op.getValueType();
1835
1836 if (VT == MVT::i64) {
1837 SmallVector<SDValue, 2> Results;
1838 LowerUDIVREM64(Op, DAG, Results);
1839 return DAG.getMergeValues(Results, DL);
1840 }
1841
1842 if (VT == MVT::i32) {
1843 if (SDValue Res = LowerDIVREM24(Op, DAG, false))
1844 return Res;
1845 }
1846
1847 SDValue Num = Op.getOperand(0);
1848 SDValue Den = Op.getOperand(1);
1849
1850 // RCP = URECIP(Den) = 2^32 / Den + e
1851 // e is rounding error.
1852 SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
1853
1854 // RCP_LO = mul(RCP, Den) */
1855 SDValue RCP_LO = DAG.getNode(ISD::MUL, DL, VT, RCP, Den);
1856
1857 // RCP_HI = mulhu (RCP, Den) */
1858 SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den);
1859
1860 // NEG_RCP_LO = -RCP_LO
1861 SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
1862 RCP_LO);
1863
1864 // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO)
1865 SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
1866 NEG_RCP_LO, RCP_LO,
1867 ISD::SETEQ);
1868 // Calculate the rounding error from the URECIP instruction
1869 // E = mulhu(ABS_RCP_LO, RCP)
1870 SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP);
1871
1872 // RCP_A_E = RCP + E
1873 SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E);
1874
1875 // RCP_S_E = RCP - E
1876 SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E);
1877
1878 // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E)
1879 SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, DL, VT),
1880 RCP_A_E, RCP_S_E,
1881 ISD::SETEQ);
1882 // Quotient = mulhu(Tmp0, Num)
1883 SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num);
1884
1885 // Num_S_Remainder = Quotient * Den
1886 SDValue Num_S_Remainder = DAG.getNode(ISD::MUL, DL, VT, Quotient, Den);
1887
1888 // Remainder = Num - Num_S_Remainder
1889 SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder);
1890
1891 // Remainder_GE_Den = (Remainder >= Den ? -1 : 0)
1892 SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den,
1893 DAG.getConstant(-1, DL, VT),
1894 DAG.getConstant(0, DL, VT),
1895 ISD::SETUGE);
1896 // Remainder_GE_Zero = (Num >= Num_S_Remainder ? -1 : 0)
1897 SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Num,
1898 Num_S_Remainder,
1899 DAG.getConstant(-1, DL, VT),
1900 DAG.getConstant(0, DL, VT),
1901 ISD::SETUGE);
1902 // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero
1903 SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den,
1904 Remainder_GE_Zero);
1905
1906 // Calculate Division result:
1907
1908 // Quotient_A_One = Quotient + 1
1909 SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient,
1910 DAG.getConstant(1, DL, VT));
1911
1912 // Quotient_S_One = Quotient - 1
1913 SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient,
1914 DAG.getConstant(1, DL, VT));
1915
1916 // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One)
1917 SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
1918 Quotient, Quotient_A_One, ISD::SETEQ);
1919
1920 // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div)
1921 Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
1922 Quotient_S_One, Div, ISD::SETEQ);
1923
1924 // Calculate Rem result:
1925
1926 // Remainder_S_Den = Remainder - Den
1927 SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den);
1928
1929 // Remainder_A_Den = Remainder + Den
1930 SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den);
1931
1932 // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den)
1933 SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, DL, VT),
1934 Remainder, Remainder_S_Den, ISD::SETEQ);
1935
1936 // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem)
1937 Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, DL, VT),
1938 Remainder_A_Den, Rem, ISD::SETEQ);
1939 SDValue Ops[2] = {
1940 Div,
1941 Rem
1942 };
1943 return DAG.getMergeValues(Ops, DL);
1944 }
1945
1946 SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
1947 SelectionDAG &DAG) const {
1948 SDLoc DL(Op);
1949 EVT VT = Op.getValueType();
1950
1951 SDValue LHS = Op.getOperand(0);
1952 SDValue RHS = Op.getOperand(1);
1953
1954 SDValue Zero = DAG.getConstant(0, DL, VT);
1955 SDValue NegOne = DAG.getConstant(-1, DL, VT);
1956
1957 if (VT == MVT::i32) {
1958 if (SDValue Res = LowerDIVREM24(Op, DAG, true))
1959 return Res;
1960 }
1961
1962 if (VT == MVT::i64 &&
1963 DAG.ComputeNumSignBits(LHS) > 32 &&
1964 DAG.ComputeNumSignBits(RHS) > 32) {
1965 EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
1966
1967 //HiLo split
1968 SDValue LHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, LHS, Zero);
1969 SDValue RHS_Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, HalfVT, RHS, Zero);
1970 SDValue DIVREM = DAG.getNode(ISD::SDIVREM, DL, DAG.getVTList(HalfVT, HalfVT),
1971 LHS_Lo, RHS_Lo);
1972 SDValue Res[2] = {
1973 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(0)),
1974 DAG.getNode(ISD::SIGN_EXTEND, DL, VT, DIVREM.getValue(1))
1975 };
1976 return DAG.getMergeValues(Res, DL);
1977 }
1978
1979 SDValue LHSign = DAG.getSelectCC(DL, LHS, Zero, NegOne, Zero, ISD::SETLT);
1980 SDValue RHSign = DAG.getSelectCC(DL, RHS, Zero, NegOne, Zero, ISD::SETLT);
1981 SDValue DSign = DAG.getNode(ISD::XOR, DL, VT, LHSign, RHSign);
1982 SDValue RSign = LHSign; // Remainder sign is the same as LHS
1983
1984 LHS = DAG.getNode(ISD::ADD, DL, VT, LHS, LHSign);
1985 RHS = DAG.getNode(ISD::ADD, DL, VT, RHS, RHSign);
1986
1987 LHS = DAG.getNode(ISD::XOR, DL, VT, LHS, LHSign);
1988 RHS = DAG.getNode(ISD::XOR, DL, VT, RHS, RHSign);
1989
1990 SDValue Div = DAG.getNode(ISD::UDIVREM, DL, DAG.getVTList(VT, VT), LHS, RHS);
1991 SDValue Rem = Div.getValue(1);
1992
1993 Div = DAG.getNode(ISD::XOR, DL, VT, Div, DSign);
1994 Rem = DAG.getNode(ISD::XOR, DL, VT, Rem, RSign);
1995
1996 Div = DAG.getNode(ISD::SUB, DL, VT, Div, DSign);
1997 Rem = DAG.getNode(ISD::SUB, DL, VT, Rem, RSign);
1998
1999 SDValue Res[2] = {
2000 Div,
2001 Rem
2002 };
2003 return DAG.getMergeValues(Res, DL);
2004 }
2005
2006 // (frem x, y) -> (fsub x, (fmul (ftrunc (fdiv x, y)), y))
2007 SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const {
2008 SDLoc SL(Op);
2009 EVT VT = Op.getValueType();
2010 SDValue X = Op.getOperand(0);
2011 SDValue Y = Op.getOperand(1);
2012
2013 // TODO: Should this propagate fast-math-flags?
2014
2015 SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y);
2016 SDValue Floor = DAG.getNode(ISD::FTRUNC, SL, VT, Div);
2017 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Floor, Y);
2018
2019 return DAG.getNode(ISD::FSUB, SL, VT, X, Mul);
2020 }
2021
2022 SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const {
2023 SDLoc SL(Op);
2024 SDValue Src = Op.getOperand(0);
2025
2026 // result = trunc(src)
2027 // if (src > 0.0 && src != result)
2028 // result += 1.0
2029
2030 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2031
2032 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2033 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64);
2034
2035 EVT SetCCVT =
2036 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2037
2038 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOGT);
2039 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2040 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2041
2042 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero);
2043 // TODO: Should this propagate fast-math-flags?
2044 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2045 }
2046
2047 static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL,
2048 SelectionDAG &DAG) {
2049 const unsigned FractBits = 52;
2050 const unsigned ExpBits = 11;
2051
2052 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32,
2053 Hi,
2054 DAG.getConstant(FractBits - 32, SL, MVT::i32),
2055 DAG.getConstant(ExpBits, SL, MVT::i32));
2056 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart,
2057 DAG.getConstant(1023, SL, MVT::i32));
2058
2059 return Exp;
2060 }
2061
2062 SDValue AMDGPUTargetLowering::LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const {
2063 SDLoc SL(Op);
2064 SDValue Src = Op.getOperand(0);
2065
2066 assert(Op.getValueType() == MVT::f64);
2067
2068 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2069 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2070
2071 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2072
2073 // Extract the upper half, since this is where we will find the sign and
2074 // exponent.
2075 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One);
2076
2077 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2078
2079 const unsigned FractBits = 52;
2080
2081 // Extract the sign bit.
2082 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32);
2083 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask);
2084
2085 // Extend back to 64-bits.
2086 SDValue SignBit64 = DAG.getBuildVector(MVT::v2i32, SL, {Zero, SignBit});
2087 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64);
2088
2089 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src);
2090 const SDValue FractMask
2091 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
2092
2093 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp);
2094 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64);
2095 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not);
2096
2097 EVT SetCCVT =
2098 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
2099
2100 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32);
2101
2102 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2103 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2104
2105 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0);
2106 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1);
2107
2108 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2);
2109 }
2110
2111 SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const {
2112 SDLoc SL(Op);
2113 SDValue Src = Op.getOperand(0);
2114
2115 assert(Op.getValueType() == MVT::f64);
2116
2117 APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
2118 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64);
2119 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src);
2120
2121 // TODO: Should this propagate fast-math-flags?
2122
2123 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign);
2124 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign);
2125
2126 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src);
2127
2128 APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
2129 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64);
2130
2131 EVT SetCCVT =
2132 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2133 SDValue Cond = DAG.getSetCC(SL, SetCCVT, Fabs, C2, ISD::SETOGT);
2134
2135 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2);
2136 }
2137
2138 SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const {
2139 // FNEARBYINT and FRINT are the same, except in their handling of FP
2140 // exceptions. Those aren't really meaningful for us, and OpenCL only has
2141 // rint, so just treat them as equivalent.
2142 return DAG.getNode(ISD::FRINT, SDLoc(Op), Op.getValueType(), Op.getOperand(0));
2143 }
2144
2145 // XXX - May require not supporting f32 denormals?
2146
2147 // Don't handle v2f16. The extra instructions to scalarize and repack around the
2148 // compare and vselect end up producing worse code than scalarizing the whole
2149 // operation.
2150 SDValue AMDGPUTargetLowering::LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const {
2151 SDLoc SL(Op);
2152 SDValue X = Op.getOperand(0);
2153 EVT VT = Op.getValueType();
2154
2155 SDValue T = DAG.getNode(ISD::FTRUNC, SL, VT, X);
2156
2157 // TODO: Should this propagate fast-math-flags?
2158
2159 SDValue Diff = DAG.getNode(ISD::FSUB, SL, VT, X, T);
2160
2161 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, VT, Diff);
2162
2163 const SDValue Zero = DAG.getConstantFP(0.0, SL, VT);
2164 const SDValue One = DAG.getConstantFP(1.0, SL, VT);
2165 const SDValue Half = DAG.getConstantFP(0.5, SL, VT);
2166
2167 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, VT, One, X);
2168
2169 EVT SetCCVT =
2170 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2171
2172 SDValue Cmp = DAG.getSetCC(SL, SetCCVT, AbsDiff, Half, ISD::SETOGE);
2173
2174 SDValue Sel = DAG.getNode(ISD::SELECT, SL, VT, Cmp, SignOne, Zero);
2175
2176 return DAG.getNode(ISD::FADD, SL, VT, T, Sel);
2177 }
2178
2179 SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const {
2180 SDLoc SL(Op);
2181 SDValue X = Op.getOperand(0);
2182
2183 SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X);
2184
2185 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2186 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2187 const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32);
2188 const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32);
2189 EVT SetCCVT =
2190 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32);
2191
2192 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X);
2193
2194 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One);
2195
2196 SDValue Exp = extractF64Exponent(Hi, SL, DAG);
2197
2198 const SDValue Mask = DAG.getConstant(INT64_C(0x000fffffffffffff), SL,
2199 MVT::i64);
2200
2201 SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp);
2202 SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64,
2203 DAG.getConstant(INT64_C(0x0008000000000000), SL,
2204 MVT::i64),
2205 Exp);
2206
2207 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M);
2208 SDValue Tmp1 = DAG.getSetCC(SL, SetCCVT,
2209 DAG.getConstant(0, SL, MVT::i64), Tmp0,
2210 ISD::SETNE);
2211
2212 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1,
2213 D, DAG.getConstant(0, SL, MVT::i64));
2214 SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2);
2215
2216 K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64));
2217 K = DAG.getNode(ISD::BITCAST, SL, MVT::f64, K);
2218
2219 SDValue ExpLt0 = DAG.getSetCC(SL, SetCCVT, Exp, Zero, ISD::SETLT);
2220 SDValue ExpGt51 = DAG.getSetCC(SL, SetCCVT, Exp, FiftyOne, ISD::SETGT);
2221 SDValue ExpEqNegOne = DAG.getSetCC(SL, SetCCVT, NegOne, Exp, ISD::SETEQ);
2222
2223 SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64,
2224 ExpEqNegOne,
2225 DAG.getConstantFP(1.0, SL, MVT::f64),
2226 DAG.getConstantFP(0.0, SL, MVT::f64));
2227
2228 SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X);
2229
2230 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpLt0, S, K);
2231 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpGt51, X, K);
2232
2233 return K;
2234 }
2235
2236 SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const {
2237 EVT VT = Op.getValueType();
2238
2239 if (VT == MVT::f32 || VT == MVT::f16)
2240 return LowerFROUND32_16(Op, DAG);
2241
2242 if (VT == MVT::f64)
2243 return LowerFROUND64(Op, DAG);
2244
2245 llvm_unreachable("unhandled type");
2246 }
2247
2248 SDValue AMDGPUTargetLowering::LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const {
2249 SDLoc SL(Op);
2250 SDValue Src = Op.getOperand(0);
2251
2252 // result = trunc(src);
2253 // if (src < 0.0 && src != result)
2254 // result += -1.0.
2255
2256 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2257
2258 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64);
2259 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64);
2260
2261 EVT SetCCVT =
2262 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64);
2263
2264 SDValue Lt0 = DAG.getSetCC(SL, SetCCVT, Src, Zero, ISD::SETOLT);
2265 SDValue NeTrunc = DAG.getSetCC(SL, SetCCVT, Src, Trunc, ISD::SETONE);
2266 SDValue And = DAG.getNode(ISD::AND, SL, SetCCVT, Lt0, NeTrunc);
2267
2268 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero);
2269 // TODO: Should this propagate fast-math-flags?
2270 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add);
2271 }
2272
2273 SDValue AMDGPUTargetLowering::LowerFLOG(SDValue Op, SelectionDAG &DAG,
2274 double Log2BaseInverted) const {
2275 EVT VT = Op.getValueType();
2276
2277 SDLoc SL(Op);
2278 SDValue Operand = Op.getOperand(0);
2279 SDValue Log2Operand = DAG.getNode(ISD::FLOG2, SL, VT, Operand);
2280 SDValue Log2BaseInvertedOperand = DAG.getConstantFP(Log2BaseInverted, SL, VT);
2281
2282 return DAG.getNode(ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand);
2283 }
2284
2285 // exp2(M_LOG2E_F * f);
2286 SDValue AMDGPUTargetLowering::lowerFEXP(SDValue Op, SelectionDAG &DAG) const {
2287 EVT VT = Op.getValueType();
2288 SDLoc SL(Op);
2289 SDValue Src = Op.getOperand(0);
2290
2291 const SDValue K = DAG.getConstantFP(numbers::log2e, SL, VT);
2292 SDValue Mul = DAG.getNode(ISD::FMUL, SL, VT, Src, K, Op->getFlags());
2293 return DAG.getNode(ISD::FEXP2, SL, VT, Mul, Op->getFlags());
2294 }
2295
2296 static bool isCtlzOpc(unsigned Opc) {
2297 return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF;
2298 }
2299
2300 static bool isCttzOpc(unsigned Opc) {
2301 return Opc == ISD::CTTZ || Opc == ISD::CTTZ_ZERO_UNDEF;
2302 }
2303
2304 SDValue AMDGPUTargetLowering::LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const {
2305 SDLoc SL(Op);
2306 SDValue Src = Op.getOperand(0);
2307 bool ZeroUndef = Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
2308 Op.getOpcode() == ISD::CTLZ_ZERO_UNDEF;
2309
2310 unsigned ISDOpc, NewOpc;
2311 if (isCtlzOpc(Op.getOpcode())) {
2312 ISDOpc = ISD::CTLZ_ZERO_UNDEF;
2313 NewOpc = AMDGPUISD::FFBH_U32;
2314 } else if (isCttzOpc(Op.getOpcode())) {
2315 ISDOpc = ISD::CTTZ_ZERO_UNDEF;
2316 NewOpc = AMDGPUISD::FFBL_B32;
2317 } else
2318 llvm_unreachable("Unexpected OPCode!!!");
2319
2320
2321 if (ZeroUndef && Src.getValueType() == MVT::i32)
2322 return DAG.getNode(NewOpc, SL, MVT::i32, Src);
2323
2324 SDValue Vec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2325
2326 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
2327 const SDValue One = DAG.getConstant(1, SL, MVT::i32);
2328
2329 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, Zero);
2330 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, Vec, One);
2331
2332 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
2333 *DAG.getContext(), MVT::i32);
2334
2335 SDValue HiOrLo = isCtlzOpc(Op.getOpcode()) ? Hi : Lo;
2336 SDValue Hi0orLo0 = DAG.getSetCC(SL, SetCCVT, HiOrLo, Zero, ISD::SETEQ);
2337
2338 SDValue OprLo = DAG.getNode(ISDOpc, SL, MVT::i32, Lo);
2339 SDValue OprHi = DAG.getNode(ISDOpc, SL, MVT::i32, Hi);
2340
2341 const SDValue Bits32 = DAG.getConstant(32, SL, MVT::i32);
2342 SDValue Add, NewOpr;
2343 if (isCtlzOpc(Op.getOpcode())) {
2344 Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprLo, Bits32);
2345 // ctlz(x) = hi_32(x) == 0 ? ctlz(lo_32(x)) + 32 : ctlz(hi_32(x))
2346 NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprHi);
2347 } else {
2348 Add = DAG.getNode(ISD::ADD, SL, MVT::i32, OprHi, Bits32);
2349 // cttz(x) = lo_32(x) == 0 ? cttz(hi_32(x)) + 32 : cttz(lo_32(x))
2350 NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32, Hi0orLo0, Add, OprLo);
2351 }
2352
2353 if (!ZeroUndef) {
2354 // Test if the full 64-bit input is zero.
2355
2356 // FIXME: DAG combines turn what should be an s_and_b64 into a v_or_b32,
2357 // which we probably don't want.
2358 SDValue LoOrHi = isCtlzOpc(Op.getOpcode()) ? Lo : Hi;
2359 SDValue Lo0OrHi0 = DAG.getSetCC(SL, SetCCVT, LoOrHi, Zero, ISD::SETEQ);
2360 SDValue SrcIsZero = DAG.getNode(ISD::AND, SL, SetCCVT, Lo0OrHi0, Hi0orLo0);
2361
2362 // TODO: If i64 setcc is half rate, it can result in 1 fewer instruction
2363 // with the same cycles, otherwise it is slower.
2364 // SDValue SrcIsZero = DAG.getSetCC(SL, SetCCVT, Src,
2365 // DAG.getConstant(0, SL, MVT::i64), ISD::SETEQ);
2366
2367 const SDValue Bits32 = DAG.getConstant(64, SL, MVT::i32);
2368
2369 // The instruction returns -1 for 0 input, but the defined intrinsic
2370 // behavior is to return the number of bits.
2371 NewOpr = DAG.getNode(ISD::SELECT, SL, MVT::i32,
2372 SrcIsZero, Bits32, NewOpr);
2373 }
2374
2375 return DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i64, NewOpr);
2376 }
2377
2378 SDValue AMDGPUTargetLowering::LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG,
2379 bool Signed) const {
2380 // Unsigned
2381 // cul2f(ulong u)
2382 //{
2383 // uint lz = clz(u);
2384 // uint e = (u != 0) ? 127U + 63U - lz : 0;
2385 // u = (u << lz) & 0x7fffffffffffffffUL;
2386 // ulong t = u & 0xffffffffffUL;
2387 // uint v = (e << 23) | (uint)(u >> 40);
2388 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
2389 // return as_float(v + r);
2390 //}
2391 // Signed
2392 // cl2f(long l)
2393 //{
2394 // long s = l >> 63;
2395 // float r = cul2f((l + s) ^ s);
2396 // return s ? -r : r;
2397 //}
2398
2399 SDLoc SL(Op);
2400 SDValue Src = Op.getOperand(0);
2401 SDValue L = Src;
2402
2403 SDValue S;
2404 if (Signed) {
2405 const SDValue SignBit = DAG.getConstant(63, SL, MVT::i64);
2406 S = DAG.getNode(ISD::SRA, SL, MVT::i64, L, SignBit);
2407
2408 SDValue LPlusS = DAG.getNode(ISD::ADD, SL, MVT::i64, L, S);
2409 L = DAG.getNode(ISD::XOR, SL, MVT::i64, LPlusS, S);
2410 }
2411
2412 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(),
2413 *DAG.getContext(), MVT::f32);
2414
2415
2416 SDValue ZeroI32 = DAG.getConstant(0, SL, MVT::i32);
2417 SDValue ZeroI64 = DAG.getConstant(0, SL, MVT::i64);
2418 SDValue LZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SL, MVT::i64, L);
2419 LZ = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LZ);
2420
2421 SDValue K = DAG.getConstant(127U + 63U, SL, MVT::i32);
2422 SDValue E = DAG.getSelect(SL, MVT::i32,
2423 DAG.getSetCC(SL, SetCCVT, L, ZeroI64, ISD::SETNE),
2424 DAG.getNode(ISD::SUB, SL, MVT::i32, K, LZ),
2425 ZeroI32);
2426
2427 SDValue U = DAG.getNode(ISD::AND, SL, MVT::i64,
2428 DAG.getNode(ISD::SHL, SL, MVT::i64, L, LZ),
2429 DAG.getConstant((-1ULL) >> 1, SL, MVT::i64));
2430
2431 SDValue T = DAG.getNode(ISD::AND, SL, MVT::i64, U,
2432 DAG.getConstant(0xffffffffffULL, SL, MVT::i64));
2433
2434 SDValue UShl = DAG.getNode(ISD::SRL, SL, MVT::i64,
2435 U, DAG.getConstant(40, SL, MVT::i64));
2436
2437 SDValue V = DAG.getNode(ISD::OR, SL, MVT::i32,
2438 DAG.getNode(ISD::SHL, SL, MVT::i32, E, DAG.getConstant(23, SL, MVT::i32)),
2439 DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, UShl));
2440
2441 SDValue C = DAG.getConstant(0x8000000000ULL, SL, MVT::i64);
2442 SDValue RCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETUGT);
2443 SDValue TCmp = DAG.getSetCC(SL, SetCCVT, T, C, ISD::SETEQ);
2444
2445 SDValue One = DAG.getConstant(1, SL, MVT::i32);
2446
2447 SDValue VTrunc1 = DAG.getNode(ISD::AND, SL, MVT::i32, V, One);
2448
2449 SDValue R = DAG.getSelect(SL, MVT::i32,
2450 RCmp,
2451 One,
2452 DAG.getSelect(SL, MVT::i32, TCmp, VTrunc1, ZeroI32));
2453 R = DAG.getNode(ISD::ADD, SL, MVT::i32, V, R);
2454 R = DAG.getNode(ISD::BITCAST, SL, MVT::f32, R);
2455
2456 if (!Signed)
2457 return R;
2458
2459 SDValue RNeg = DAG.getNode(ISD::FNEG, SL, MVT::f32, R);
2460 return DAG.getSelect(SL, MVT::f32, DAG.getSExtOrTrunc(S, SL, SetCCVT), RNeg, R);
2461 }
2462
2463 SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG,
2464 bool Signed) const {
2465 SDLoc SL(Op);
2466 SDValue Src = Op.getOperand(0);
2467
2468 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src);
2469
2470 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
2471 DAG.getConstant(0, SL, MVT::i32));
2472 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC,
2473 DAG.getConstant(1, SL, MVT::i32));
2474
2475 SDValue CvtHi = DAG.getNode(Signed ? ISD::SINT_TO_FP : ISD::UINT_TO_FP,
2476 SL, MVT::f64, Hi);
2477
2478 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo);
2479
2480 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi,
2481 DAG.getConstant(32, SL, MVT::i32));
2482 // TODO: Should this propagate fast-math-flags?
2483 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo);
2484 }
2485
2486 SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op,
2487 SelectionDAG &DAG) const {
2488 assert(Op.getOperand(0).getValueType() == MVT::i64 &&
2489 "operation should be legal");
2490
2491 // TODO: Factor out code common with LowerSINT_TO_FP.
2492
2493 EVT DestVT = Op.getValueType();
2494 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
2495 SDLoc DL(Op);
2496 SDValue Src = Op.getOperand(0);
2497
2498 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2499 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2500 SDValue FPRound =
2501 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
2502
2503 return FPRound;
2504 }
2505
2506 if (DestVT == MVT::f32)
2507 return LowerINT_TO_FP32(Op, DAG, false);
2508
2509 assert(DestVT == MVT::f64);
2510 return LowerINT_TO_FP64(Op, DAG, false);
2511 }
2512
2513 SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op,
2514 SelectionDAG &DAG) const {
2515 assert(Op.getOperand(0).getValueType() == MVT::i64 &&
2516 "operation should be legal");
2517
2518 // TODO: Factor out code common with LowerUINT_TO_FP.
2519
2520 EVT DestVT = Op.getValueType();
2521 if (Subtarget->has16BitInsts() && DestVT == MVT::f16) {
2522 SDLoc DL(Op);
2523 SDValue Src = Op.getOperand(0);
2524
2525 SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src);
2526 SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op));
2527 SDValue FPRound =
2528 DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag);
2529
2530 return FPRound;
2531 }
2532
2533 if (DestVT == MVT::f32)
2534 return LowerINT_TO_FP32(Op, DAG, true);
2535
2536 assert(DestVT == MVT::f64);
2537 return LowerINT_TO_FP64(Op, DAG, true);
2538 }
2539
2540 SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG,
2541 bool Signed) const {
2542 SDLoc SL(Op);
2543
2544 SDValue Src = Op.getOperand(0);
2545
2546 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src);
2547
2548 SDValue K0 = DAG.getConstantFP(BitsToDouble(UINT64_C(0x3df0000000000000)), SL,
2549 MVT::f64);
2550 SDValue K1 = DAG.getConstantFP(BitsToDouble(UINT64_C(0xc1f0000000000000)), SL,
2551 MVT::f64);
2552 // TODO: Should this propagate fast-math-flags?
2553 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0);
2554
2555 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul);
2556
2557
2558 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc);
2559
2560 SDValue Hi = DAG.getNode(Signed ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, SL,
2561 MVT::i32, FloorMul);
2562 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma);
2563
2564 SDValue Result = DAG.getBuildVector(MVT::v2i32, SL, {Lo, Hi});
2565
2566 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result);
2567 }
2568
2569 SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const {
2570 SDLoc DL(Op);
2571 SDValue N0 = Op.getOperand(0);
2572
2573 // Convert to target node to get known bits
2574 if (N0.getValueType() == MVT::f32)
2575 return DAG.getNode(AMDGPUISD::FP_TO_FP16, DL, Op.getValueType(), N0);
2576
2577 if (getTargetMachine().Options.UnsafeFPMath) {
2578 // There is a generic expand for FP_TO_FP16 with unsafe fast math.
2579 return SDValue();
2580 }
2581
2582 assert(N0.getSimpleValueType() == MVT::f64);
2583
2584 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
2585 const unsigned ExpMask = 0x7ff;
2586 const unsigned ExpBiasf64 = 1023;
2587 const unsigned ExpBiasf16 = 15;
2588 SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
2589 SDValue One = DAG.getConstant(1, DL, MVT::i32);
2590 SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
2591 SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
2592 DAG.getConstant(32, DL, MVT::i64));
2593 UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
2594 U = DAG.getZExtOrTrunc(U, DL, MVT::i32);
2595 SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2596 DAG.getConstant(20, DL, MVT::i64));
2597 E = DAG.getNode(ISD::AND, DL, MVT::i32, E,
2598 DAG.getConstant(ExpMask, DL, MVT::i32));
2599 // Subtract the fp64 exponent bias (1023) to get the real exponent and
2600 // add the f16 bias (15) to get the biased exponent for the f16 format.
2601 E = DAG.getNode(ISD::ADD, DL, MVT::i32, E,
2602 DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32));
2603
2604 SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2605 DAG.getConstant(8, DL, MVT::i32));
2606 M = DAG.getNode(ISD::AND, DL, MVT::i32, M,
2607 DAG.getConstant(0xffe, DL, MVT::i32));
2608
2609 SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH,
2610 DAG.getConstant(0x1ff, DL, MVT::i32));
2611 MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U);
2612
2613 SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ);
2614 M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set);
2615
2616 // (M != 0 ? 0x0200 : 0) | 0x7c00;
2617 SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32,
2618 DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32),
2619 Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32));
2620
2621 // N = M | (E << 12);
2622 SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2623 DAG.getNode(ISD::SHL, DL, MVT::i32, E,
2624 DAG.getConstant(12, DL, MVT::i32)));
2625
2626 // B = clamp(1-E, 0, 13);
2627 SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32,
2628 One, E);
2629 SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero);
2630 B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B,
2631 DAG.getConstant(13, DL, MVT::i32));
2632
2633 SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M,
2634 DAG.getConstant(0x1000, DL, MVT::i32));
2635
2636 SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B);
2637 SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B);
2638 SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE);
2639 D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1);
2640
2641 SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT);
2642 SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V,
2643 DAG.getConstant(0x7, DL, MVT::i32));
2644 V = DAG.getNode(ISD::SRL, DL, MVT::i32, V,
2645 DAG.getConstant(2, DL, MVT::i32));
2646 SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32),
2647 One, Zero, ISD::SETEQ);
2648 SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32),
2649 One, Zero, ISD::SETGT);
2650 V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1);
2651 V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1);
2652
2653 V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32),
2654 DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT);
2655 V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32),
2656 I, V, ISD::SETEQ);
2657
2658 // Extract the sign bit.
2659 SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH,
2660 DAG.getConstant(16, DL, MVT::i32));
2661 Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
2662 DAG.getConstant(0x8000, DL, MVT::i32));
2663
2664 V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
2665 return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
2666 }
2667
2668 SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op,
2669 SelectionDAG &DAG) const {
2670 SDValue Src = Op.getOperand(0);
2671
2672 // TODO: Factor out code common with LowerFP_TO_UINT.
2673
2674 EVT SrcVT = Src.getValueType();
2675 if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
2676 SDLoc DL(Op);
2677
2678 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2679 SDValue FpToInt32 =
2680 DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
2681
2682 return FpToInt32;
2683 }
2684
2685 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
2686 return LowerFP64_TO_INT(Op, DAG, true);
2687
2688 return SDValue();
2689 }
2690
2691 SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op,
2692 SelectionDAG &DAG) const {
2693 SDValue Src = Op.getOperand(0);
2694
2695 // TODO: Factor out code common with LowerFP_TO_SINT.
2696
2697 EVT SrcVT = Src.getValueType();
2698 if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) {
2699 SDLoc DL(Op);
2700
2701 SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
2702 SDValue FpToInt32 =
2703 DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend);
2704
2705 return FpToInt32;
2706 }
2707
2708 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64)
2709 return LowerFP64_TO_INT(Op, DAG, false);
2710
2711 return SDValue();
2712 }
2713
2714 SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2715 SelectionDAG &DAG) const {
2716 EVT ExtraVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2717 MVT VT = Op.getSimpleValueType();
2718 MVT ScalarVT = VT.getScalarType();
2719
2720 assert(VT.isVector());
2721
2722 SDValue Src = Op.getOperand(0);
2723 SDLoc DL(Op);
2724
2725 // TODO: Don't scalarize on Evergreen?
2726 unsigned NElts = VT.getVectorNumElements();
2727 SmallVector<SDValue, 8> Args;
2728 DAG.ExtractVectorElements(Src, Args, 0, NElts);
2729
2730 SDValue VTOp = DAG.getValueType(ExtraVT.getScalarType());
2731 for (unsigned I = 0; I < NElts; ++I)
2732 Args[I] = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, ScalarVT, Args[I], VTOp);
2733
2734 return DAG.getBuildVector(VT, DL, Args);
2735 }
2736
2737 //===----------------------------------------------------------------------===//
2738 // Custom DAG optimizations
2739 //===----------------------------------------------------------------------===//
2740
2741 static bool isU24(SDValue Op, SelectionDAG &DAG) {
2742 return AMDGPUTargetLowering::numBitsUnsigned(Op, DAG) <= 24;
2743 }
2744
2745 static bool isI24(SDValue Op, SelectionDAG &DAG) {
2746 EVT VT = Op.getValueType();
2747 return VT.getSizeInBits() >= 24 && // Types less than 24-bit should be treated
2748 // as unsigned 24-bit values.
2749 AMDGPUTargetLowering::numBitsSigned(Op, DAG) < 24;
2750 }
2751
2752 static SDValue simplifyI24(SDNode *Node24,
2753 TargetLowering::DAGCombinerInfo &DCI) {
2754 SelectionDAG &DAG = DCI.DAG;
2755 bool IsIntrin = Node24->getOpcode() == ISD::INTRINSIC_WO_CHAIN;
2756
2757 SDValue LHS = IsIntrin ? Node24->getOperand(1) : Node24->getOperand(0);
2758 SDValue RHS = IsIntrin ? Node24->getOperand(2) : Node24->getOperand(1);
2759 unsigned NewOpcode = Node24->getOpcode();
2760 if (IsIntrin) {
2761 unsigned IID = cast<ConstantSDNode>(Node24->getOperand(0))->getZExtValue();
2762 NewOpcode = IID == Intrinsic::amdgcn_mul_i24 ?
2763 AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
2764 }
2765
2766 APInt Demanded = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 24);
2767
2768 // First try to simplify using GetDemandedBits which allows the operands to
2769 // have other uses, but will only perform simplifications that involve
2770 // bypassing some nodes for this user.
2771 SDValue DemandedLHS = DAG.GetDemandedBits(LHS, Demanded);
2772 SDValue DemandedRHS = DAG.GetDemandedBits(RHS, Demanded);
2773 if (DemandedLHS || DemandedRHS)
2774 return DAG.getNode(NewOpcode, SDLoc(Node24), Node24->getVTList(),
2775 DemandedLHS ? DemandedLHS : LHS,
2776 DemandedRHS ? DemandedRHS : RHS);
2777
2778 // Now try SimplifyDemandedBits which can simplify the nodes used by our
2779 // operands if this node is the only user.
2780 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2781 if (TLI.SimplifyDemandedBits(LHS, Demanded, DCI))
2782 return SDValue(Node24, 0);
2783 if (TLI.SimplifyDemandedBits(RHS, Demanded, DCI))
2784 return SDValue(Node24, 0);
2785
2786 return SDValue();
2787 }
2788
2789 template <typename IntTy>
2790 static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset,
2791 uint32_t Width, const SDLoc &DL) {
2792 if (Width + Offset < 32) {
2793 uint32_t Shl = static_cast<uint32_t>(Src0) << (32 - Offset - Width);
2794 IntTy Result = static_cast<IntTy>(Shl) >> (32 - Width);
2795 return DAG.getConstant(Result, DL, MVT::i32);
2796 }
2797
2798 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32);
2799 }
2800
2801 static bool hasVolatileUser(SDNode *Val) {
2802 for (SDNode *U : Val->uses()) {
2803 if (MemSDNode *M = dyn_cast<MemSDNode>(U)) {
2804 if (M->isVolatile())
2805 return true;
2806 }
2807 }
2808
2809 return false;
2810 }
2811
2812 bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
2813 // i32 vectors are the canonical memory type.
2814 if (VT.getScalarType() == MVT::i32 || isTypeLegal(VT))
2815 return false;
2816
2817 if (!VT.isByteSized())
2818 return false;
2819
2820 unsigned Size = VT.getStoreSize();
2821
2822 if ((Size == 1 || Size == 2 || Size == 4) && !VT.isVector())
2823 return false;
2824
2825 if (Size == 3 || (Size > 4 && (Size % 4 != 0)))
2826 return false;
2827
2828 return true;
2829 }
2830
2831 // Find a load or store from corresponding pattern root.
2832 // Roots may be build_vector, bitconvert or their combinations.
2833 static MemSDNode* findMemSDNode(SDNode *N) {
2834 N = AMDGPUTargetLowering::stripBitcast(SDValue(N,0)).getNode();
2835 if (MemSDNode *MN = dyn_cast<MemSDNode>(N))
2836 return MN;
2837 assert(isa<BuildVectorSDNode>(N));
2838 for (SDValue V : N->op_values())
2839 if (MemSDNode *MN =
2840 dyn_cast<MemSDNode>(AMDGPUTargetLowering::stripBitcast(V)))
2841 return MN;
2842 llvm_unreachable("cannot find MemSDNode in the pattern!");
2843 }
2844
2845 bool AMDGPUTargetLowering::SelectFlatOffset(bool IsSigned,
2846 SelectionDAG &DAG,
2847 SDNode *N,
2848 SDValue Addr,
2849 SDValue &VAddr,
2850 SDValue &Offset,
2851 SDValue &SLC) const {
2852 const GCNSubtarget &ST =
2853 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
2854 int64_t OffsetVal = 0;
2855
2856 if (ST.hasFlatInstOffsets() &&
2857 (!ST.hasFlatSegmentOffsetBug() ||
2858 findMemSDNode(N)->getAddressSpace() != AMDGPUAS::FLAT_ADDRESS) &&
2859 DAG.isBaseWithConstantOffset(Addr)) {
2860 SDValue N0 = Addr.getOperand(0);
2861 SDValue N1 = Addr.getOperand(1);
2862 int64_t COffsetVal = cast<ConstantSDNode>(N1)->getSExtValue();
2863
2864 const SIInstrInfo *TII = ST.getInstrInfo();
2865 if (TII->isLegalFLATOffset(COffsetVal, findMemSDNode(N)->getAddressSpace(),
2866 IsSigned)) {
2867 Addr = N0;
2868 OffsetVal = COffsetVal;
2869 }
2870 }
2871
2872 VAddr = Addr;
2873 Offset = DAG.getTargetConstant(OffsetVal, SDLoc(), MVT::i16);
2874 SLC = DAG.getTargetConstant(0, SDLoc(), MVT::i1);
2875
2876 return true;
2877 }
2878
2879 // Replace load of an illegal type with a store of a bitcast to a friendlier
2880 // type.
2881 SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
2882 DAGCombinerInfo &DCI) const {
2883 if (!DCI.isBeforeLegalize())
2884 return SDValue();
2885
2886 LoadSDNode *LN = cast<LoadSDNode>(N);
2887 if (LN->isVolatile() || !ISD::isNormalLoad(LN) || hasVolatileUser(LN))
2888 return SDValue();
2889
2890 SDLoc SL(N);
2891 SelectionDAG &DAG = DCI.DAG;
2892 EVT VT = LN->getMemoryVT();
2893
2894 unsigned Size = VT.getStoreSize();
2895 unsigned Align = LN->getAlignment();
2896 if (Align < Size && isTypeLegal(VT)) {
2897 bool IsFast;
2898 unsigned AS = LN->getAddressSpace();
2899
2900 // Expand unaligned loads earlier than legalization. Due to visitation order
2901 // problems during legalization, the emitted instructions to pack and unpack
2902 // the bytes again are not eliminated in the case of an unaligned copy.
2903 if (!allowsMisalignedMemoryAccesses(
2904 VT, AS, Align, LN->getMemOperand()->getFlags(), &IsFast)) {
2905 if (VT.isVector())
2906 return scalarizeVectorLoad(LN, DAG);
2907
2908 SDValue Ops[2];
2909 std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG);
2910 return DAG.getMergeValues(Ops, SDLoc(N));
2911 }
2912
2913 if (!IsFast)
2914 return SDValue();
2915 }
2916
2917 if (!shouldCombineMemoryType(VT))
2918 return SDValue();
2919
2920 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
2921
2922 SDValue NewLoad
2923 = DAG.getLoad(NewVT, SL, LN->getChain(),
2924 LN->getBasePtr(), LN->getMemOperand());
2925
2926 SDValue BC = DAG.getNode(ISD::BITCAST, SL, VT, NewLoad);
2927 DCI.CombineTo(N, BC, NewLoad.getValue(1));
2928 return SDValue(N, 0);
2929 }
2930
2931 // Replace store of an illegal type with a store of a bitcast to a friendlier
2932 // type.
2933 SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
2934 DAGCombinerInfo &DCI) const {
2935 if (!DCI.isBeforeLegalize())
2936 return SDValue();
2937
2938 StoreSDNode *SN = cast<StoreSDNode>(N);
2939 if (SN->isVolatile() || !ISD::isNormalStore(SN))
2940 return SDValue();
2941
2942 EVT VT = SN->getMemoryVT();
2943 unsigned Size = VT.getStoreSize();
2944
2945 SDLoc SL(N);
2946 SelectionDAG &DAG = DCI.DAG;
2947 unsigned Align = SN->getAlignment();
2948 if (Align < Size && isTypeLegal(VT)) {
2949 bool IsFast;
2950 unsigned AS = SN->getAddressSpace();
2951
2952 // Expand unaligned stores earlier than legalization. Due to visitation
2953 // order problems during legalization, the emitted instructions to pack and
2954 // unpack the bytes again are not eliminated in the case of an unaligned
2955 // copy.
2956 if (!allowsMisalignedMemoryAccesses(
2957 VT, AS, Align, SN->getMemOperand()->getFlags(), &IsFast)) {
2958 if (VT.isVector())
2959 return scalarizeVectorStore(SN, DAG);
2960
2961 return expandUnalignedStore(SN, DAG);
2962 }
2963
2964 if (!IsFast)
2965 return SDValue();
2966 }
2967
2968 if (!shouldCombineMemoryType(VT))
2969 return SDValue();
2970
2971 EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
2972 SDValue Val = SN->getValue();
2973
2974 //DCI.AddToWorklist(Val.getNode());
2975
2976 bool OtherUses = !Val.hasOneUse();
2977 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NewVT, Val);
2978 if (OtherUses) {
2979 SDValue CastBack = DAG.getNode(ISD::BITCAST, SL, VT, CastVal);
2980 DAG.ReplaceAllUsesOfValueWith(Val, CastBack);
2981 }
2982
2983 return DAG.getStore(SN->getChain(), SL, CastVal,
2984 SN->getBasePtr(), SN->getMemOperand());
2985 }
2986
2987 // FIXME: This should go in generic DAG combiner with an isTruncateFree check,
2988 // but isTruncateFree is inaccurate for i16 now because of SALU vs. VALU
2989 // issues.
2990 SDValue AMDGPUTargetLowering::performAssertSZExtCombine(SDNode *N,
2991 DAGCombinerInfo &DCI) const {
2992 SelectionDAG &DAG = DCI.DAG;
2993 SDValue N0 = N->getOperand(0);
2994
2995 // (vt2 (assertzext (truncate vt0:x), vt1)) ->
2996 // (vt2 (truncate (assertzext vt0:x, vt1)))
2997 if (N0.getOpcode() == ISD::TRUNCATE) {
2998 SDValue N1 = N->getOperand(1);
2999 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
3000 SDLoc SL(N);
3001
3002 SDValue Src = N0.getOperand(0);
3003 EVT SrcVT = Src.getValueType();
3004 if (SrcVT.bitsGE(ExtVT)) {
3005 SDValue NewInReg = DAG.getNode(N->getOpcode(), SL, SrcVT, Src, N1);
3006 return DAG.getNode(ISD::TRUNCATE, SL, N->getValueType(0), NewInReg);
3007 }
3008 }
3009
3010 return SDValue();
3011 }
3012
3013 SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
3014 SDNode *N, DAGCombinerInfo &DCI) const {
3015 unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
3016 switch (IID) {
3017 case Intrinsic::amdgcn_mul_i24:
3018 case Intrinsic::amdgcn_mul_u24:
3019 return simplifyI24(N, DCI);
3020 default:
3021 return SDValue();
3022 }
3023 }
3024
3025 /// Split the 64-bit value \p LHS into two 32-bit components, and perform the
3026 /// binary operation \p Opc to it with the corresponding constant operands.
3027 SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
3028 DAGCombinerInfo &DCI, const SDLoc &SL,
3029 unsigned Opc, SDValue LHS,
3030 uint32_t ValLo, uint32_t ValHi) const {
3031 SelectionDAG &DAG = DCI.DAG;
3032 SDValue Lo, Hi;
3033 std::tie(Lo, Hi) = split64BitValue(LHS, DAG);
3034
3035 SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32);
3036 SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32);
3037
3038 SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS);
3039 SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS);
3040
3041 // Re-visit the ands. It's possible we eliminated one of them and it could
3042 // simplify the vector.
3043 DCI.AddToWorklist(Lo.getNode());
3044 DCI.AddToWorklist(Hi.getNode());
3045
3046 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {LoAnd, HiAnd});
3047 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
3048 }
3049
3050 SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
3051 DAGCombinerInfo &DCI) const {
3052 EVT VT = N->getValueType(0);
3053
3054 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
3055 if (!RHS)
3056 return SDValue();
3057
3058 SDValue LHS = N->getOperand(0);
3059 unsigned RHSVal = RHS->getZExtValue();
3060 if (!RHSVal)
3061 return LHS;
3062
3063 SDLoc SL(N);
3064 SelectionDAG &DAG = DCI.DAG;
3065
3066 switch (LHS->getOpcode()) {
3067 default:
3068 break;
3069 case ISD::ZERO_EXTEND:
3070 case ISD::SIGN_EXTEND:
3071 case ISD::ANY_EXTEND: {
3072 SDValue X = LHS->getOperand(0);
3073
3074 if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
3075 isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) {
3076 // Prefer build_vector as the canonical form if packed types are legal.
3077 // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
3078 SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
3079 { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) });
3080 return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
3081 }
3082
3083 // shl (ext x) => zext (shl x), if shift does not overflow int
3084 if (VT != MVT::i64)
3085 break;
3086 KnownBits Known = DAG.computeKnownBits(X);
3087 unsigned LZ = Known.countMinLeadingZeros();
3088 if (LZ < RHSVal)
3089 break;
3090 EVT XVT = X.getValueType();
3091 SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
3092 return DAG.getZExtOrTrunc(Shl, SL, VT);
3093 }
3094 }
3095
3096 if (VT != MVT::i64)
3097 return SDValue();
3098
3099 // i64 (shl x, C) -> (build_pair 0, (shl x, C -32))
3100
3101 // On some subtargets, 64-bit shift is a quarter rate instruction. In the
3102 // common case, splitting this into a move and a 32-bit shift is faster and
3103 // the same code size.
3104 if (RHSVal < 32)
3105 return SDValue();
3106
3107 SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
3108
3109 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
3110 SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
3111
3112 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
3113
3114 SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
3115 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
3116 }
3117
3118 SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
3119 DAGCombinerInfo &DCI) const {
3120 if (N->getValueType(0) != MVT::i64)
3121 return SDValue();
3122
3123 const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
3124 if (!RHS)
3125 return SDValue();
3126
3127 SelectionDAG &DAG = DCI.DAG;
3128 SDLoc SL(N);
3129 unsigned RHSVal = RHS->getZExtValue();
3130
3131 // (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31)
3132 if (RHSVal == 32) {
3133 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
3134 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
3135 DAG.getConstant(31, SL, MVT::i32));
3136
3137 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
3138 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
3139 }
3140
3141 // (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31)
3142 if (RHSVal == 63) {
3143 SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
3144 SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
3145 DAG.getConstant(31, SL, MVT::i32));
3146 SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
3147 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
3148 }
3149
3150 return SDValue();
3151 }
3152
3153 SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
3154 DAGCombinerInfo &DCI) const {
3155 auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
3156 if (!RHS)
3157 return SDValue();
3158
3159 EVT VT = N->getValueType(0);
3160 SDValue LHS = N->getOperand(0);
3161 unsigned ShiftAmt = RHS->getZExtValue();
3162 SelectionDAG &DAG = DCI.DAG;
3163 SDLoc SL(N);
3164
3165 // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
3166 // this improves the ability to match BFE patterns in isel.
3167 if (LHS.getOpcode() == ISD::AND) {
3168 if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
3169 if (Mask->getAPIntValue().isShiftedMask() &&
3170 Mask->getAPIntValue().countTrailingZeros() == ShiftAmt) {
3171 return DAG.getNode(
3172 ISD::AND, SL, VT,
3173 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
3174 DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
3175 }
3176 }
3177 }
3178
3179 if (VT != MVT::i64)
3180 return SDValue();
3181
3182 if (ShiftAmt < 32)
3183 return SDValue();
3184
3185 // srl i64:x, C for C >= 32
3186 // =>
3187 // build_pair (srl hi_32(x), C - 32), 0
3188 SDValue One = DAG.getConstant(1, SL, MVT::i32);
3189 SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
3190
3191 SDValue VecOp = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, LHS);
3192 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecOp, One);
3193
3194 SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
3195 SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
3196
3197 SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
3198
3199 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
3200 }
3201
3202 SDValue AMDGPUTargetLowering::performTruncateCombine(
3203 SDNode *N, DAGCombinerInfo &DCI) const {
3204 SDLoc SL(N);
3205 SelectionDAG &DAG = DCI.DAG;
3206 EVT VT = N->getValueType(0);
3207 SDValue Src = N->getOperand(0);
3208
3209 // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
3210 if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
3211 SDValue Vec = Src.getOperand(0);
3212 if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
3213 SDValue Elt0 = Vec.getOperand(0);
3214 EVT EltVT = Elt0.getValueType();
3215 if (VT.getSizeInBits() <= EltVT.getSizeInBits()) {
3216 if (EltVT.isFloatingPoint()) {
3217 Elt0 = DAG.getNode(ISD::BITCAST, SL,
3218 EltVT.changeTypeToInteger(), Elt0);
3219 }
3220
3221 return DAG.getNode(ISD::TRUNCATE, SL, VT, Elt0);
3222 }
3223 }
3224 }
3225
3226 // Equivalent of above for accessing the high element of a vector as an
3227 // integer operation.
3228 // trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
3229 if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
3230 if (auto K = isConstOrConstSplat(Src.getOperand(1))) {
3231 if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
3232 SDValue BV = stripBitcast(Src.getOperand(0));
3233 if (BV.getOpcode() == ISD::BUILD_VECTOR &&
3234 BV.getValueType().getVectorNumElements() == 2) {
3235 SDValue SrcElt = BV.getOperand(1);
3236 EVT SrcEltVT = SrcElt.getValueType();
3237 if (SrcEltVT.isFloatingPoint()) {
3238 SrcElt = DAG.getNode(ISD::BITCAST, SL,
3239 SrcEltVT.changeTypeToInteger(), SrcElt);
3240 }
3241
3242 return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
3243 }
3244 }
3245 }
3246 }
3247
3248 // Partially shrink 64-bit shifts to 32-bit if reduced to 16-bit.
3249 //
3250 // i16 (trunc (srl i64:x, K)), K <= 16 ->
3251 // i16 (trunc (srl (i32 (trunc x), K)))
3252 if (VT.getScalarSizeInBits() < 32) {
3253 EVT SrcVT = Src.getValueType();
3254 if (SrcVT.getScalarSizeInBits() > 32 &&
3255 (Src.getOpcode() == ISD::SRL ||
3256 Src.getOpcode() == ISD::SRA ||
3257 Src.getOpcode() == ISD::SHL)) {
3258 SDValue Amt = Src.getOperand(1);
3259 KnownBits Known = DAG.computeKnownBits(Amt);
3260 unsigned Size = VT.getScalarSizeInBits();
3261 if ((Known.isConstant() && Known.getConstant().ule(Size)) ||
3262 (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size))) {
3263 EVT MidVT = VT.isVector() ?
3264 EVT::getVectorVT(*DAG.getContext(), MVT::i32,
3265 VT.getVectorNumElements()) : MVT::i32;
3266
3267 EVT NewShiftVT = getShiftAmountTy(MidVT, DAG.getDataLayout());
3268 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MidVT,
3269 Src.getOperand(0));
3270 DCI.AddToWorklist(Trunc.getNode());
3271
3272 if (Amt.getValueType() != NewShiftVT) {
3273 Amt = DAG.getZExtOrTrunc(Amt, SL, NewShiftVT);
3274 DCI.AddToWorklist(Amt.getNode());
3275 }
3276
3277 SDValue ShrunkShift = DAG.getNode(Src.getOpcode(), SL, MidVT,
3278 Trunc, Amt);
3279 return DAG.getNode(ISD::TRUNCATE, SL, VT, ShrunkShift);
3280 }
3281 }
3282 }
3283
3284 return SDValue();
3285 }
3286
3287 // We need to specifically handle i64 mul here to avoid unnecessary conversion
3288 // instructions. If we only match on the legalized i64 mul expansion,
3289 // SimplifyDemandedBits will be unable to remove them because there will be
3290 // multiple uses due to the separate mul + mulh[su].
3291 static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL,
3292 SDValue N0, SDValue N1, unsigned Size, bool Signed) {
3293 if (Size <= 32) {
3294 unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
3295 return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1);
3296 }
3297
3298 // Because we want to eliminate extension instructions before the
3299 // operation, we need to create a single user here (i.e. not the separate
3300 // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it.
3301
3302 unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24;
3303
3304 SDValue Mul = DAG.getNode(MulOpc, SL,
3305 DAG.getVTList(MVT::i32, MVT::i32), N0, N1);
3306
3307 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64,
3308 Mul.getValue(0), Mul.getValue(1));
3309 }
3310
3311 SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N,
3312 DAGCombinerInfo &DCI) const {
3313 EVT VT = N->getValueType(0);
3314
3315 unsigned Size = VT.getSizeInBits();
3316 if (VT.isVector() || Size > 64)
3317 return SDValue();
3318
3319 // There are i16 integer mul/mad.
3320 if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16))
3321 return SDValue();
3322
3323 SelectionDAG &DAG = DCI.DAG;
3324 SDLoc DL(N);
3325
3326 SDValue N0 = N->getOperand(0);
3327 SDValue N1 = N->getOperand(1);
3328
3329 // SimplifyDemandedBits has the annoying habit of turning useful zero_extends
3330 // in the source into any_extends if the result of the mul is truncated. Since
3331 // we can assume the high bits are whatever we want, use the underlying value
3332 // to avoid the unknown high bits from interfering.
3333 if (N0.getOpcode() == ISD::ANY_EXTEND)
3334 N0 = N0.getOperand(0);
3335
3336 if (N1.getOpcode() == ISD::ANY_EXTEND)
3337 N1 = N1.getOperand(0);
3338
3339 SDValue Mul;
3340
3341 if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) {
3342 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
3343 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
3344 Mul = getMul24(DAG, DL, N0, N1, Size, false);
3345 } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) {
3346 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
3347 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
3348 Mul = getMul24(DAG, DL, N0, N1, Size, true);
3349 } else {
3350 return SDValue();
3351 }
3352
3353 // We need to use sext even for MUL_U24, because MUL_U24 is used
3354 // for signed multiply of 8 and 16-bit types.
3355 return DAG.getSExtOrTrunc(Mul, DL, VT);
3356 }
3357
3358 SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N,
3359 DAGCombinerInfo &DCI) const {
3360 EVT VT = N->getValueType(0);
3361
3362 if (!Subtarget->hasMulI24() || VT.isVector())
3363 return SDValue();
3364
3365 SelectionDAG &DAG = DCI.DAG;
3366 SDLoc DL(N);
3367
3368 SDValue N0 = N->getOperand(0);
3369 SDValue N1 = N->getOperand(1);
3370
3371 if (!isI24(N0, DAG) || !isI24(N1, DAG))
3372 return SDValue();
3373
3374 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32);
3375 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32);
3376
3377 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1);
3378 DCI.AddToWorklist(Mulhi.getNode());
3379 return DAG.getSExtOrTrunc(Mulhi, DL, VT);
3380 }
3381
3382 SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N,
3383 DAGCombinerInfo &DCI) const {
3384 EVT VT = N->getValueType(0);
3385
3386 if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32)
3387 return SDValue();
3388
3389 SelectionDAG &DAG = DCI.DAG;
3390 SDLoc DL(N);
3391
3392 SDValue N0 = N->getOperand(0);
3393 SDValue N1 = N->getOperand(1);
3394
3395 if (!isU24(N0, DAG) || !isU24(N1, DAG))
3396 return SDValue();
3397
3398 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32);
3399 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32);
3400
3401 SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1);
3402 DCI.AddToWorklist(Mulhi.getNode());
3403 return DAG.getZExtOrTrunc(Mulhi, DL, VT);
3404 }
3405
3406 SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
3407 SDNode *N, DAGCombinerInfo &DCI) const {
3408 SelectionDAG &DAG = DCI.DAG;
3409
3410 // Simplify demanded bits before splitting into multiple users.
3411 if (SDValue V = simplifyI24(N, DCI))
3412 return V;
3413
3414 SDValue N0 = N->getOperand(0);
3415 SDValue N1 = N->getOperand(1);
3416
3417 bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24);
3418
3419 unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24;
3420 unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24;
3421
3422 SDLoc SL(N);
3423
3424 SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1);
3425 SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1);
3426 return DAG.getMergeValues({ MulLo, MulHi }, SL);
3427 }
3428
3429 static bool isNegativeOne(SDValue Val) {
3430 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val))
3431 return C->isAllOnesValue();
3432 return false;
3433 }
3434
3435 SDValue AMDGPUTargetLowering::getFFBX_U32(SelectionDAG &DAG,
3436 SDValue Op,
3437 const SDLoc &DL,
3438 unsigned Opc) const {
3439 EVT VT = Op.getValueType();
3440 EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT);
3441 if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() &&
3442 LegalVT != MVT::i16))
3443 return SDValue();
3444
3445 if (VT != MVT::i32)
3446 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op);
3447
3448 SDValue FFBX = DAG.getNode(Opc, DL, MVT::i32, Op);
3449 if (VT != MVT::i32)
3450 FFBX = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBX);
3451
3452 return FFBX;
3453 }
3454
3455 // The native instructions return -1 on 0 input. Optimize out a select that
3456 // produces -1 on 0.
3457 //
3458 // TODO: If zero is not undef, we could also do this if the output is compared
3459 // against the bitwidth.
3460 //
3461 // TODO: Should probably combine against FFBH_U32 instead of ctlz directly.
3462 SDValue AMDGPUTargetLowering::performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond,
3463 SDValue LHS, SDValue RHS,
3464 DAGCombinerInfo &DCI) const {
3465 ConstantSDNode *CmpRhs = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
3466 if (!CmpRhs || !CmpRhs->isNullValue())
3467 return SDValue();
3468
3469 SelectionDAG &DAG = DCI.DAG;
3470 ISD::CondCode CCOpcode = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
3471 SDValue CmpLHS = Cond.getOperand(0);
3472
3473 unsigned Opc = isCttzOpc(RHS.getOpcode()) ? AMDGPUISD::FFBL_B32 :
3474 AMDGPUISD::FFBH_U32;
3475
3476 // select (setcc x, 0, eq), -1, (ctlz_zero_undef x) -> ffbh_u32 x
3477 // select (setcc x, 0, eq), -1, (cttz_zero_undef x) -> ffbl_u32 x
3478 if (CCOpcode == ISD::SETEQ &&
3479 (isCtlzOpc(RHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
3480 RHS.getOperand(0) == CmpLHS &&
3481 isNegativeOne(LHS)) {
3482 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
3483 }
3484
3485 // select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x
3486 // select (setcc x, 0, ne), (cttz_zero_undef x), -1 -> ffbl_u32 x
3487 if (CCOpcode == ISD::SETNE &&
3488 (isCtlzOpc(LHS.getOpcode()) || isCttzOpc(RHS.getOpcode())) &&
3489 LHS.getOperand(0) == CmpLHS &&
3490 isNegativeOne(RHS)) {
3491 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
3492 }
3493
3494 return SDValue();
3495 }
3496
3497 static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI,
3498 unsigned Op,
3499 const SDLoc &SL,
3500 SDValue Cond,
3501 SDValue N1,
3502 SDValue N2) {
3503 SelectionDAG &DAG = DCI.DAG;
3504 EVT VT = N1.getValueType();
3505
3506 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT, Cond,
3507 N1.getOperand(0), N2.getOperand(0));
3508 DCI.AddToWorklist(NewSelect.getNode());
3509 return DAG.getNode(Op, SL, VT, NewSelect);
3510 }
3511
3512 // Pull a free FP operation out of a select so it may fold into uses.
3513 //
3514 // select c, (fneg x), (fneg y) -> fneg (select c, x, y)
3515 // select c, (fneg x), k -> fneg (select c, x, (fneg k))
3516 //
3517 // select c, (fabs x), (fabs y) -> fabs (select c, x, y)
3518 // select c, (fabs x), +k -> fabs (select c, x, k)
3519 static SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI,
3520 SDValue N) {
3521 SelectionDAG &DAG = DCI.DAG;
3522 SDValue Cond = N.getOperand(0);
3523 SDValue LHS = N.getOperand(1);
3524 SDValue RHS = N.getOperand(2);
3525
3526 EVT VT = N.getValueType();
3527 if ((LHS.getOpcode() == ISD::FABS && RHS.getOpcode() == ISD::FABS) ||
3528 (LHS.getOpcode() == ISD::FNEG && RHS.getOpcode() == ISD::FNEG)) {
3529 return distributeOpThroughSelect(DCI, LHS.getOpcode(),
3530 SDLoc(N), Cond, LHS, RHS);
3531 }
3532
3533 bool Inv = false;
3534 if (RHS.getOpcode() == ISD::FABS || RHS.getOpcode() == ISD::FNEG) {
3535 std::swap(LHS, RHS);
3536 Inv = true;
3537 }
3538
3539 // TODO: Support vector constants.
3540 ConstantFPSDNode *CRHS = dyn_cast<ConstantFPSDNode>(RHS);
3541 if ((LHS.getOpcode() == ISD::FNEG || LHS.getOpcode() == ISD::FABS) && CRHS) {
3542 SDLoc SL(N);
3543 // If one side is an fneg/fabs and the other is a constant, we can push the
3544 // fneg/fabs down. If it's an fabs, the constant needs to be non-negative.
3545 SDValue NewLHS = LHS.getOperand(0);
3546 SDValue NewRHS = RHS;
3547
3548 // Careful: if the neg can be folded up, don't try to pull it back down.
3549 bool ShouldFoldNeg = true;
3550
3551 if (NewLHS.hasOneUse()) {
3552 unsigned Opc = NewLHS.getOpcode();
3553 if (LHS.getOpcode() == ISD::FNEG && fnegFoldsIntoOp(Opc))
3554 ShouldFoldNeg = false;
3555 if (LHS.getOpcode() == ISD::FABS && Opc == ISD::FMUL)
3556 ShouldFoldNeg = false;
3557 }
3558
3559 if (ShouldFoldNeg) {
3560 if (LHS.getOpcode() == ISD::FNEG)
3561 NewRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3562 else if (CRHS->isNegative())
3563 return SDValue();
3564
3565 if (Inv)
3566 std::swap(NewLHS, NewRHS);
3567
3568 SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, VT,
3569 Cond, NewLHS, NewRHS);
3570 DCI.AddToWorklist(NewSelect.getNode());
3571 return DAG.getNode(LHS.getOpcode(), SL, VT, NewSelect);
3572 }
3573 }
3574
3575 return SDValue();
3576 }
3577
3578
3579 SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N,
3580 DAGCombinerInfo &DCI) const {
3581 if (SDValue Folded = foldFreeOpFromSelect(DCI, SDValue(N, 0)))
3582 return Folded;
3583
3584 SDValue Cond = N->getOperand(0);
3585 if (Cond.getOpcode() != ISD::SETCC)
3586 return SDValue();
3587
3588 EVT VT = N->getValueType(0);
3589 SDValue LHS = Cond.getOperand(0);
3590 SDValue RHS = Cond.getOperand(1);
3591 SDValue CC = Cond.getOperand(2);
3592
3593 SDValue True = N->getOperand(1);
3594 SDValue False = N->getOperand(2);
3595
3596 if (Cond.hasOneUse()) { // TODO: Look for multiple select uses.
3597 SelectionDAG &DAG = DCI.DAG;
3598 if (DAG.isConstantValueOfAnyType(True) &&
3599 !DAG.isConstantValueOfAnyType(False)) {
3600 // Swap cmp + select pair to move constant to false input.
3601 // This will allow using VOPC cndmasks more often.
3602 // select (setcc x, y), k, x -> select (setccinv x, y), x, k
3603
3604 SDLoc SL(N);
3605 ISD::CondCode NewCC = getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
3606 LHS.getValueType().isInteger());
3607
3608 SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC);
3609 return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True);
3610 }
3611
3612 if (VT == MVT::f32 && Subtarget->hasFminFmaxLegacy()) {
3613 SDValue MinMax
3614 = combineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI);
3615 // Revisit this node so we can catch min3/max3/med3 patterns.
3616 //DCI.AddToWorklist(MinMax.getNode());
3617 return MinMax;
3618 }
3619 }
3620
3621 // There's no reason to not do this if the condition has other uses.
3622 return performCtlz_CttzCombine(SDLoc(N), Cond, True, False, DCI);
3623 }
3624
3625 static bool isInv2Pi(const APFloat &APF) {
3626 static const APFloat KF16(APFloat::IEEEhalf(), APInt(16, 0x3118));
3627 static const APFloat KF32(APFloat::IEEEsingle(), APInt(32, 0x3e22f983));
3628 static const APFloat KF64(APFloat::IEEEdouble(), APInt(64, 0x3fc45f306dc9c882));
3629
3630 return APF.bitwiseIsEqual(KF16) ||
3631 APF.bitwiseIsEqual(KF32) ||
3632 APF.bitwiseIsEqual(KF64);
3633 }
3634
3635 // 0 and 1.0 / (0.5 * pi) do not have inline immmediates, so there is an
3636 // additional cost to negate them.
3637 bool AMDGPUTargetLowering::isConstantCostlierToNegate(SDValue N) const {
3638 if (const ConstantFPSDNode *C = isConstOrConstSplatFP(N)) {
3639 if (C->isZero() && !C->isNegative())
3640 return true;
3641
3642 if (Subtarget->hasInv2PiInlineImm() && isInv2Pi(C->getValueAPF()))
3643 return true;
3644 }
3645
3646 return false;
3647 }
3648
3649 static unsigned inverseMinMax(unsigned Opc) {
3650 switch (Opc) {
3651 case ISD::FMAXNUM:
3652 return ISD::FMINNUM;
3653 case ISD::FMINNUM:
3654 return ISD::FMAXNUM;
3655 case ISD::FMAXNUM_IEEE:
3656 return ISD::FMINNUM_IEEE;
3657 case ISD::FMINNUM_IEEE:
3658 return ISD::FMAXNUM_IEEE;
3659 case AMDGPUISD::FMAX_LEGACY:
3660 return AMDGPUISD::FMIN_LEGACY;
3661 case AMDGPUISD::FMIN_LEGACY:
3662 return AMDGPUISD::FMAX_LEGACY;
3663 default:
3664 llvm_unreachable("invalid min/max opcode");
3665 }
3666 }
3667
3668 SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
3669 DAGCombinerInfo &DCI) const {
3670 SelectionDAG &DAG = DCI.DAG;
3671 SDValue N0 = N->getOperand(0);
3672 EVT VT = N->getValueType(0);
3673
3674 unsigned Opc = N0.getOpcode();
3675
3676 // If the input has multiple uses and we can either fold the negate down, or
3677 // the other uses cannot, give up. This both prevents unprofitable
3678 // transformations and infinite loops: we won't repeatedly try to fold around
3679 // a negate that has no 'good' form.
3680 if (N0.hasOneUse()) {
3681 // This may be able to fold into the source, but at a code size cost. Don't
3682 // fold if the fold into the user is free.
3683 if (allUsesHaveSourceMods(N, 0))
3684 return SDValue();
3685 } else {
3686 if (fnegFoldsIntoOp(Opc) &&
3687 (allUsesHaveSourceMods(N) || !allUsesHaveSourceMods(N0.getNode())))
3688 return SDValue();
3689 }
3690
3691 SDLoc SL(N);
3692 switch (Opc) {
3693 case ISD::FADD: {
3694 if (!mayIgnoreSignedZero(N0))
3695 return SDValue();
3696
3697 // (fneg (fadd x, y)) -> (fadd (fneg x), (fneg y))
3698 SDValue LHS = N0.getOperand(0);
3699 SDValue RHS = N0.getOperand(1);
3700
3701 if (LHS.getOpcode() != ISD::FNEG)
3702 LHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3703 else
3704 LHS = LHS.getOperand(0);
3705
3706 if (RHS.getOpcode() != ISD::FNEG)
3707 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3708 else
3709 RHS = RHS.getOperand(0);
3710
3711 SDValue Res = DAG.getNode(ISD::FADD, SL, VT, LHS, RHS, N0->getFlags());
3712 if (Res.getOpcode() != ISD::FADD)
3713 return SDValue(); // Op got folded away.
3714 if (!N0.hasOneUse())
3715 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3716 return Res;
3717 }
3718 case ISD::FMUL:
3719 case AMDGPUISD::FMUL_LEGACY: {
3720 // (fneg (fmul x, y)) -> (fmul x, (fneg y))
3721 // (fneg (fmul_legacy x, y)) -> (fmul_legacy x, (fneg y))
3722 SDValue LHS = N0.getOperand(0);
3723 SDValue RHS = N0.getOperand(1);
3724
3725 if (LHS.getOpcode() == ISD::FNEG)
3726 LHS = LHS.getOperand(0);
3727 else if (RHS.getOpcode() == ISD::FNEG)
3728 RHS = RHS.getOperand(0);
3729 else
3730 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3731
3732 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, RHS, N0->getFlags());
3733 if (Res.getOpcode() != Opc)
3734 return SDValue(); // Op got folded away.
3735 if (!N0.hasOneUse())
3736 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3737 return Res;
3738 }
3739 case ISD::FMA:
3740 case ISD::FMAD: {
3741 if (!mayIgnoreSignedZero(N0))
3742 return SDValue();
3743
3744 // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
3745 SDValue LHS = N0.getOperand(0);
3746 SDValue MHS = N0.getOperand(1);
3747 SDValue RHS = N0.getOperand(2);
3748
3749 if (LHS.getOpcode() == ISD::FNEG)
3750 LHS = LHS.getOperand(0);
3751 else if (MHS.getOpcode() == ISD::FNEG)
3752 MHS = MHS.getOperand(0);
3753 else
3754 MHS = DAG.getNode(ISD::FNEG, SL, VT, MHS);
3755
3756 if (RHS.getOpcode() != ISD::FNEG)
3757 RHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3758 else
3759 RHS = RHS.getOperand(0);
3760
3761 SDValue Res = DAG.getNode(Opc, SL, VT, LHS, MHS, RHS);
3762 if (Res.getOpcode() != Opc)
3763 return SDValue(); // Op got folded away.
3764 if (!N0.hasOneUse())
3765 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3766 return Res;
3767 }
3768 case ISD::FMAXNUM:
3769 case ISD::FMINNUM:
3770 case ISD::FMAXNUM_IEEE:
3771 case ISD::FMINNUM_IEEE:
3772 case AMDGPUISD::FMAX_LEGACY:
3773 case AMDGPUISD::FMIN_LEGACY: {
3774 // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
3775 // fneg (fminnum x, y) -> fmaxnum (fneg x), (fneg y)
3776 // fneg (fmax_legacy x, y) -> fmin_legacy (fneg x), (fneg y)
3777 // fneg (fmin_legacy x, y) -> fmax_legacy (fneg x), (fneg y)
3778
3779 SDValue LHS = N0.getOperand(0);
3780 SDValue RHS = N0.getOperand(1);
3781
3782 // 0 doesn't have a negated inline immediate.
3783 // TODO: This constant check should be generalized to other operations.
3784 if (isConstantCostlierToNegate(RHS))
3785 return SDValue();
3786
3787 SDValue NegLHS = DAG.getNode(ISD::FNEG, SL, VT, LHS);
3788 SDValue NegRHS = DAG.getNode(ISD::FNEG, SL, VT, RHS);
3789 unsigned Opposite = inverseMinMax(Opc);
3790
3791 SDValue Res = DAG.getNode(Opposite, SL, VT, NegLHS, NegRHS, N0->getFlags());
3792 if (Res.getOpcode() != Opposite)
3793 return SDValue(); // Op got folded away.
3794 if (!N0.hasOneUse())
3795 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3796 return Res;
3797 }
3798 case AMDGPUISD::FMED3: {
3799 SDValue Ops[3];
3800 for (unsigned I = 0; I < 3; ++I)
3801 Ops[I] = DAG.getNode(ISD::FNEG, SL, VT, N0->getOperand(I), N0->getFlags());
3802
3803 SDValue Res = DAG.getNode(AMDGPUISD::FMED3, SL, VT, Ops, N0->getFlags());
3804 if (Res.getOpcode() != AMDGPUISD::FMED3)
3805 return SDValue(); // Op got folded away.
3806 if (!N0.hasOneUse())
3807 DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
3808 return Res;
3809 }
3810 case ISD::FP_EXTEND:
3811 case ISD::FTRUNC:
3812 case ISD::FRINT:
3813 case ISD::FNEARBYINT: // XXX - Should fround be handled?
3814 case ISD::FSIN:
3815 case ISD::FCANONICALIZE:
3816 case AMDGPUISD::RCP:
3817 case AMDGPUISD::RCP_LEGACY:
3818 case AMDGPUISD::RCP_IFLAG:
3819 case AMDGPUISD::SIN_HW: {
3820 SDValue CvtSrc = N0.getOperand(0);
3821 if (CvtSrc.getOpcode() == ISD::FNEG) {
3822 // (fneg (fp_extend (fneg x))) -> (fp_extend x)
3823 // (fneg (rcp (fneg x))) -> (rcp x)
3824 return DAG.getNode(Opc, SL, VT, CvtSrc.getOperand(0));
3825 }
3826
3827 if (!N0.hasOneUse())
3828 return SDValue();
3829
3830 // (fneg (fp_extend x)) -> (fp_extend (fneg x))
3831 // (fneg (rcp x)) -> (rcp (fneg x))
3832 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3833 return DAG.getNode(Opc, SL, VT, Neg, N0->getFlags());
3834 }
3835 case ISD::FP_ROUND: {
3836 SDValue CvtSrc = N0.getOperand(0);
3837
3838 if (CvtSrc.getOpcode() == ISD::FNEG) {
3839 // (fneg (fp_round (fneg x))) -> (fp_round x)
3840 return DAG.getNode(ISD::FP_ROUND, SL, VT,
3841 CvtSrc.getOperand(0), N0.getOperand(1));
3842 }
3843
3844 if (!N0.hasOneUse())
3845 return SDValue();
3846
3847 // (fneg (fp_round x)) -> (fp_round (fneg x))
3848 SDValue Neg = DAG.getNode(ISD::FNEG, SL, CvtSrc.getValueType(), CvtSrc);
3849 return DAG.getNode(ISD::FP_ROUND, SL, VT, Neg, N0.getOperand(1));
3850 }
3851 case ISD::FP16_TO_FP: {
3852 // v_cvt_f32_f16 supports source modifiers on pre-VI targets without legal
3853 // f16, but legalization of f16 fneg ends up pulling it out of the source.
3854 // Put the fneg back as a legal source operation that can be matched later.
3855 SDLoc SL(N);
3856
3857 SDValue Src = N0.getOperand(0);
3858 EVT SrcVT = Src.getValueType();
3859
3860 // fneg (fp16_to_fp x) -> fp16_to_fp (xor x, 0x8000)
3861 SDValue IntFNeg = DAG.getNode(ISD::XOR, SL, SrcVT, Src,
3862 DAG.getConstant(0x8000, SL, SrcVT));
3863 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFNeg);
3864 }
3865 default:
3866 return SDValue();
3867 }
3868 }
3869
3870 SDValue AMDGPUTargetLowering::performFAbsCombine(SDNode *N,
3871 DAGCombinerInfo &DCI) const {
3872 SelectionDAG &DAG = DCI.DAG;
3873 SDValue N0 = N->getOperand(0);
3874
3875 if (!N0.hasOneUse())
3876 return SDValue();
3877
3878 switch (N0.getOpcode()) {
3879 case ISD::FP16_TO_FP: {
3880 assert(!Subtarget->has16BitInsts() && "should only see if f16 is illegal");
3881 SDLoc SL(N);
3882 SDValue Src = N0.getOperand(0);
3883 EVT SrcVT = Src.getValueType();
3884
3885 // fabs (fp16_to_fp x) -> fp16_to_fp (and x, 0x7fff)
3886 SDValue IntFAbs = DAG.getNode(ISD::AND, SL, SrcVT, Src,
3887 DAG.getConstant(0x7fff, SL, SrcVT));
3888 return DAG.getNode(ISD::FP16_TO_FP, SL, N->getValueType(0), IntFAbs);
3889 }
3890 default:
3891 return SDValue();
3892 }
3893 }
3894
3895 SDValue AMDGPUTargetLowering::performRcpCombine(SDNode *N,
3896 DAGCombinerInfo &DCI) const {
3897 const auto *CFP = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
3898 if (!CFP)
3899 return SDValue();
3900
3901 // XXX - Should this flush denormals?
3902 const APFloat &Val = CFP->getValueAPF();
3903 APFloat One(Val.getSemantics(), "1.0");
3904 return DCI.DAG.getConstantFP(One / Val, SDLoc(N), N->getValueType(0));
3905 }
3906
3907 SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
3908 DAGCombinerInfo &DCI) const {
3909 SelectionDAG &DAG = DCI.DAG;
3910 SDLoc DL(N);
3911
3912 switch(N->getOpcode()) {
3913 default:
3914 break;
3915 case ISD::BITCAST: {
3916 EVT DestVT = N->getValueType(0);
3917
3918 // Push casts through vector builds. This helps avoid emitting a large
3919 // number of copies when materializing floating point vector constants.
3920 //
3921 // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) =>
3922 // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y))
3923 if (DestVT.isVector()) {
3924 SDValue Src = N->getOperand(0);
3925 if (Src.getOpcode() == ISD::BUILD_VECTOR) {
3926 EVT SrcVT = Src.getValueType();
3927 unsigned NElts = DestVT.getVectorNumElements();
3928
3929 if (SrcVT.getVectorNumElements() == NElts) {
3930 EVT DestEltVT = DestVT.getVectorElementType();
3931
3932 SmallVector<SDValue, 8> CastedElts;
3933 SDLoc SL(N);
3934 for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) {
3935 SDValue Elt = Src.getOperand(I);
3936 CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt));
3937 }
3938
3939 return DAG.getBuildVector(DestVT, SL, CastedElts);
3940 }
3941 }
3942 }
3943
3944 if (DestVT.getSizeInBits() != 64 && !DestVT.isVector())
3945 break;
3946
3947 // Fold bitcasts of constants.
3948 //
3949 // v2i32 (bitcast i64:k) -> build_vector lo_32(k), hi_32(k)
3950 // TODO: Generalize and move to DAGCombiner
3951 SDValue Src = N->getOperand(0);
3952 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Src)) {
3953 if (Src.getValueType() == MVT::i64) {
3954 SDLoc SL(N);
3955 uint64_t CVal = C->getZExtValue();
3956 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3957 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
3958 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
3959 return DAG.getNode(ISD::BITCAST, SL, DestVT, BV);
3960 }
3961 }
3962
3963 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Src)) {
3964 const APInt &Val = C->getValueAPF().bitcastToAPInt();
3965 SDLoc SL(N);
3966 uint64_t CVal = Val.getZExtValue();
3967 SDValue Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32,
3968 DAG.getConstant(Lo_32(CVal), SL, MVT::i32),
3969 DAG.getConstant(Hi_32(CVal), SL, MVT::i32));
3970
3971 return DAG.getNode(ISD::BITCAST, SL, DestVT, Vec);
3972 }
3973
3974 break;
3975 }
3976 case ISD::SHL: {
3977 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
3978 break;
3979
3980 return performShlCombine(N, DCI);
3981 }
3982 case ISD::SRL: {
3983 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
3984 break;
3985
3986 return performSrlCombine(N, DCI);
3987 }
3988 case ISD::SRA: {
3989 if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
3990 break;
3991
3992 return performSraCombine(N, DCI);
3993 }
3994 case ISD::TRUNCATE:
3995 return performTruncateCombine(N, DCI);
3996 case ISD::MUL:
3997 return performMulCombine(N, DCI);
3998 case ISD::MULHS:
3999 return performMulhsCombine(N, DCI);
4000 case ISD::MULHU:
4001 return performMulhuCombine(N, DCI);
4002 case AMDGPUISD::MUL_I24:
4003 case AMDGPUISD::MUL_U24:
4004 case AMDGPUISD::MULHI_I24:
4005 case AMDGPUISD::MULHI_U24: {
4006 if (SDValue V = simplifyI24(N, DCI))
4007 return V;
4008 return SDValue();
4009 }
4010 case AMDGPUISD::MUL_LOHI_I24:
4011 case AMDGPUISD::MUL_LOHI_U24:
4012 return performMulLoHi24Combine(N, DCI);
4013 case ISD::SELECT:
4014 return performSelectCombine(N, DCI);
4015 case ISD::FNEG:
4016 return performFNegCombine(N, DCI);
4017 case ISD::FABS:
4018 return performFAbsCombine(N, DCI);
4019 case AMDGPUISD::BFE_I32:
4020 case AMDGPUISD::BFE_U32: {
4021 assert(!N->getValueType(0).isVector() &&
4022 "Vector handling of BFE not implemented");
4023 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
4024 if (!Width)
4025 break;
4026
4027 uint32_t WidthVal = Width->getZExtValue() & 0x1f;
4028 if (WidthVal == 0)
4029 return DAG.getConstant(0, DL, MVT::i32);
4030
4031 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
4032 if (!Offset)
4033 break;
4034
4035 SDValue BitsFrom = N->getOperand(0);
4036 uint32_t OffsetVal = Offset->getZExtValue() & 0x1f;
4037
4038 bool Signed = N->getOpcode() == AMDGPUISD::BFE_I32;
4039
4040 if (OffsetVal == 0) {
4041 // This is already sign / zero extended, so try to fold away extra BFEs.
4042 unsigned SignBits = Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
4043
4044 unsigned OpSignBits = DAG.ComputeNumSignBits(BitsFrom);
4045 if (OpSignBits >= SignBits)
4046 return BitsFrom;
4047
4048 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), WidthVal);
4049 if (Signed) {
4050 // This is a sign_extend_inreg. Replace it to take advantage of existing
4051 // DAG Combines. If not eliminated, we will match back to BFE during
4052 // selection.
4053
4054 // TODO: The sext_inreg of extended types ends, although we can could
4055 // handle them in a single BFE.
4056 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom,
4057 DAG.getValueType(SmallVT));
4058 }
4059
4060 return DAG.getZeroExtendInReg(BitsFrom, DL, SmallVT);
4061 }
4062
4063 if (ConstantSDNode *CVal = dyn_cast<ConstantSDNode>(BitsFrom)) {
4064 if (Signed) {
4065 return constantFoldBFE<int32_t>(DAG,
4066 CVal->getSExtValue(),
4067 OffsetVal,
4068 WidthVal,
4069 DL);
4070 }
4071
4072 return constantFoldBFE<uint32_t>(DAG,
4073 CVal->getZExtValue(),
4074 OffsetVal,
4075 WidthVal,
4076 DL);
4077 }
4078
4079 if ((OffsetVal + WidthVal) >= 32 &&
4080 !(Subtarget->hasSDWA() && OffsetVal == 16 && WidthVal == 16)) {
4081 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32);
4082 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32,
4083 BitsFrom, ShiftVal);
4084 }
4085
4086 if (BitsFrom.hasOneUse()) {
4087 APInt Demanded = APInt::getBitsSet(32,
4088 OffsetVal,
4089 OffsetVal + WidthVal);
4090
4091 KnownBits Known;
4092 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
4093 !DCI.isBeforeLegalizeOps());
4094 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4095 if (TLI.ShrinkDemandedConstant(BitsFrom, Demanded, TLO) ||
4096 TLI.SimplifyDemandedBits(BitsFrom, Demanded, Known, TLO)) {
4097 DCI.CommitTargetLoweringOpt(TLO);
4098 }
4099 }
4100
4101 break;
4102 }
4103 case ISD::LOAD:
4104 return performLoadCombine(N, DCI);
4105 case ISD::STORE:
4106 return performStoreCombine(N, DCI);
4107 case AMDGPUISD::RCP:
4108 case AMDGPUISD::RCP_IFLAG:
4109 return performRcpCombine(N, DCI);
4110 case ISD::AssertZext:
4111 case ISD::AssertSext:
4112 return performAssertSZExtCombine(N, DCI);
4113 case ISD::INTRINSIC_WO_CHAIN:
4114 return performIntrinsicWOChainCombine(N, DCI);
4115 }
4116 return SDValue();
4117 }
4118
4119 //===----------------------------------------------------------------------===//
4120 // Helper functions
4121 //===----------------------------------------------------------------------===//
4122
4123 SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG,
4124 const TargetRegisterClass *RC,
4125 unsigned Reg, EVT VT,
4126 const SDLoc &SL,
4127 bool RawReg) const {
4128 MachineFunction &MF = DAG.getMachineFunction();
4129 MachineRegisterInfo &MRI = MF.getRegInfo();
4130 unsigned VReg;
4131
4132 if (!MRI.isLiveIn(Reg)) {
4133 VReg = MRI.createVirtualRegister(RC);
4134 MRI.addLiveIn(Reg, VReg);
4135 } else {
4136 VReg = MRI.getLiveInVirtReg(Reg);
4137 }
4138
4139 if (RawReg)
4140 return DAG.getRegister(VReg, VT);
4141
4142 return DAG.getCopyFromReg(DAG.getEntryNode(), SL, VReg, VT);
4143 }
4144
4145 // This may be called multiple times, and nothing prevents creating multiple
4146 // objects at the same offset. See if we already defined this object.
4147 static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size,
4148 int64_t Offset) {
4149 for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) {
4150 if (MFI.getObjectOffset(I) == Offset) {
4151 assert(MFI.getObjectSize(I) == Size);
4152 return I;
4153 }
4154 }
4155
4156 return MFI.CreateFixedObject(Size, Offset, true);
4157 }
4158
4159 SDValue AMDGPUTargetLowering::loadStackInputValue(SelectionDAG &DAG,
4160 EVT VT,
4161 const SDLoc &SL,
4162 int64_t Offset) const {
4163 MachineFunction &MF = DAG.getMachineFunction();
4164 MachineFrameInfo &MFI = MF.getFrameInfo();
4165 int FI = getOrCreateFixedStackObject(MFI, VT.getStoreSize(), Offset);
4166
4167 auto SrcPtrInfo = MachinePointerInfo::getStack(MF, Offset);
4168 SDValue Ptr = DAG.getFrameIndex(FI, MVT::i32);
4169
4170 return DAG.getLoad(VT, SL, DAG.getEntryNode(), Ptr, SrcPtrInfo, 4,
4171 MachineMemOperand::MODereferenceable |
4172 MachineMemOperand::MOInvariant);
4173 }
4174
4175 SDValue AMDGPUTargetLowering::storeStackInputValue(SelectionDAG &DAG,
4176 const SDLoc &SL,
4177 SDValue Chain,
4178 SDValue ArgVal,
4179 int64_t Offset) const {
4180 MachineFunction &MF = DAG.getMachineFunction();
4181 MachinePointerInfo DstInfo = MachinePointerInfo::getStack(MF, Offset);
4182
4183 SDValue Ptr = DAG.getConstant(Offset, SL, MVT::i32);
4184 SDValue Store = DAG.getStore(Chain, SL, ArgVal, Ptr, DstInfo, 4,
4185 MachineMemOperand::MODereferenceable);
4186 return Store;
4187 }
4188
4189 SDValue AMDGPUTargetLowering::loadInputValue(SelectionDAG &DAG,
4190 const TargetRegisterClass *RC,
4191 EVT VT, const SDLoc &SL,
4192 const ArgDescriptor &Arg) const {
4193 assert(Arg && "Attempting to load missing argument");
4194
4195 SDValue V = Arg.isRegister() ?
4196 CreateLiveInRegister(DAG, RC, Arg.getRegister(), VT, SL) :
4197 loadStackInputValue(DAG, VT, SL, Arg.getStackOffset());
4198
4199 if (!Arg.isMasked())
4200 return V;
4201
4202 unsigned Mask = Arg.getMask();
4203 unsigned Shift = countTrailingZeros<unsigned>(Mask);
4204 V = DAG.getNode(ISD::SRL, SL, VT, V,
4205 DAG.getShiftAmountConstant(Shift, VT, SL));
4206 return DAG.getNode(ISD::AND, SL, VT, V,
4207 DAG.getConstant(Mask >> Shift, SL, VT));
4208 }
4209
4210 uint32_t AMDGPUTargetLowering::getImplicitParameterOffset(
4211 const MachineFunction &MF, const ImplicitParameter Param) const {
4212 const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>();
4213 const AMDGPUSubtarget &ST =
4214 AMDGPUSubtarget::get(getTargetMachine(), MF.getFunction());
4215 unsigned ExplicitArgOffset = ST.getExplicitKernelArgOffset(MF.getFunction());
4216 unsigned Alignment = ST.getAlignmentForImplicitArgPtr();
4217 uint64_t ArgOffset = alignTo(MFI->getExplicitKernArgSize(), Alignment) +
4218 ExplicitArgOffset;
4219 switch (Param) {
4220 case GRID_DIM:
4221 return ArgOffset;
4222 case GRID_OFFSET:
4223 return ArgOffset + 4;
4224 }
4225 llvm_unreachable("unexpected implicit parameter type");
4226 }
4227
4228 #define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
4229
4230 const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
4231 switch ((AMDGPUISD::NodeType)Opcode) {
4232 case AMDGPUISD::FIRST_NUMBER: break;
4233 // AMDIL DAG nodes
4234 NODE_NAME_CASE(UMUL);
4235 NODE_NAME_CASE(BRANCH_COND);
4236
4237 // AMDGPU DAG nodes
4238 NODE_NAME_CASE(IF)
4239 NODE_NAME_CASE(ELSE)
4240 NODE_NAME_CASE(LOOP)
4241 NODE_NAME_CASE(CALL)
4242 NODE_NAME_CASE(TC_RETURN)
4243 NODE_NAME_CASE(TRAP)
4244 NODE_NAME_CASE(RET_FLAG)
4245 NODE_NAME_CASE(RETURN_TO_EPILOG)
4246 NODE_NAME_CASE(ENDPGM)
4247 NODE_NAME_CASE(DWORDADDR)
4248 NODE_NAME_CASE(FRACT)
4249 NODE_NAME_CASE(SETCC)
4250 NODE_NAME_CASE(SETREG)
4251 NODE_NAME_CASE(DENORM_MODE)
4252 NODE_NAME_CASE(FMA_W_CHAIN)
4253 NODE_NAME_CASE(FMUL_W_CHAIN)
4254 NODE_NAME_CASE(CLAMP)
4255 NODE_NAME_CASE(COS_HW)
4256 NODE_NAME_CASE(SIN_HW)
4257 NODE_NAME_CASE(FMAX_LEGACY)
4258 NODE_NAME_CASE(FMIN_LEGACY)
4259 NODE_NAME_CASE(FMAX3)
4260 NODE_NAME_CASE(SMAX3)
4261 NODE_NAME_CASE(UMAX3)
4262 NODE_NAME_CASE(FMIN3)
4263 NODE_NAME_CASE(SMIN3)
4264 NODE_NAME_CASE(UMIN3)
4265 NODE_NAME_CASE(FMED3)
4266 NODE_NAME_CASE(SMED3)
4267 NODE_NAME_CASE(UMED3)
4268 NODE_NAME_CASE(FDOT2)
4269 NODE_NAME_CASE(URECIP)
4270 NODE_NAME_CASE(DIV_SCALE)
4271 NODE_NAME_CASE(DIV_FMAS)
4272 NODE_NAME_CASE(DIV_FIXUP)
4273 NODE_NAME_CASE(FMAD_FTZ)
4274 NODE_NAME_CASE(TRIG_PREOP)
4275 NODE_NAME_CASE(RCP)
4276 NODE_NAME_CASE(RSQ)
4277 NODE_NAME_CASE(RCP_LEGACY)
4278 NODE_NAME_CASE(RSQ_LEGACY)
4279 NODE_NAME_CASE(RCP_IFLAG)
4280 NODE_NAME_CASE(FMUL_LEGACY)
4281 NODE_NAME_CASE(RSQ_CLAMP)
4282 NODE_NAME_CASE(LDEXP)
4283 NODE_NAME_CASE(FP_CLASS)
4284 NODE_NAME_CASE(DOT4)
4285 NODE_NAME_CASE(CARRY)
4286 NODE_NAME_CASE(BORROW)
4287 NODE_NAME_CASE(BFE_U32)
4288 NODE_NAME_CASE(BFE_I32)
4289 NODE_NAME_CASE(BFI)
4290 NODE_NAME_CASE(BFM)
4291 NODE_NAME_CASE(FFBH_U32)
4292 NODE_NAME_CASE(FFBH_I32)
4293 NODE_NAME_CASE(FFBL_B32)
4294 NODE_NAME_CASE(MUL_U24)
4295 NODE_NAME_CASE(MUL_I24)
4296 NODE_NAME_CASE(MULHI_U24)
4297 NODE_NAME_CASE(MULHI_I24)
4298 NODE_NAME_CASE(MUL_LOHI_U24)
4299 NODE_NAME_CASE(MUL_LOHI_I24)
4300 NODE_NAME_CASE(MAD_U24)
4301 NODE_NAME_CASE(MAD_I24)
4302 NODE_NAME_CASE(MAD_I64_I32)
4303 NODE_NAME_CASE(MAD_U64_U32)
4304 NODE_NAME_CASE(PERM)
4305 NODE_NAME_CASE(TEXTURE_FETCH)
4306 NODE_NAME_CASE(EXPORT)
4307 NODE_NAME_CASE(EXPORT_DONE)
4308 NODE_NAME_CASE(R600_EXPORT)
4309 NODE_NAME_CASE(CONST_ADDRESS)
4310 NODE_NAME_CASE(REGISTER_LOAD)
4311 NODE_NAME_CASE(REGISTER_STORE)
4312 NODE_NAME_CASE(SAMPLE)
4313 NODE_NAME_CASE(SAMPLEB)
4314 NODE_NAME_CASE(SAMPLED)
4315 NODE_NAME_CASE(SAMPLEL)
4316 NODE_NAME_CASE(CVT_F32_UBYTE0)
4317 NODE_NAME_CASE(CVT_F32_UBYTE1)
4318 NODE_NAME_CASE(CVT_F32_UBYTE2)
4319 NODE_NAME_CASE(CVT_F32_UBYTE3)
4320 NODE_NAME_CASE(CVT_PKRTZ_F16_F32)
4321 NODE_NAME_CASE(CVT_PKNORM_I16_F32)
4322 NODE_NAME_CASE(CVT_PKNORM_U16_F32)
4323 NODE_NAME_CASE(CVT_PK_I16_I32)
4324 NODE_NAME_CASE(CVT_PK_U16_U32)
4325 NODE_NAME_CASE(FP_TO_FP16)
4326 NODE_NAME_CASE(FP16_ZEXT)
4327 NODE_NAME_CASE(BUILD_VERTICAL_VECTOR)
4328 NODE_NAME_CASE(CONST_DATA_PTR)
4329 NODE_NAME_CASE(PC_ADD_REL_OFFSET)
4330 NODE_NAME_CASE(LDS)
4331 NODE_NAME_CASE(KILL)
4332 NODE_NAME_CASE(DUMMY_CHAIN)
4333 case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
4334 NODE_NAME_CASE(INTERP_MOV)
4335 NODE_NAME_CASE(INTERP_P1)
4336 NODE_NAME_CASE(INTERP_P2)
4337 NODE_NAME_CASE(INTERP_P1LL_F16)
4338 NODE_NAME_CASE(INTERP_P1LV_F16)
4339 NODE_NAME_CASE(INTERP_P2_F16)
4340 NODE_NAME_CASE(LOAD_D16_HI)
4341 NODE_NAME_CASE(LOAD_D16_LO)
4342 NODE_NAME_CASE(LOAD_D16_HI_I8)
4343 NODE_NAME_CASE(LOAD_D16_HI_U8)
4344 NODE_NAME_CASE(LOAD_D16_LO_I8)
4345 NODE_NAME_CASE(LOAD_D16_LO_U8)
4346 NODE_NAME_CASE(STORE_MSKOR)
4347 NODE_NAME_CASE(LOAD_CONSTANT)
4348 NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
4349 NODE_NAME_CASE(TBUFFER_STORE_FORMAT_D16)
4350 NODE_NAME_CASE(TBUFFER_LOAD_FORMAT)
4351 NODE_NAME_CASE(TBUFFER_LOAD_FORMAT_D16)
4352 NODE_NAME_CASE(DS_ORDERED_COUNT)
4353 NODE_NAME_CASE(ATOMIC_CMP_SWAP)
4354 NODE_NAME_CASE(ATOMIC_INC)
4355 NODE_NAME_CASE(ATOMIC_DEC)
4356 NODE_NAME_CASE(ATOMIC_LOAD_FMIN)
4357 NODE_NAME_CASE(ATOMIC_LOAD_FMAX)
4358 NODE_NAME_CASE(BUFFER_LOAD)
4359 NODE_NAME_CASE(BUFFER_LOAD_UBYTE)
4360 NODE_NAME_CASE(BUFFER_LOAD_USHORT)
4361 NODE_NAME_CASE(BUFFER_LOAD_BYTE)
4362 NODE_NAME_CASE(BUFFER_LOAD_SHORT)
4363 NODE_NAME_CASE(BUFFER_LOAD_FORMAT)
4364 NODE_NAME_CASE(BUFFER_LOAD_FORMAT_D16)
4365 NODE_NAME_CASE(SBUFFER_LOAD)
4366 NODE_NAME_CASE(BUFFER_STORE)
4367 NODE_NAME_CASE(BUFFER_STORE_BYTE)
4368 NODE_NAME_CASE(BUFFER_STORE_SHORT)
4369 NODE_NAME_CASE(BUFFER_STORE_FORMAT)
4370 NODE_NAME_CASE(BUFFER_STORE_FORMAT_D16)
4371 NODE_NAME_CASE(BUFFER_ATOMIC_SWAP)
4372 NODE_NAME_CASE(BUFFER_ATOMIC_ADD)
4373 NODE_NAME_CASE(BUFFER_ATOMIC_SUB)
4374 NODE_NAME_CASE(BUFFER_ATOMIC_SMIN)
4375 NODE_NAME_CASE(BUFFER_ATOMIC_UMIN)
4376 NODE_NAME_CASE(BUFFER_ATOMIC_SMAX)
4377 NODE_NAME_CASE(BUFFER_ATOMIC_UMAX)
4378 NODE_NAME_CASE(BUFFER_ATOMIC_AND)
4379 NODE_NAME_CASE(BUFFER_ATOMIC_OR)
4380 NODE_NAME_CASE(BUFFER_ATOMIC_XOR)
4381 NODE_NAME_CASE(BUFFER_ATOMIC_INC)
4382 NODE_NAME_CASE(BUFFER_ATOMIC_DEC)
4383 NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP)
4384 NODE_NAME_CASE(BUFFER_ATOMIC_FADD)
4385 NODE_NAME_CASE(BUFFER_ATOMIC_PK_FADD)
4386 NODE_NAME_CASE(ATOMIC_FADD)
4387 NODE_NAME_CASE(ATOMIC_PK_FADD)
4388
4389 case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break;
4390 }
4391 return nullptr;
4392 }
4393
4394 SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand,
4395 SelectionDAG &DAG, int Enabled,
4396 int &RefinementSteps,
4397 bool &UseOneConstNR,
4398 bool Reciprocal) const {
4399 EVT VT = Operand.getValueType();
4400
4401 if (VT == MVT::f32) {
4402 RefinementSteps = 0;
4403 return DAG.getNode(AMDGPUISD::RSQ, SDLoc(Operand), VT, Operand);
4404 }
4405
4406 // TODO: There is also f64 rsq instruction, but the documentation is less
4407 // clear on its precision.
4408
4409 return SDValue();
4410 }
4411
4412 SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand,
4413 SelectionDAG &DAG, int Enabled,
4414 int &RefinementSteps) const {
4415 EVT VT = Operand.getValueType();
4416
4417 if (VT == MVT::f32) {
4418 // Reciprocal, < 1 ulp error.
4419 //
4420 // This reciprocal approximation converges to < 0.5 ulp error with one
4421 // newton rhapson performed with two fused multiple adds (FMAs).
4422
4423 RefinementSteps = 0;
4424 return DAG.getNode(AMDGPUISD::RCP, SDLoc(Operand), VT, Operand);
4425 }
4426
4427 // TODO: There is also f64 rcp instruction, but the documentation is less
4428 // clear on its precision.
4429
4430 return SDValue();
4431 }
4432
4433 void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
4434 const SDValue Op, KnownBits &Known,
4435 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
4436
4437 Known.resetAll(); // Don't know anything.
4438
4439 unsigned Opc = Op.getOpcode();
4440
4441 switch (Opc) {
4442 default:
4443 break;
4444 case AMDGPUISD::CARRY:
4445 case AMDGPUISD::BORROW: {
4446 Known.Zero = APInt::getHighBitsSet(32, 31);
4447 break;
4448 }
4449
4450 case AMDGPUISD::BFE_I32:
4451 case AMDGPUISD::BFE_U32: {
4452 ConstantSDNode *CWidth = dyn_cast<ConstantSDNode>(Op.getOperand(2));
4453 if (!CWidth)
4454 return;
4455
4456 uint32_t Width = CWidth->getZExtValue() & 0x1f;
4457
4458 if (Opc == AMDGPUISD::BFE_U32)
4459 Known.Zero = APInt::getHighBitsSet(32, 32 - Width);
4460
4461 break;
4462 }
4463 case AMDGPUISD::FP_TO_FP16:
4464 case AMDGPUISD::FP16_ZEXT: {
4465 unsigned BitWidth = Known.getBitWidth();
4466
4467 // High bits are zero.
4468 Known.Zero = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
4469 break;
4470 }
4471 case AMDGPUISD::MUL_U24:
4472 case AMDGPUISD::MUL_I24: {
4473 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
4474 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
4475 unsigned TrailZ = LHSKnown.countMinTrailingZeros() +
4476 RHSKnown.countMinTrailingZeros();
4477 Known.Zero.setLowBits(std::min(TrailZ, 32u));
4478
4479 // Truncate to 24 bits.
4480 LHSKnown = LHSKnown.trunc(24);
4481 RHSKnown = RHSKnown.trunc(24);
4482
4483 bool Negative = false;
4484 if (Opc == AMDGPUISD::MUL_I24) {
4485 unsigned LHSValBits = 24 - LHSKnown.countMinSignBits();
4486 unsigned RHSValBits = 24 - RHSKnown.countMinSignBits();
4487 unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
4488 if (MaxValBits >= 32)
4489 break;
4490 bool LHSNegative = LHSKnown.isNegative();
4491 bool LHSPositive = LHSKnown.isNonNegative();
4492 bool RHSNegative = RHSKnown.isNegative();
4493 bool RHSPositive = RHSKnown.isNonNegative();
4494 if ((!LHSNegative && !LHSPositive) || (!RHSNegative && !RHSPositive))
4495 break;
4496 Negative = (LHSNegative && RHSPositive) || (LHSPositive && RHSNegative);
4497 if (Negative)
4498 Known.One.setHighBits(32 - MaxValBits);
4499 else
4500 Known.Zero.setHighBits(32 - MaxValBits);
4501 } else {
4502 unsigned LHSValBits = 24 - LHSKnown.countMinLeadingZeros();
4503 unsigned RHSValBits = 24 - RHSKnown.countMinLeadingZeros();
4504 unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
4505 if (MaxValBits >= 32)
4506 break;
4507 Known.Zero.setHighBits(32 - MaxValBits);
4508 }
4509 break;
4510 }
4511 case AMDGPUISD::PERM: {
4512 ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(Op.getOperand(2));
4513 if (!CMask)
4514 return;
4515
4516 KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
4517 KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
4518 unsigned Sel = CMask->getZExtValue();
4519
4520 for (unsigned I = 0; I < 32; I += 8) {
4521 unsigned SelBits = Sel & 0xff;
4522 if (SelBits < 4) {
4523 SelBits *= 8;
4524 Known.One |= ((RHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
4525 Known.Zero |= ((RHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
4526 } else if (SelBits < 7) {
4527 SelBits = (SelBits & 3) * 8;
4528 Known.One |= ((LHSKnown.One.getZExtValue() >> SelBits) & 0xff) << I;
4529 Known.Zero |= ((LHSKnown.Zero.getZExtValue() >> SelBits) & 0xff) << I;
4530 } else if (SelBits == 0x0c) {
4531 Known.Zero |= 0xFFull << I;
4532 } else if (SelBits > 0x0c) {
4533 Known.One |= 0xFFull << I;
4534 }
4535 Sel >>= 8;
4536 }
4537 break;
4538 }
4539 case AMDGPUISD::BUFFER_LOAD_UBYTE: {
4540 Known.Zero.setHighBits(24);
4541 break;
4542 }
4543 case AMDGPUISD::BUFFER_LOAD_USHORT: {
4544 Known.Zero.setHighBits(16);
4545 break;
4546 }
4547 case AMDGPUISD::LDS: {
4548 auto GA = cast<GlobalAddressSDNode>(Op.getOperand(0).getNode());
4549 unsigned Align = GA->getGlobal()->getAlignment();
4550
4551 Known.Zero.setHighBits(16);
4552 if (Align)
4553 Known.Zero.setLowBits(Log2_32(Align));
4554 break;
4555 }
4556 case ISD::INTRINSIC_WO_CHAIN: {
4557 unsigned IID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4558 switch (IID) {
4559 case Intrinsic::amdgcn_mbcnt_lo:
4560 case Intrinsic::amdgcn_mbcnt_hi: {
4561 const GCNSubtarget &ST =
4562 DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
4563 // These return at most the wavefront size - 1.
4564 unsigned Size = Op.getValueType().getSizeInBits();
4565 Known.Zero.setHighBits(Size - ST.getWavefrontSizeLog2());
4566 break;
4567 }
4568 default:
4569 break;
4570 }
4571 }
4572 }
4573 }
4574
4575 unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
4576 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4577 unsigned Depth) const {
4578 switch (Op.getOpcode()) {
4579 case AMDGPUISD::BFE_I32: {
4580 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
4581 if (!Width)
4582 return 1;
4583
4584 unsigned SignBits = 32 - Width->getZExtValue() + 1;
4585 if (!isNullConstant(Op.getOperand(1)))
4586 return SignBits;
4587
4588 // TODO: Could probably figure something out with non-0 offsets.
4589 unsigned Op0SignBits = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4590 return std::max(SignBits, Op0SignBits);
4591 }
4592
4593 case AMDGPUISD::BFE_U32: {
4594 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(Op.getOperand(2));
4595 return Width ? 32 - (Width->getZExtValue() & 0x1f) : 1;
4596 }
4597
4598 case AMDGPUISD::CARRY:
4599 case AMDGPUISD::BORROW:
4600 return 31;
4601 case AMDGPUISD::BUFFER_LOAD_BYTE:
4602 return 25;
4603 case AMDGPUISD::BUFFER_LOAD_SHORT:
4604 return 17;
4605 case AMDGPUISD::BUFFER_LOAD_UBYTE:
4606 return 24;
4607 case AMDGPUISD::BUFFER_LOAD_USHORT:
4608 return 16;
4609 case AMDGPUISD::FP_TO_FP16:
4610 case AMDGPUISD::FP16_ZEXT:
4611 return 16;
4612 default:
4613 return 1;
4614 }
4615 }
4616
4617 bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
4618 const SelectionDAG &DAG,
4619 bool SNaN,
4620 unsigned Depth) const {
4621 unsigned Opcode = Op.getOpcode();
4622 switch (Opcode) {
4623 case AMDGPUISD::FMIN_LEGACY:
4624 case AMDGPUISD::FMAX_LEGACY: {
4625 if (SNaN)
4626 return true;
4627
4628 // TODO: Can check no nans on one of the operands for each one, but which
4629 // one?
4630 return false;
4631 }
4632 case AMDGPUISD::FMUL_LEGACY:
4633 case AMDGPUISD::CVT_PKRTZ_F16_F32: {
4634 if (SNaN)
4635 return true;
4636 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4637 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4638 }
4639 case AMDGPUISD::FMED3:
4640 case AMDGPUISD::FMIN3:
4641 case AMDGPUISD::FMAX3:
4642 case AMDGPUISD::FMAD_FTZ: {
4643 if (SNaN)
4644 return true;
4645 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4646 DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4647 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4648 }
4649 case AMDGPUISD::CVT_F32_UBYTE0:
4650 case AMDGPUISD::CVT_F32_UBYTE1:
4651 case AMDGPUISD::CVT_F32_UBYTE2:
4652 case AMDGPUISD::CVT_F32_UBYTE3:
4653 return true;
4654
4655 case AMDGPUISD::RCP:
4656 case AMDGPUISD::RSQ:
4657 case AMDGPUISD::RCP_LEGACY:
4658 case AMDGPUISD::RSQ_LEGACY:
4659 case AMDGPUISD::RSQ_CLAMP: {
4660 if (SNaN)
4661 return true;
4662
4663 // TODO: Need is known positive check.
4664 return false;
4665 }
4666 case AMDGPUISD::LDEXP:
4667 case AMDGPUISD::FRACT: {
4668 if (SNaN)
4669 return true;
4670 return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4671 }
4672 case AMDGPUISD::DIV_SCALE:
4673 case AMDGPUISD::DIV_FMAS:
4674 case AMDGPUISD::DIV_FIXUP:
4675 case AMDGPUISD::TRIG_PREOP:
4676 // TODO: Refine on operands.
4677 return SNaN;
4678 case AMDGPUISD::SIN_HW:
4679 case AMDGPUISD::COS_HW: {
4680 // TODO: Need check for infinity
4681 return SNaN;
4682 }
4683 case ISD::INTRINSIC_WO_CHAIN: {
4684 unsigned IntrinsicID
4685 = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4686 // TODO: Handle more intrinsics
4687 switch (IntrinsicID) {
4688 case Intrinsic::amdgcn_cubeid:
4689 return true;
4690
4691 case Intrinsic::amdgcn_frexp_mant: {
4692 if (SNaN)
4693 return true;
4694 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4695 }
4696 case Intrinsic::amdgcn_cvt_pkrtz: {
4697 if (SNaN)
4698 return true;
4699 return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4700 DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
4701 }
4702 case Intrinsic::amdgcn_fdot2:
4703 // TODO: Refine on operand
4704 return SNaN;
4705 default:
4706 return false;
4707 }
4708 }
4709 default:
4710 return false;
4711 }
4712 }
4713
4714 TargetLowering::AtomicExpansionKind
4715 AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
4716 switch (RMW->getOperation()) {
4717 case AtomicRMWInst::Nand:
4718 case AtomicRMWInst::FAdd:
4719 case AtomicRMWInst::FSub:
4720 return AtomicExpansionKind::CmpXChg;
4721 default:
4722 return AtomicExpansionKind::None;
4723 }
4724 }
The low level virtual machine