?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[Codegen] Alter the default promotion for saturating adds and subs
[llvm-complete.git] / lib / Target / AMDGPU / AMDGPUISelDAGToDAG.cpp
blob8ab8d16ed91ac6dc696188e6d1f0d10133b26bcb
1 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //==-----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines an instruction selector for the AMDGPU target.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "AMDGPU.h"
15 #include "AMDGPUArgumentUsageInfo.h"
16 #include "AMDGPUISelLowering.h" // For AMDGPUISD
17 #include "AMDGPUInstrInfo.h"
18 #include "AMDGPUPerfHintAnalysis.h"
19 #include "AMDGPURegisterInfo.h"
20 #include "AMDGPUSubtarget.h"
21 #include "AMDGPUTargetMachine.h"
22 #include "SIDefines.h"
23 #include "SIISelLowering.h"
24 #include "SIInstrInfo.h"
25 #include "SIMachineFunctionInfo.h"
26 #include "SIRegisterInfo.h"
27 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
28 #include "llvm/ADT/APInt.h"
29 #include "llvm/ADT/SmallVector.h"
30 #include "llvm/ADT/StringRef.h"
31 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
32 #include "llvm/Analysis/ValueTracking.h"
33 #include "llvm/CodeGen/FunctionLoweringInfo.h"
34 #include "llvm/CodeGen/ISDOpcodes.h"
35 #include "llvm/CodeGen/MachineFunction.h"
36 #include "llvm/CodeGen/MachineRegisterInfo.h"
37 #include "llvm/CodeGen/SelectionDAG.h"
38 #include "llvm/CodeGen/SelectionDAGISel.h"
39 #include "llvm/CodeGen/SelectionDAGNodes.h"
40 #include "llvm/CodeGen/ValueTypes.h"
41 #include "llvm/IR/BasicBlock.h"
42 #ifdef EXPENSIVE_CHECKS
43 #include "llvm/IR/Dominators.h"
44 #endif
45 #include "llvm/IR/Instruction.h"
46 #include "llvm/MC/MCInstrDesc.h"
47 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/CodeGen.h"
49 #include "llvm/Support/ErrorHandling.h"
50 #include "llvm/Support/MachineValueType.h"
51 #include "llvm/Support/MathExtras.h"
52 #include <cassert>
53 #include <cstdint>
54 #include <new>
55 #include <vector>
56
57 #define DEBUG_TYPE "isel"
58
59 using namespace llvm;
60
61 namespace llvm {
62
63 class R600InstrInfo;
64
65 } // end namespace llvm
66
67 //===----------------------------------------------------------------------===//
68 // Instruction Selector Implementation
69 //===----------------------------------------------------------------------===//
70
71 namespace {
72
73 static bool isNullConstantOrUndef(SDValue V) {
74 if (V.isUndef())
75 return true;
76
77 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
78 return Const != nullptr && Const->isNullValue();
79 }
80
81 static bool getConstantValue(SDValue N, uint32_t &Out) {
82 // This is only used for packed vectors, where ussing 0 for undef should
83 // always be good.
84 if (N.isUndef()) {
85 Out = 0;
86 return true;
87 }
88
89 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
90 Out = C->getAPIntValue().getSExtValue();
91 return true;
92 }
93
94 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
95 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
96 return true;
97 }
98
99 return false;
100 }
101
102 // TODO: Handle undef as zero
103 static SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG,
104 bool Negate = false) {
105 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
106 uint32_t LHSVal, RHSVal;
107 if (getConstantValue(N->getOperand(0), LHSVal) &&
108 getConstantValue(N->getOperand(1), RHSVal)) {
109 SDLoc SL(N);
110 uint32_t K = Negate ?
111 (-LHSVal & 0xffff) | (-RHSVal << 16) :
112 (LHSVal & 0xffff) | (RHSVal << 16);
113 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
114 DAG.getTargetConstant(K, SL, MVT::i32));
115 }
116
117 return nullptr;
118 }
119
120 static SDNode *packNegConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
121 return packConstantV2I16(N, DAG, true);
122 }
123
124 /// AMDGPU specific code to select AMDGPU machine instructions for
125 /// SelectionDAG operations.
126 class AMDGPUDAGToDAGISel : public SelectionDAGISel {
127 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
128 // make the right decision when generating code for different targets.
129 const GCNSubtarget *Subtarget;
130 bool EnableLateStructurizeCFG;
131
132 public:
133 explicit AMDGPUDAGToDAGISel(TargetMachine *TM = nullptr,
134 CodeGenOpt::Level OptLevel = CodeGenOpt::Default)
135 : SelectionDAGISel(*TM, OptLevel) {
136 EnableLateStructurizeCFG = AMDGPUTargetMachine::EnableLateStructurizeCFG;
137 }
138 ~AMDGPUDAGToDAGISel() override = default;
139
140 void getAnalysisUsage(AnalysisUsage &AU) const override {
141 AU.addRequired<AMDGPUArgumentUsageInfo>();
142 AU.addRequired<LegacyDivergenceAnalysis>();
143 #ifdef EXPENSIVE_CHECKS
144 AU.addRequired<DominatorTreeWrapperPass>();
145 AU.addRequired<LoopInfoWrapperPass>();
146 #endif
147 SelectionDAGISel::getAnalysisUsage(AU);
148 }
149
150 bool matchLoadD16FromBuildVector(SDNode *N) const;
151
152 bool runOnMachineFunction(MachineFunction &MF) override;
153 void PreprocessISelDAG() override;
154 void Select(SDNode *N) override;
155 StringRef getPassName() const override;
156 void PostprocessISelDAG() override;
157
158 protected:
159 void SelectBuildVector(SDNode *N, unsigned RegClassID);
160
161 private:
162 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
163 bool isNoNanSrc(SDValue N) const;
164 bool isInlineImmediate(const SDNode *N, bool Negated = false) const;
165 bool isNegInlineImmediate(const SDNode *N) const {
166 return isInlineImmediate(N, true);
167 }
168
169 bool isVGPRImm(const SDNode *N) const;
170 bool isUniformLoad(const SDNode *N) const;
171 bool isUniformBr(const SDNode *N) const;
172
173 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
174
175 SDNode *glueCopyToM0LDSInit(SDNode *N) const;
176 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
177
178 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
179 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
180 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
181 bool isDSOffsetLegal(SDValue Base, unsigned Offset,
182 unsigned OffsetBits) const;
183 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
184 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
185 SDValue &Offset1) const;
186 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
187 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
188 SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
189 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
190 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
191 SDValue &SOffset, SDValue &Offset, SDValue &GLC,
192 SDValue &SLC, SDValue &TFE, SDValue &DLC,
193 SDValue &SWZ) const;
194 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
195 SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
196 SDValue &SLC) const;
197 bool SelectMUBUFScratchOffen(SDNode *Parent,
198 SDValue Addr, SDValue &RSrc, SDValue &VAddr,
199 SDValue &SOffset, SDValue &ImmOffset) const;
200 bool SelectMUBUFScratchOffset(SDNode *Parent,
201 SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
202 SDValue &Offset) const;
203
204 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
205 SDValue &Offset, SDValue &GLC, SDValue &SLC,
206 SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
207 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
208 SDValue &Offset, SDValue &SLC) const;
209 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
210 SDValue &Offset) const;
211
212 bool SelectFlatAtomic(SDNode *N, SDValue Addr, SDValue &VAddr,
213 SDValue &Offset, SDValue &SLC) const;
214 bool SelectFlatAtomicSigned(SDNode *N, SDValue Addr, SDValue &VAddr,
215 SDValue &Offset, SDValue &SLC) const;
216
217 template <bool IsSigned>
218 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
219 SDValue &Offset, SDValue &SLC) const;
220
221 bool SelectSMRDOffset(SDValue ByteOffsetNode, SDValue &Offset,
222 bool &Imm) const;
223 SDValue Expand32BitAddress(SDValue Addr) const;
224 bool SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset,
225 bool &Imm) const;
226 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
227 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
228 bool SelectSMRDSgpr(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
229 bool SelectSMRDBufferImm(SDValue Addr, SDValue &Offset) const;
230 bool SelectSMRDBufferImm32(SDValue Addr, SDValue &Offset) const;
231 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
232
233 bool SelectVOP3Mods_NNaN(SDValue In, SDValue &Src, SDValue &SrcMods) const;
234 bool SelectVOP3Mods_f32(SDValue In, SDValue &Src, SDValue &SrcMods) const;
235 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods) const;
236 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
237 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
238 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
239 SDValue &Clamp, SDValue &Omod) const;
240 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
241 SDValue &Clamp, SDValue &Omod) const;
242
243 bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods,
244 SDValue &Clamp,
245 SDValue &Omod) const;
246
247 bool SelectVOP3OMods(SDValue In, SDValue &Src,
248 SDValue &Clamp, SDValue &Omod) const;
249
250 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
251 bool SelectVOP3PMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
252 SDValue &Clamp) const;
253
254 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
255 bool SelectVOP3OpSel0(SDValue In, SDValue &Src, SDValue &SrcMods,
256 SDValue &Clamp) const;
257
258 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
259 bool SelectVOP3OpSelMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
260 SDValue &Clamp) const;
261 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods) const;
262 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
263
264 SDValue getHi16Elt(SDValue In) const;
265
266 void SelectADD_SUB_I64(SDNode *N);
267 void SelectAddcSubb(SDNode *N);
268 void SelectUADDO_USUBO(SDNode *N);
269 void SelectDIV_SCALE(SDNode *N);
270 void SelectDIV_FMAS(SDNode *N);
271 void SelectMAD_64_32(SDNode *N);
272 void SelectFMA_W_CHAIN(SDNode *N);
273 void SelectFMUL_W_CHAIN(SDNode *N);
274
275 SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val,
276 uint32_t Offset, uint32_t Width);
277 void SelectS_BFEFromShifts(SDNode *N);
278 void SelectS_BFE(SDNode *N);
279 bool isCBranchSCC(const SDNode *N) const;
280 void SelectBRCOND(SDNode *N);
281 void SelectFMAD_FMA(SDNode *N);
282 void SelectATOMIC_CMP_SWAP(SDNode *N);
283 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
284 void SelectDS_GWS(SDNode *N, unsigned IntrID);
285 void SelectINTRINSIC_W_CHAIN(SDNode *N);
286 void SelectINTRINSIC_WO_CHAIN(SDNode *N);
287 void SelectINTRINSIC_VOID(SDNode *N);
288
289 protected:
290 // Include the pieces autogenerated from the target description.
291 #include "AMDGPUGenDAGISel.inc"
292 };
293
294 class R600DAGToDAGISel : public AMDGPUDAGToDAGISel {
295 const R600Subtarget *Subtarget;
296
297 bool isConstantLoad(const MemSDNode *N, int cbID) const;
298 bool SelectGlobalValueConstantOffset(SDValue Addr, SDValue& IntPtr);
299 bool SelectGlobalValueVariableOffset(SDValue Addr, SDValue &BaseReg,
300 SDValue& Offset);
301 public:
302 explicit R600DAGToDAGISel(TargetMachine *TM, CodeGenOpt::Level OptLevel) :
303 AMDGPUDAGToDAGISel(TM, OptLevel) {}
304
305 void Select(SDNode *N) override;
306
307 bool SelectADDRIndirect(SDValue Addr, SDValue &Base,
308 SDValue &Offset) override;
309 bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
310 SDValue &Offset) override;
311
312 bool runOnMachineFunction(MachineFunction &MF) override;
313
314 void PreprocessISelDAG() override {}
315
316 protected:
317 // Include the pieces autogenerated from the target description.
318 #include "R600GenDAGISel.inc"
319 };
320
321 static SDValue stripBitcast(SDValue Val) {
322 return Val.getOpcode() == ISD::BITCAST ? Val.getOperand(0) : Val;
323 }
324
325 // Figure out if this is really an extract of the high 16-bits of a dword.
326 static bool isExtractHiElt(SDValue In, SDValue &Out) {
327 In = stripBitcast(In);
328 if (In.getOpcode() != ISD::TRUNCATE)
329 return false;
330
331 SDValue Srl = In.getOperand(0);
332 if (Srl.getOpcode() == ISD::SRL) {
333 if (ConstantSDNode *ShiftAmt = dyn_cast<ConstantSDNode>(Srl.getOperand(1))) {
334 if (ShiftAmt->getZExtValue() == 16) {
335 Out = stripBitcast(Srl.getOperand(0));
336 return true;
337 }
338 }
339 }
340
341 return false;
342 }
343
344 // Look through operations that obscure just looking at the low 16-bits of the
345 // same register.
346 static SDValue stripExtractLoElt(SDValue In) {
347 if (In.getOpcode() == ISD::TRUNCATE) {
348 SDValue Src = In.getOperand(0);
349 if (Src.getValueType().getSizeInBits() == 32)
350 return stripBitcast(Src);
351 }
352
353 return In;
354 }
355
356 } // end anonymous namespace
357
358 INITIALIZE_PASS_BEGIN(AMDGPUDAGToDAGISel, "amdgpu-isel",
359 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
360 INITIALIZE_PASS_DEPENDENCY(AMDGPUArgumentUsageInfo)
361 INITIALIZE_PASS_DEPENDENCY(AMDGPUPerfHintAnalysis)
362 INITIALIZE_PASS_DEPENDENCY(LegacyDivergenceAnalysis)
363 #ifdef EXPENSIVE_CHECKS
364 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
365 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
366 #endif
367 INITIALIZE_PASS_END(AMDGPUDAGToDAGISel, "amdgpu-isel",
368 "AMDGPU DAG->DAG Pattern Instruction Selection", false, false)
369
370 /// This pass converts a legalized DAG into a AMDGPU-specific
371 // DAG, ready for instruction scheduling.
372 FunctionPass *llvm::createAMDGPUISelDag(TargetMachine *TM,
373 CodeGenOpt::Level OptLevel) {
374 return new AMDGPUDAGToDAGISel(TM, OptLevel);
375 }
376
377 /// This pass converts a legalized DAG into a R600-specific
378 // DAG, ready for instruction scheduling.
379 FunctionPass *llvm::createR600ISelDag(TargetMachine *TM,
380 CodeGenOpt::Level OptLevel) {
381 return new R600DAGToDAGISel(TM, OptLevel);
382 }
383
384 bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
385 #ifdef EXPENSIVE_CHECKS
386 DominatorTree & DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
387 LoopInfo * LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
388 for (auto &L : LI->getLoopsInPreorder()) {
389 assert(L->isLCSSAForm(DT));
390 }
391 #endif
392 Subtarget = &MF.getSubtarget<GCNSubtarget>();
393 return SelectionDAGISel::runOnMachineFunction(MF);
394 }
395
396 bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
397 assert(Subtarget->d16PreservesUnusedBits());
398 MVT VT = N->getValueType(0).getSimpleVT();
399 if (VT != MVT::v2i16 && VT != MVT::v2f16)
400 return false;
401
402 SDValue Lo = N->getOperand(0);
403 SDValue Hi = N->getOperand(1);
404
405 LoadSDNode *LdHi = dyn_cast<LoadSDNode>(stripBitcast(Hi));
406
407 // build_vector lo, (load ptr) -> load_d16_hi ptr, lo
408 // build_vector lo, (zextload ptr from i8) -> load_d16_hi_u8 ptr, lo
409 // build_vector lo, (sextload ptr from i8) -> load_d16_hi_i8 ptr, lo
410
411 // Need to check for possible indirect dependencies on the other half of the
412 // vector to avoid introducing a cycle.
413 if (LdHi && Hi.hasOneUse() && !LdHi->isPredecessorOf(Lo.getNode())) {
414 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
415
416 SDValue TiedIn = CurDAG->getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Lo);
417 SDValue Ops[] = {
418 LdHi->getChain(), LdHi->getBasePtr(), TiedIn
419 };
420
421 unsigned LoadOp = AMDGPUISD::LOAD_D16_HI;
422 if (LdHi->getMemoryVT() == MVT::i8) {
423 LoadOp = LdHi->getExtensionType() == ISD::SEXTLOAD ?
424 AMDGPUISD::LOAD_D16_HI_I8 : AMDGPUISD::LOAD_D16_HI_U8;
425 } else {
426 assert(LdHi->getMemoryVT() == MVT::i16);
427 }
428
429 SDValue NewLoadHi =
430 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdHi), VTList,
431 Ops, LdHi->getMemoryVT(),
432 LdHi->getMemOperand());
433
434 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadHi);
435 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdHi, 1), NewLoadHi.getValue(1));
436 return true;
437 }
438
439 // build_vector (load ptr), hi -> load_d16_lo ptr, hi
440 // build_vector (zextload ptr from i8), hi -> load_d16_lo_u8 ptr, hi
441 // build_vector (sextload ptr from i8), hi -> load_d16_lo_i8 ptr, hi
442 LoadSDNode *LdLo = dyn_cast<LoadSDNode>(stripBitcast(Lo));
443 if (LdLo && Lo.hasOneUse()) {
444 SDValue TiedIn = getHi16Elt(Hi);
445 if (!TiedIn || LdLo->isPredecessorOf(TiedIn.getNode()))
446 return false;
447
448 SDVTList VTList = CurDAG->getVTList(VT, MVT::Other);
449 unsigned LoadOp = AMDGPUISD::LOAD_D16_LO;
450 if (LdLo->getMemoryVT() == MVT::i8) {
451 LoadOp = LdLo->getExtensionType() == ISD::SEXTLOAD ?
452 AMDGPUISD::LOAD_D16_LO_I8 : AMDGPUISD::LOAD_D16_LO_U8;
453 } else {
454 assert(LdLo->getMemoryVT() == MVT::i16);
455 }
456
457 TiedIn = CurDAG->getNode(ISD::BITCAST, SDLoc(N), VT, TiedIn);
458
459 SDValue Ops[] = {
460 LdLo->getChain(), LdLo->getBasePtr(), TiedIn
461 };
462
463 SDValue NewLoadLo =
464 CurDAG->getMemIntrinsicNode(LoadOp, SDLoc(LdLo), VTList,
465 Ops, LdLo->getMemoryVT(),
466 LdLo->getMemOperand());
467
468 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), NewLoadLo);
469 CurDAG->ReplaceAllUsesOfValueWith(SDValue(LdLo, 1), NewLoadLo.getValue(1));
470 return true;
471 }
472
473 return false;
474 }
475
476 void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
477 if (!Subtarget->d16PreservesUnusedBits())
478 return;
479
480 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
481
482 bool MadeChange = false;
483 while (Position != CurDAG->allnodes_begin()) {
484 SDNode *N = &*--Position;
485 if (N->use_empty())
486 continue;
487
488 switch (N->getOpcode()) {
489 case ISD::BUILD_VECTOR:
490 MadeChange |= matchLoadD16FromBuildVector(N);
491 break;
492 default:
493 break;
494 }
495 }
496
497 if (MadeChange) {
498 CurDAG->RemoveDeadNodes();
499 LLVM_DEBUG(dbgs() << "After PreProcess:\n";
500 CurDAG->dump(););
501 }
502 }
503
504 bool AMDGPUDAGToDAGISel::isNoNanSrc(SDValue N) const {
505 if (TM.Options.NoNaNsFPMath)
506 return true;
507
508 // TODO: Move into isKnownNeverNaN
509 if (N->getFlags().isDefined())
510 return N->getFlags().hasNoNaNs();
511
512 return CurDAG->isKnownNeverNaN(N);
513 }
514
515 bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N,
516 bool Negated) const {
517 if (N->isUndef())
518 return true;
519
520 const SIInstrInfo *TII = Subtarget->getInstrInfo();
521 if (Negated) {
522 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
523 return TII->isInlineConstant(-C->getAPIntValue());
524
525 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
526 return TII->isInlineConstant(-C->getValueAPF().bitcastToAPInt());
527
528 } else {
529 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N))
530 return TII->isInlineConstant(C->getAPIntValue());
531
532 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N))
533 return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt());
534 }
535
536 return false;
537 }
538
539 /// Determine the register class for \p OpNo
540 /// \returns The register class of the virtual register that will be used for
541 /// the given operand number \OpNo or NULL if the register class cannot be
542 /// determined.
543 const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
544 unsigned OpNo) const {
545 if (!N->isMachineOpcode()) {
546 if (N->getOpcode() == ISD::CopyToReg) {
547 unsigned Reg = cast<RegisterSDNode>(N->getOperand(1))->getReg();
548 if (Register::isVirtualRegister(Reg)) {
549 MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo();
550 return MRI.getRegClass(Reg);
551 }
552
553 const SIRegisterInfo *TRI
554 = static_cast<const GCNSubtarget *>(Subtarget)->getRegisterInfo();
555 return TRI->getPhysRegClass(Reg);
556 }
557
558 return nullptr;
559 }
560
561 switch (N->getMachineOpcode()) {
562 default: {
563 const MCInstrDesc &Desc =
564 Subtarget->getInstrInfo()->get(N->getMachineOpcode());
565 unsigned OpIdx = Desc.getNumDefs() + OpNo;
566 if (OpIdx >= Desc.getNumOperands())
567 return nullptr;
568 int RegClass = Desc.OpInfo[OpIdx].RegClass;
569 if (RegClass == -1)
570 return nullptr;
571
572 return Subtarget->getRegisterInfo()->getRegClass(RegClass);
573 }
574 case AMDGPU::REG_SEQUENCE: {
575 unsigned RCID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
576 const TargetRegisterClass *SuperRC =
577 Subtarget->getRegisterInfo()->getRegClass(RCID);
578
579 SDValue SubRegOp = N->getOperand(OpNo + 1);
580 unsigned SubRegIdx = cast<ConstantSDNode>(SubRegOp)->getZExtValue();
581 return Subtarget->getRegisterInfo()->getSubClassWithSubReg(SuperRC,
582 SubRegIdx);
583 }
584 }
585 }
586
587 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
588 const SITargetLowering& Lowering =
589 *static_cast<const SITargetLowering*>(getTargetLowering());
590
591 assert(N->getOperand(0).getValueType() == MVT::Other && "Expected chain");
592
593 SDValue M0 = Lowering.copyToM0(*CurDAG, N->getOperand(0), SDLoc(N),
594 Val);
595
596 SDValue Glue = M0.getValue(1);
597
598 SmallVector <SDValue, 8> Ops;
599 Ops.push_back(M0); // Replace the chain.
600 for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
601 Ops.push_back(N->getOperand(i));
602
603 Ops.push_back(Glue);
604 return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
605 }
606
607 SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
608 unsigned AS = cast<MemSDNode>(N)->getAddressSpace();
609 if (AS == AMDGPUAS::LOCAL_ADDRESS) {
610 if (Subtarget->ldsRequiresM0Init())
611 return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
612 } else if (AS == AMDGPUAS::REGION_ADDRESS) {
613 MachineFunction &MF = CurDAG->getMachineFunction();
614 unsigned Value = MF.getInfo<SIMachineFunctionInfo>()->getGDSSize();
615 return
616 glueCopyToM0(N, CurDAG->getTargetConstant(Value, SDLoc(N), MVT::i32));
617 }
618 return N;
619 }
620
621 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
622 EVT VT) const {
623 SDNode *Lo = CurDAG->getMachineNode(
624 AMDGPU::S_MOV_B32, DL, MVT::i32,
625 CurDAG->getTargetConstant(Imm & 0xFFFFFFFF, DL, MVT::i32));
626 SDNode *Hi =
627 CurDAG->getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32,
628 CurDAG->getTargetConstant(Imm >> 32, DL, MVT::i32));
629 const SDValue Ops[] = {
630 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
631 SDValue(Lo, 0), CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
632 SDValue(Hi, 0), CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
633
634 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops);
635 }
636
637 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
638 switch (NumVectorElts) {
639 case 1:
640 return AMDGPU::SReg_32_XM0RegClassID;
641 case 2:
642 return AMDGPU::SReg_64RegClassID;
643 case 3:
644 return AMDGPU::SGPR_96RegClassID;
645 case 4:
646 return AMDGPU::SGPR_128RegClassID;
647 case 5:
648 return AMDGPU::SGPR_160RegClassID;
649 case 8:
650 return AMDGPU::SReg_256RegClassID;
651 case 16:
652 return AMDGPU::SReg_512RegClassID;
653 case 32:
654 return AMDGPU::SReg_1024RegClassID;
655 }
656
657 llvm_unreachable("invalid vector size");
658 }
659
660 void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
661 EVT VT = N->getValueType(0);
662 unsigned NumVectorElts = VT.getVectorNumElements();
663 EVT EltVT = VT.getVectorElementType();
664 SDLoc DL(N);
665 SDValue RegClass = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
666
667 if (NumVectorElts == 1) {
668 CurDAG->SelectNodeTo(N, AMDGPU::COPY_TO_REGCLASS, EltVT, N->getOperand(0),
669 RegClass);
670 return;
671 }
672
673 assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
674 "supported yet");
675 // 32 = Max Num Vector Elements
676 // 2 = 2 REG_SEQUENCE operands per element (value, subreg index)
677 // 1 = Vector Register Class
678 SmallVector<SDValue, 32 * 2 + 1> RegSeqArgs(NumVectorElts * 2 + 1);
679
680 RegSeqArgs[0] = CurDAG->getTargetConstant(RegClassID, DL, MVT::i32);
681 bool IsRegSeq = true;
682 unsigned NOps = N->getNumOperands();
683 for (unsigned i = 0; i < NOps; i++) {
684 // XXX: Why is this here?
685 if (isa<RegisterSDNode>(N->getOperand(i))) {
686 IsRegSeq = false;
687 break;
688 }
689 unsigned Sub = AMDGPURegisterInfo::getSubRegFromChannel(i);
690 RegSeqArgs[1 + (2 * i)] = N->getOperand(i);
691 RegSeqArgs[1 + (2 * i) + 1] = CurDAG->getTargetConstant(Sub, DL, MVT::i32);
692 }
693 if (NOps != NumVectorElts) {
694 // Fill in the missing undef elements if this was a scalar_to_vector.
695 assert(N->getOpcode() == ISD::SCALAR_TO_VECTOR && NOps < NumVectorElts);
696 MachineSDNode *ImpDef = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
697 DL, EltVT);
698 for (unsigned i = NOps; i < NumVectorElts; ++i) {
699 unsigned Sub = AMDGPURegisterInfo::getSubRegFromChannel(i);
700 RegSeqArgs[1 + (2 * i)] = SDValue(ImpDef, 0);
701 RegSeqArgs[1 + (2 * i) + 1] =
702 CurDAG->getTargetConstant(Sub, DL, MVT::i32);
703 }
704 }
705
706 if (!IsRegSeq)
707 SelectCode(N);
708 CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), RegSeqArgs);
709 }
710
711 void AMDGPUDAGToDAGISel::Select(SDNode *N) {
712 unsigned int Opc = N->getOpcode();
713 if (N->isMachineOpcode()) {
714 N->setNodeId(-1);
715 return; // Already selected.
716 }
717
718 if (isa<AtomicSDNode>(N) ||
719 (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
720 Opc == ISD::ATOMIC_LOAD_FADD ||
721 Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
722 Opc == AMDGPUISD::ATOMIC_LOAD_FMAX))
723 N = glueCopyToM0LDSInit(N);
724
725 switch (Opc) {
726 default:
727 break;
728 // We are selecting i64 ADD here instead of custom lower it during
729 // DAG legalization, so we can fold some i64 ADDs used for address
730 // calculation into the LOAD and STORE instructions.
731 case ISD::ADDC:
732 case ISD::ADDE:
733 case ISD::SUBC:
734 case ISD::SUBE: {
735 if (N->getValueType(0) != MVT::i64)
736 break;
737
738 SelectADD_SUB_I64(N);
739 return;
740 }
741 case ISD::ADDCARRY:
742 case ISD::SUBCARRY:
743 if (N->getValueType(0) != MVT::i32)
744 break;
745
746 SelectAddcSubb(N);
747 return;
748 case ISD::UADDO:
749 case ISD::USUBO: {
750 SelectUADDO_USUBO(N);
751 return;
752 }
753 case AMDGPUISD::FMUL_W_CHAIN: {
754 SelectFMUL_W_CHAIN(N);
755 return;
756 }
757 case AMDGPUISD::FMA_W_CHAIN: {
758 SelectFMA_W_CHAIN(N);
759 return;
760 }
761
762 case ISD::SCALAR_TO_VECTOR:
763 case ISD::BUILD_VECTOR: {
764 EVT VT = N->getValueType(0);
765 unsigned NumVectorElts = VT.getVectorNumElements();
766 if (VT.getScalarSizeInBits() == 16) {
767 if (Opc == ISD::BUILD_VECTOR && NumVectorElts == 2) {
768 if (SDNode *Packed = packConstantV2I16(N, *CurDAG)) {
769 ReplaceNode(N, Packed);
770 return;
771 }
772 }
773
774 break;
775 }
776
777 assert(VT.getVectorElementType().bitsEq(MVT::i32));
778 unsigned RegClassID = selectSGPRVectorRegClassID(NumVectorElts);
779 SelectBuildVector(N, RegClassID);
780 return;
781 }
782 case ISD::BUILD_PAIR: {
783 SDValue RC, SubReg0, SubReg1;
784 SDLoc DL(N);
785 if (N->getValueType(0) == MVT::i128) {
786 RC = CurDAG->getTargetConstant(AMDGPU::SGPR_128RegClassID, DL, MVT::i32);
787 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, DL, MVT::i32);
788 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, DL, MVT::i32);
789 } else if (N->getValueType(0) == MVT::i64) {
790 RC = CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32);
791 SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
792 SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
793 } else {
794 llvm_unreachable("Unhandled value type for BUILD_PAIR");
795 }
796 const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
797 N->getOperand(1), SubReg1 };
798 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL,
799 N->getValueType(0), Ops));
800 return;
801 }
802
803 case ISD::Constant:
804 case ISD::ConstantFP: {
805 if (N->getValueType(0).getSizeInBits() != 64 || isInlineImmediate(N))
806 break;
807
808 uint64_t Imm;
809 if (ConstantFPSDNode *FP = dyn_cast<ConstantFPSDNode>(N))
810 Imm = FP->getValueAPF().bitcastToAPInt().getZExtValue();
811 else {
812 ConstantSDNode *C = cast<ConstantSDNode>(N);
813 Imm = C->getZExtValue();
814 }
815
816 SDLoc DL(N);
817 ReplaceNode(N, buildSMovImm64(DL, Imm, N->getValueType(0)));
818 return;
819 }
820 case ISD::LOAD:
821 case ISD::STORE:
822 case ISD::ATOMIC_LOAD:
823 case ISD::ATOMIC_STORE: {
824 N = glueCopyToM0LDSInit(N);
825 break;
826 }
827
828 case AMDGPUISD::BFE_I32:
829 case AMDGPUISD::BFE_U32: {
830 // There is a scalar version available, but unlike the vector version which
831 // has a separate operand for the offset and width, the scalar version packs
832 // the width and offset into a single operand. Try to move to the scalar
833 // version if the offsets are constant, so that we can try to keep extended
834 // loads of kernel arguments in SGPRs.
835
836 // TODO: Technically we could try to pattern match scalar bitshifts of
837 // dynamic values, but it's probably not useful.
838 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
839 if (!Offset)
840 break;
841
842 ConstantSDNode *Width = dyn_cast<ConstantSDNode>(N->getOperand(2));
843 if (!Width)
844 break;
845
846 bool Signed = Opc == AMDGPUISD::BFE_I32;
847
848 uint32_t OffsetVal = Offset->getZExtValue();
849 uint32_t WidthVal = Width->getZExtValue();
850
851 ReplaceNode(N, getS_BFE(Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32,
852 SDLoc(N), N->getOperand(0), OffsetVal, WidthVal));
853 return;
854 }
855 case AMDGPUISD::DIV_SCALE: {
856 SelectDIV_SCALE(N);
857 return;
858 }
859 case AMDGPUISD::DIV_FMAS: {
860 SelectDIV_FMAS(N);
861 return;
862 }
863 case AMDGPUISD::MAD_I64_I32:
864 case AMDGPUISD::MAD_U64_U32: {
865 SelectMAD_64_32(N);
866 return;
867 }
868 case ISD::CopyToReg: {
869 const SITargetLowering& Lowering =
870 *static_cast<const SITargetLowering*>(getTargetLowering());
871 N = Lowering.legalizeTargetIndependentNode(N, *CurDAG);
872 break;
873 }
874 case ISD::AND:
875 case ISD::SRL:
876 case ISD::SRA:
877 case ISD::SIGN_EXTEND_INREG:
878 if (N->getValueType(0) != MVT::i32)
879 break;
880
881 SelectS_BFE(N);
882 return;
883 case ISD::BRCOND:
884 SelectBRCOND(N);
885 return;
886 case ISD::FMAD:
887 case ISD::FMA:
888 SelectFMAD_FMA(N);
889 return;
890 case AMDGPUISD::ATOMIC_CMP_SWAP:
891 SelectATOMIC_CMP_SWAP(N);
892 return;
893 case AMDGPUISD::CVT_PKRTZ_F16_F32:
894 case AMDGPUISD::CVT_PKNORM_I16_F32:
895 case AMDGPUISD::CVT_PKNORM_U16_F32:
896 case AMDGPUISD::CVT_PK_U16_U32:
897 case AMDGPUISD::CVT_PK_I16_I32: {
898 // Hack around using a legal type if f16 is illegal.
899 if (N->getValueType(0) == MVT::i32) {
900 MVT NewVT = Opc == AMDGPUISD::CVT_PKRTZ_F16_F32 ? MVT::v2f16 : MVT::v2i16;
901 N = CurDAG->MorphNodeTo(N, N->getOpcode(), CurDAG->getVTList(NewVT),
902 { N->getOperand(0), N->getOperand(1) });
903 SelectCode(N);
904 return;
905 }
906
907 break;
908 }
909 case ISD::INTRINSIC_W_CHAIN: {
910 SelectINTRINSIC_W_CHAIN(N);
911 return;
912 }
913 case ISD::INTRINSIC_WO_CHAIN: {
914 SelectINTRINSIC_WO_CHAIN(N);
915 return;
916 }
917 case ISD::INTRINSIC_VOID: {
918 SelectINTRINSIC_VOID(N);
919 return;
920 }
921 }
922
923 SelectCode(N);
924 }
925
926 bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const {
927 const BasicBlock *BB = FuncInfo->MBB->getBasicBlock();
928 const Instruction *Term = BB->getTerminator();
929 return Term->getMetadata("amdgpu.uniform") ||
930 Term->getMetadata("structurizecfg.uniform");
931 }
932
933 StringRef AMDGPUDAGToDAGISel::getPassName() const {
934 return "AMDGPU DAG->DAG Pattern Instruction Selection";
935 }
936
937 //===----------------------------------------------------------------------===//
938 // Complex Patterns
939 //===----------------------------------------------------------------------===//
940
941 bool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
942 SDValue &Offset) {
943 return false;
944 }
945
946 bool AMDGPUDAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
947 SDValue &Offset) {
948 ConstantSDNode *C;
949 SDLoc DL(Addr);
950
951 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
952 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
953 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
954 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
955 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
956 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
957 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
958 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
959 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
960 Base = Addr.getOperand(0);
961 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
962 } else {
963 Base = Addr;
964 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
965 }
966
967 return true;
968 }
969
970 // FIXME: Should only handle addcarry/subcarry
971 void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) {
972 SDLoc DL(N);
973 SDValue LHS = N->getOperand(0);
974 SDValue RHS = N->getOperand(1);
975
976 unsigned Opcode = N->getOpcode();
977 bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE);
978 bool ProduceCarry =
979 ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC;
980 bool IsAdd = Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE;
981
982 SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32);
983 SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32);
984
985 SDNode *Lo0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
986 DL, MVT::i32, LHS, Sub0);
987 SDNode *Hi0 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
988 DL, MVT::i32, LHS, Sub1);
989
990 SDNode *Lo1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
991 DL, MVT::i32, RHS, Sub0);
992 SDNode *Hi1 = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
993 DL, MVT::i32, RHS, Sub1);
994
995 SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue);
996
997 unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
998 unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
999
1000 SDNode *AddLo;
1001 if (!ConsumeCarry) {
1002 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) };
1003 AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args);
1004 } else {
1005 SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) };
1006 AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args);
1007 }
1008 SDValue AddHiArgs[] = {
1009 SDValue(Hi0, 0),
1010 SDValue(Hi1, 0),
1011 SDValue(AddLo, 1)
1012 };
1013 SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs);
1014
1015 SDValue RegSequenceArgs[] = {
1016 CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32),
1017 SDValue(AddLo,0),
1018 Sub0,
1019 SDValue(AddHi,0),
1020 Sub1,
1021 };
1022 SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL,
1023 MVT::i64, RegSequenceArgs);
1024
1025 if (ProduceCarry) {
1026 // Replace the carry-use
1027 ReplaceUses(SDValue(N, 1), SDValue(AddHi, 1));
1028 }
1029
1030 // Replace the remaining uses.
1031 ReplaceNode(N, RegSequence);
1032 }
1033
1034 void AMDGPUDAGToDAGISel::SelectAddcSubb(SDNode *N) {
1035 SDLoc DL(N);
1036 SDValue LHS = N->getOperand(0);
1037 SDValue RHS = N->getOperand(1);
1038 SDValue CI = N->getOperand(2);
1039
1040 unsigned Opc = N->getOpcode() == ISD::ADDCARRY ? AMDGPU::V_ADDC_U32_e64
1041 : AMDGPU::V_SUBB_U32_e64;
1042 CurDAG->SelectNodeTo(
1043 N, Opc, N->getVTList(),
1044 {LHS, RHS, CI, CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1045 }
1046
1047 void AMDGPUDAGToDAGISel::SelectUADDO_USUBO(SDNode *N) {
1048 // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
1049 // carry out despite the _i32 name. These were renamed in VI to _U32.
1050 // FIXME: We should probably rename the opcodes here.
1051 unsigned Opc = N->getOpcode() == ISD::UADDO ?
1052 AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
1053
1054 CurDAG->SelectNodeTo(
1055 N, Opc, N->getVTList(),
1056 {N->getOperand(0), N->getOperand(1),
1057 CurDAG->getTargetConstant(0, {}, MVT::i1) /*clamp bit*/});
1058 }
1059
1060 void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) {
1061 SDLoc SL(N);
1062 // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod
1063 SDValue Ops[10];
1064
1065 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]);
1066 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1067 SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]);
1068 Ops[8] = N->getOperand(0);
1069 Ops[9] = N->getOperand(4);
1070
1071 CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops);
1072 }
1073
1074 void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) {
1075 SDLoc SL(N);
1076 // src0_modifiers, src0, src1_modifiers, src1, clamp, omod
1077 SDValue Ops[8];
1078
1079 SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]);
1080 SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]);
1081 Ops[6] = N->getOperand(0);
1082 Ops[7] = N->getOperand(3);
1083
1084 CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops);
1085 }
1086
1087 // We need to handle this here because tablegen doesn't support matching
1088 // instructions with multiple outputs.
1089 void AMDGPUDAGToDAGISel::SelectDIV_SCALE(SDNode *N) {
1090 SDLoc SL(N);
1091 EVT VT = N->getValueType(0);
1092
1093 assert(VT == MVT::f32 || VT == MVT::f64);
1094
1095 unsigned Opc
1096 = (VT == MVT::f64) ? AMDGPU::V_DIV_SCALE_F64 : AMDGPU::V_DIV_SCALE_F32;
1097
1098 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2) };
1099 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1100 }
1101
1102 void AMDGPUDAGToDAGISel::SelectDIV_FMAS(SDNode *N) {
1103 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
1104 const SIRegisterInfo *TRI = ST->getRegisterInfo();
1105
1106 SDLoc SL(N);
1107 EVT VT = N->getValueType(0);
1108
1109 assert(VT == MVT::f32 || VT == MVT::f64);
1110
1111 unsigned Opc
1112 = (VT == MVT::f64) ? AMDGPU::V_DIV_FMAS_F64 : AMDGPU::V_DIV_FMAS_F32;
1113
1114 SDValue CarryIn = N->getOperand(3);
1115 // V_DIV_FMAS implicitly reads VCC.
1116 SDValue VCC = CurDAG->getCopyToReg(CurDAG->getEntryNode(), SL,
1117 TRI->getVCC(), CarryIn, SDValue());
1118
1119 SDValue Ops[10];
1120
1121 SelectVOP3Mods0(N->getOperand(0), Ops[1], Ops[0], Ops[6], Ops[7]);
1122 SelectVOP3Mods(N->getOperand(1), Ops[3], Ops[2]);
1123 SelectVOP3Mods(N->getOperand(2), Ops[5], Ops[4]);
1124
1125 Ops[8] = VCC;
1126 Ops[9] = VCC.getValue(1);
1127
1128 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1129 }
1130
1131 // We need to handle this here because tablegen doesn't support matching
1132 // instructions with multiple outputs.
1133 void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
1134 SDLoc SL(N);
1135 bool Signed = N->getOpcode() == AMDGPUISD::MAD_I64_I32;
1136 unsigned Opc = Signed ? AMDGPU::V_MAD_I64_I32 : AMDGPU::V_MAD_U64_U32;
1137
1138 SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
1139 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
1140 Clamp };
1141 CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
1142 }
1143
1144 bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset,
1145 unsigned OffsetBits) const {
1146 if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
1147 (OffsetBits == 8 && !isUInt<8>(Offset)))
1148 return false;
1149
1150 if (Subtarget->hasUsableDSOffset() ||
1151 Subtarget->unsafeDSOffsetFoldingEnabled())
1152 return true;
1153
1154 // On Southern Islands instruction with a negative base value and an offset
1155 // don't seem to work.
1156 return CurDAG->SignBitIsZero(Base);
1157 }
1158
1159 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
1160 SDValue &Offset) const {
1161 SDLoc DL(Addr);
1162 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1163 SDValue N0 = Addr.getOperand(0);
1164 SDValue N1 = Addr.getOperand(1);
1165 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1166 if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
1167 // (add n0, c0)
1168 Base = N0;
1169 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1170 return true;
1171 }
1172 } else if (Addr.getOpcode() == ISD::SUB) {
1173 // sub C, x -> add (sub 0, x), C
1174 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1175 int64_t ByteOffset = C->getSExtValue();
1176 if (isUInt<16>(ByteOffset)) {
1177 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1178
1179 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1180 // the known bits in isDSOffsetLegal. We need to emit the selected node
1181 // here, so this is thrown away.
1182 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1183 Zero, Addr.getOperand(1));
1184
1185 if (isDSOffsetLegal(Sub, ByteOffset, 16)) {
1186 SmallVector<SDValue, 3> Opnds;
1187 Opnds.push_back(Zero);
1188 Opnds.push_back(Addr.getOperand(1));
1189
1190 // FIXME: Select to VOP3 version for with-carry.
1191 unsigned SubOp = AMDGPU::V_SUB_I32_e32;
1192 if (Subtarget->hasAddNoCarry()) {
1193 SubOp = AMDGPU::V_SUB_U32_e64;
1194 Opnds.push_back(
1195 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1196 }
1197
1198 MachineSDNode *MachineSub =
1199 CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1200
1201 Base = SDValue(MachineSub, 0);
1202 Offset = CurDAG->getTargetConstant(ByteOffset, DL, MVT::i16);
1203 return true;
1204 }
1205 }
1206 }
1207 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1208 // If we have a constant address, prefer to put the constant into the
1209 // offset. This can save moves to load the constant address since multiple
1210 // operations can share the zero base address register, and enables merging
1211 // into read2 / write2 instructions.
1212
1213 SDLoc DL(Addr);
1214
1215 if (isUInt<16>(CAddr->getZExtValue())) {
1216 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1217 MachineSDNode *MovZero = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1218 DL, MVT::i32, Zero);
1219 Base = SDValue(MovZero, 0);
1220 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1221 return true;
1222 }
1223 }
1224
1225 // default case
1226 Base = Addr;
1227 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i16);
1228 return true;
1229 }
1230
1231 // TODO: If offset is too big, put low 16-bit into offset.
1232 bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
1233 SDValue &Offset0,
1234 SDValue &Offset1) const {
1235 SDLoc DL(Addr);
1236
1237 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1238 SDValue N0 = Addr.getOperand(0);
1239 SDValue N1 = Addr.getOperand(1);
1240 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1241 unsigned DWordOffset0 = C1->getZExtValue() / 4;
1242 unsigned DWordOffset1 = DWordOffset0 + 1;
1243 // (add n0, c0)
1244 if (isDSOffsetLegal(N0, DWordOffset1, 8)) {
1245 Base = N0;
1246 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
1247 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
1248 return true;
1249 }
1250 } else if (Addr.getOpcode() == ISD::SUB) {
1251 // sub C, x -> add (sub 0, x), C
1252 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
1253 unsigned DWordOffset0 = C->getZExtValue() / 4;
1254 unsigned DWordOffset1 = DWordOffset0 + 1;
1255
1256 if (isUInt<8>(DWordOffset0)) {
1257 SDLoc DL(Addr);
1258 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1259
1260 // XXX - This is kind of hacky. Create a dummy sub node so we can check
1261 // the known bits in isDSOffsetLegal. We need to emit the selected node
1262 // here, so this is thrown away.
1263 SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
1264 Zero, Addr.getOperand(1));
1265
1266 if (isDSOffsetLegal(Sub, DWordOffset1, 8)) {
1267 SmallVector<SDValue, 3> Opnds;
1268 Opnds.push_back(Zero);
1269 Opnds.push_back(Addr.getOperand(1));
1270 unsigned SubOp = AMDGPU::V_SUB_I32_e32;
1271 if (Subtarget->hasAddNoCarry()) {
1272 SubOp = AMDGPU::V_SUB_U32_e64;
1273 Opnds.push_back(
1274 CurDAG->getTargetConstant(0, {}, MVT::i1)); // clamp bit
1275 }
1276
1277 MachineSDNode *MachineSub
1278 = CurDAG->getMachineNode(SubOp, DL, MVT::i32, Opnds);
1279
1280 Base = SDValue(MachineSub, 0);
1281 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
1282 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
1283 return true;
1284 }
1285 }
1286 }
1287 } else if (const ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1288 unsigned DWordOffset0 = CAddr->getZExtValue() / 4;
1289 unsigned DWordOffset1 = DWordOffset0 + 1;
1290 assert(4 * DWordOffset0 == CAddr->getZExtValue());
1291
1292 if (isUInt<8>(DWordOffset0) && isUInt<8>(DWordOffset1)) {
1293 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
1294 MachineSDNode *MovZero
1295 = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1296 DL, MVT::i32, Zero);
1297 Base = SDValue(MovZero, 0);
1298 Offset0 = CurDAG->getTargetConstant(DWordOffset0, DL, MVT::i8);
1299 Offset1 = CurDAG->getTargetConstant(DWordOffset1, DL, MVT::i8);
1300 return true;
1301 }
1302 }
1303
1304 // default case
1305
1306 Base = Addr;
1307 Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
1308 Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
1309 return true;
1310 }
1311
1312 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
1313 SDValue &VAddr, SDValue &SOffset,
1314 SDValue &Offset, SDValue &Offen,
1315 SDValue &Idxen, SDValue &Addr64,
1316 SDValue &GLC, SDValue &SLC,
1317 SDValue &TFE, SDValue &DLC,
1318 SDValue &SWZ) const {
1319 // Subtarget prefers to use flat instruction
1320 if (Subtarget->useFlatForGlobal())
1321 return false;
1322
1323 SDLoc DL(Addr);
1324
1325 if (!GLC.getNode())
1326 GLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1327 if (!SLC.getNode())
1328 SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1329 TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
1330 DLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
1331 SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
1332
1333 Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1334 Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
1335 Addr64 = CurDAG->getTargetConstant(0, DL, MVT::i1);
1336 SOffset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1337
1338 ConstantSDNode *C1 = nullptr;
1339 SDValue N0 = Addr;
1340 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1341 C1 = cast<ConstantSDNode>(Addr.getOperand(1));
1342 if (isUInt<32>(C1->getZExtValue()))
1343 N0 = Addr.getOperand(0);
1344 else
1345 C1 = nullptr;
1346 }
1347
1348 if (N0.getOpcode() == ISD::ADD) {
1349 // (add N2, N3) -> addr64, or
1350 // (add (add N2, N3), C1) -> addr64
1351 SDValue N2 = N0.getOperand(0);
1352 SDValue N3 = N0.getOperand(1);
1353 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1354
1355 if (N2->isDivergent()) {
1356 if (N3->isDivergent()) {
1357 // Both N2 and N3 are divergent. Use N0 (the result of the add) as the
1358 // addr64, and construct the resource from a 0 address.
1359 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1360 VAddr = N0;
1361 } else {
1362 // N2 is divergent, N3 is not.
1363 Ptr = N3;
1364 VAddr = N2;
1365 }
1366 } else {
1367 // N2 is not divergent.
1368 Ptr = N2;
1369 VAddr = N3;
1370 }
1371 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1372 } else if (N0->isDivergent()) {
1373 // N0 is divergent. Use it as the addr64, and construct the resource from a
1374 // 0 address.
1375 Ptr = SDValue(buildSMovImm64(DL, 0, MVT::v2i32), 0);
1376 VAddr = N0;
1377 Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
1378 } else {
1379 // N0 -> offset, or
1380 // (N0 + C1) -> offset
1381 VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
1382 Ptr = N0;
1383 }
1384
1385 if (!C1) {
1386 // No offset.
1387 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1388 return true;
1389 }
1390
1391 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
1392 // Legal offset for instruction.
1393 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1394 return true;
1395 }
1396
1397 // Illegal offset, store it in soffset.
1398 Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1399 SOffset =
1400 SDValue(CurDAG->getMachineNode(
1401 AMDGPU::S_MOV_B32, DL, MVT::i32,
1402 CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32)),
1403 0);
1404 return true;
1405 }
1406
1407 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1408 SDValue &VAddr, SDValue &SOffset,
1409 SDValue &Offset, SDValue &GLC,
1410 SDValue &SLC, SDValue &TFE,
1411 SDValue &DLC, SDValue &SWZ) const {
1412 SDValue Ptr, Offen, Idxen, Addr64;
1413
1414 // addr64 bit was removed for volcanic islands.
1415 if (!Subtarget->hasAddr64())
1416 return false;
1417
1418 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1419 GLC, SLC, TFE, DLC, SWZ))
1420 return false;
1421
1422 ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
1423 if (C->getSExtValue()) {
1424 SDLoc DL(Addr);
1425
1426 const SITargetLowering& Lowering =
1427 *static_cast<const SITargetLowering*>(getTargetLowering());
1428
1429 SRsrc = SDValue(Lowering.wrapAddr64Rsrc(*CurDAG, DL, Ptr), 0);
1430 return true;
1431 }
1432
1433 return false;
1434 }
1435
1436 bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
1437 SDValue &VAddr, SDValue &SOffset,
1438 SDValue &Offset,
1439 SDValue &SLC) const {
1440 SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
1441 SDValue GLC, TFE, DLC, SWZ;
1442
1443 return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1444 }
1445
1446 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
1447 auto PSV = PtrInfo.V.dyn_cast<const PseudoSourceValue *>();
1448 return PSV && PSV->isStack();
1449 }
1450
1451 std::pair<SDValue, SDValue> AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const {
1452 const MachineFunction &MF = CurDAG->getMachineFunction();
1453 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1454
1455 if (auto FI = dyn_cast<FrameIndexSDNode>(N)) {
1456 SDValue TFI = CurDAG->getTargetFrameIndex(FI->getIndex(),
1457 FI->getValueType(0));
1458
1459 // If we can resolve this to a frame index access, this will be relative to
1460 // either the stack or frame pointer SGPR.
1461 return std::make_pair(
1462 TFI, CurDAG->getRegister(Info->getStackPtrOffsetReg(), MVT::i32));
1463 }
1464
1465 // If we don't know this private access is a local stack object, it needs to
1466 // be relative to the entry point's scratch wave offset register.
1467 return std::make_pair(N, CurDAG->getRegister(Info->getScratchWaveOffsetReg(),
1468 MVT::i32));
1469 }
1470
1471 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffen(SDNode *Parent,
1472 SDValue Addr, SDValue &Rsrc,
1473 SDValue &VAddr, SDValue &SOffset,
1474 SDValue &ImmOffset) const {
1475
1476 SDLoc DL(Addr);
1477 MachineFunction &MF = CurDAG->getMachineFunction();
1478 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1479
1480 Rsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1481
1482 if (ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr)) {
1483 unsigned Imm = CAddr->getZExtValue();
1484
1485 SDValue HighBits = CurDAG->getTargetConstant(Imm & ~4095, DL, MVT::i32);
1486 MachineSDNode *MovHighBits = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32,
1487 DL, MVT::i32, HighBits);
1488 VAddr = SDValue(MovHighBits, 0);
1489
1490 // In a call sequence, stores to the argument stack area are relative to the
1491 // stack pointer.
1492 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
1493 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ?
1494 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg();
1495
1496 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32);
1497 ImmOffset = CurDAG->getTargetConstant(Imm & 4095, DL, MVT::i16);
1498 return true;
1499 }
1500
1501 if (CurDAG->isBaseWithConstantOffset(Addr)) {
1502 // (add n0, c1)
1503
1504 SDValue N0 = Addr.getOperand(0);
1505 SDValue N1 = Addr.getOperand(1);
1506
1507 // Offsets in vaddr must be positive if range checking is enabled.
1508 //
1509 // The total computation of vaddr + soffset + offset must not overflow. If
1510 // vaddr is negative, even if offset is 0 the sgpr offset add will end up
1511 // overflowing.
1512 //
1513 // Prior to gfx9, MUBUF instructions with the vaddr offset enabled would
1514 // always perform a range check. If a negative vaddr base index was used,
1515 // this would fail the range check. The overall address computation would
1516 // compute a valid address, but this doesn't happen due to the range
1517 // check. For out-of-bounds MUBUF loads, a 0 is returned.
1518 //
1519 // Therefore it should be safe to fold any VGPR offset on gfx9 into the
1520 // MUBUF vaddr, but not on older subtargets which can only do this if the
1521 // sign bit is known 0.
1522 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1523 if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue()) &&
1524 (!Subtarget->privateMemoryResourceIsRangeChecked() ||
1525 CurDAG->SignBitIsZero(N0))) {
1526 std::tie(VAddr, SOffset) = foldFrameIndex(N0);
1527 ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
1528 return true;
1529 }
1530 }
1531
1532 // (node)
1533 std::tie(VAddr, SOffset) = foldFrameIndex(Addr);
1534 ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16);
1535 return true;
1536 }
1537
1538 bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
1539 SDValue Addr,
1540 SDValue &SRsrc,
1541 SDValue &SOffset,
1542 SDValue &Offset) const {
1543 ConstantSDNode *CAddr = dyn_cast<ConstantSDNode>(Addr);
1544 if (!CAddr || !SIInstrInfo::isLegalMUBUFImmOffset(CAddr->getZExtValue()))
1545 return false;
1546
1547 SDLoc DL(Addr);
1548 MachineFunction &MF = CurDAG->getMachineFunction();
1549 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1550
1551 SRsrc = CurDAG->getRegister(Info->getScratchRSrcReg(), MVT::v4i32);
1552
1553 const MachinePointerInfo &PtrInfo = cast<MemSDNode>(Parent)->getPointerInfo();
1554 unsigned SOffsetReg = isStackPtrRelative(PtrInfo) ?
1555 Info->getStackPtrOffsetReg() : Info->getScratchWaveOffsetReg();
1556
1557 // FIXME: Get from MachinePointerInfo? We should only be using the frame
1558 // offset if we know this is in a call sequence.
1559 SOffset = CurDAG->getRegister(SOffsetReg, MVT::i32);
1560
1561 Offset = CurDAG->getTargetConstant(CAddr->getZExtValue(), DL, MVT::i16);
1562 return true;
1563 }
1564
1565 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1566 SDValue &SOffset, SDValue &Offset,
1567 SDValue &GLC, SDValue &SLC,
1568 SDValue &TFE, SDValue &DLC,
1569 SDValue &SWZ) const {
1570 SDValue Ptr, VAddr, Offen, Idxen, Addr64;
1571 const SIInstrInfo *TII =
1572 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
1573
1574 if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
1575 GLC, SLC, TFE, DLC, SWZ))
1576 return false;
1577
1578 if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
1579 !cast<ConstantSDNode>(Idxen)->getSExtValue() &&
1580 !cast<ConstantSDNode>(Addr64)->getSExtValue()) {
1581 uint64_t Rsrc = TII->getDefaultRsrcDataFormat() |
1582 APInt::getAllOnesValue(32).getZExtValue(); // Size
1583 SDLoc DL(Addr);
1584
1585 const SITargetLowering& Lowering =
1586 *static_cast<const SITargetLowering*>(getTargetLowering());
1587
1588 SRsrc = SDValue(Lowering.buildRSRC(*CurDAG, DL, Ptr, 0, Rsrc), 0);
1589 return true;
1590 }
1591 return false;
1592 }
1593
1594 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1595 SDValue &Soffset, SDValue &Offset
1596 ) const {
1597 SDValue GLC, SLC, TFE, DLC, SWZ;
1598
1599 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1600 }
1601 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
1602 SDValue &Soffset, SDValue &Offset,
1603 SDValue &SLC) const {
1604 SDValue GLC, TFE, DLC, SWZ;
1605
1606 return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
1607 }
1608
1609 template <bool IsSigned>
1610 bool AMDGPUDAGToDAGISel::SelectFlatOffset(SDNode *N,
1611 SDValue Addr,
1612 SDValue &VAddr,
1613 SDValue &Offset,
1614 SDValue &SLC) const {
1615 return static_cast<const SITargetLowering*>(getTargetLowering())->
1616 SelectFlatOffset(IsSigned, *CurDAG, N, Addr, VAddr, Offset, SLC);
1617 }
1618
1619 bool AMDGPUDAGToDAGISel::SelectFlatAtomic(SDNode *N,
1620 SDValue Addr,
1621 SDValue &VAddr,
1622 SDValue &Offset,
1623 SDValue &SLC) const {
1624 return SelectFlatOffset<false>(N, Addr, VAddr, Offset, SLC);
1625 }
1626
1627 bool AMDGPUDAGToDAGISel::SelectFlatAtomicSigned(SDNode *N,
1628 SDValue Addr,
1629 SDValue &VAddr,
1630 SDValue &Offset,
1631 SDValue &SLC) const {
1632 return SelectFlatOffset<true>(N, Addr, VAddr, Offset, SLC);
1633 }
1634
1635 bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode,
1636 SDValue &Offset, bool &Imm) const {
1637
1638 // FIXME: Handle non-constant offsets.
1639 ConstantSDNode *C = dyn_cast<ConstantSDNode>(ByteOffsetNode);
1640 if (!C)
1641 return false;
1642
1643 SDLoc SL(ByteOffsetNode);
1644 GCNSubtarget::Generation Gen = Subtarget->getGeneration();
1645 int64_t ByteOffset = C->getSExtValue();
1646 int64_t EncodedOffset = AMDGPU::getSMRDEncodedOffset(*Subtarget, ByteOffset);
1647
1648 if (AMDGPU::isLegalSMRDImmOffset(*Subtarget, ByteOffset)) {
1649 Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1650 Imm = true;
1651 return true;
1652 }
1653
1654 if (!isUInt<32>(EncodedOffset) || !isUInt<32>(ByteOffset))
1655 return false;
1656
1657 if (Gen == AMDGPUSubtarget::SEA_ISLANDS && isUInt<32>(EncodedOffset)) {
1658 // 32-bit Immediates are supported on Sea Islands.
1659 Offset = CurDAG->getTargetConstant(EncodedOffset, SL, MVT::i32);
1660 } else {
1661 SDValue C32Bit = CurDAG->getTargetConstant(ByteOffset, SL, MVT::i32);
1662 Offset = SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32,
1663 C32Bit), 0);
1664 }
1665 Imm = false;
1666 return true;
1667 }
1668
1669 SDValue AMDGPUDAGToDAGISel::Expand32BitAddress(SDValue Addr) const {
1670 if (Addr.getValueType() != MVT::i32)
1671 return Addr;
1672
1673 // Zero-extend a 32-bit address.
1674 SDLoc SL(Addr);
1675
1676 const MachineFunction &MF = CurDAG->getMachineFunction();
1677 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1678 unsigned AddrHiVal = Info->get32BitAddressHighBits();
1679 SDValue AddrHi = CurDAG->getTargetConstant(AddrHiVal, SL, MVT::i32);
1680
1681 const SDValue Ops[] = {
1682 CurDAG->getTargetConstant(AMDGPU::SReg_64_XEXECRegClassID, SL, MVT::i32),
1683 Addr,
1684 CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32),
1685 SDValue(CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, MVT::i32, AddrHi),
1686 0),
1687 CurDAG->getTargetConstant(AMDGPU::sub1, SL, MVT::i32),
1688 };
1689
1690 return SDValue(CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, SL, MVT::i64,
1691 Ops), 0);
1692 }
1693
1694 bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase,
1695 SDValue &Offset, bool &Imm) const {
1696 SDLoc SL(Addr);
1697
1698 // A 32-bit (address + offset) should not cause unsigned 32-bit integer
1699 // wraparound, because s_load instructions perform the addition in 64 bits.
1700 if ((Addr.getValueType() != MVT::i32 ||
1701 Addr->getFlags().hasNoUnsignedWrap()) &&
1702 CurDAG->isBaseWithConstantOffset(Addr)) {
1703 SDValue N0 = Addr.getOperand(0);
1704 SDValue N1 = Addr.getOperand(1);
1705
1706 if (SelectSMRDOffset(N1, Offset, Imm)) {
1707 SBase = Expand32BitAddress(N0);
1708 return true;
1709 }
1710 }
1711 SBase = Expand32BitAddress(Addr);
1712 Offset = CurDAG->getTargetConstant(0, SL, MVT::i32);
1713 Imm = true;
1714 return true;
1715 }
1716
1717 bool AMDGPUDAGToDAGISel::SelectSMRDImm(SDValue Addr, SDValue &SBase,
1718 SDValue &Offset) const {
1719 bool Imm;
1720 return SelectSMRD(Addr, SBase, Offset, Imm) && Imm;
1721 }
1722
1723 bool AMDGPUDAGToDAGISel::SelectSMRDImm32(SDValue Addr, SDValue &SBase,
1724 SDValue &Offset) const {
1725
1726 if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1727 return false;
1728
1729 bool Imm;
1730 if (!SelectSMRD(Addr, SBase, Offset, Imm))
1731 return false;
1732
1733 return !Imm && isa<ConstantSDNode>(Offset);
1734 }
1735
1736 bool AMDGPUDAGToDAGISel::SelectSMRDSgpr(SDValue Addr, SDValue &SBase,
1737 SDValue &Offset) const {
1738 bool Imm;
1739 return SelectSMRD(Addr, SBase, Offset, Imm) && !Imm &&
1740 !isa<ConstantSDNode>(Offset);
1741 }
1742
1743 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm(SDValue Addr,
1744 SDValue &Offset) const {
1745 bool Imm;
1746 return SelectSMRDOffset(Addr, Offset, Imm) && Imm;
1747 }
1748
1749 bool AMDGPUDAGToDAGISel::SelectSMRDBufferImm32(SDValue Addr,
1750 SDValue &Offset) const {
1751 if (Subtarget->getGeneration() != AMDGPUSubtarget::SEA_ISLANDS)
1752 return false;
1753
1754 bool Imm;
1755 if (!SelectSMRDOffset(Addr, Offset, Imm))
1756 return false;
1757
1758 return !Imm && isa<ConstantSDNode>(Offset);
1759 }
1760
1761 bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
1762 SDValue &Base,
1763 SDValue &Offset) const {
1764 SDLoc DL(Index);
1765
1766 if (CurDAG->isBaseWithConstantOffset(Index)) {
1767 SDValue N0 = Index.getOperand(0);
1768 SDValue N1 = Index.getOperand(1);
1769 ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
1770
1771 // (add n0, c0)
1772 // Don't peel off the offset (c0) if doing so could possibly lead
1773 // the base (n0) to be negative.
1774 if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0)) {
1775 Base = N0;
1776 Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
1777 return true;
1778 }
1779 }
1780
1781 if (isa<ConstantSDNode>(Index))
1782 return false;
1783
1784 Base = Index;
1785 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
1786 return true;
1787 }
1788
1789 SDNode *AMDGPUDAGToDAGISel::getS_BFE(unsigned Opcode, const SDLoc &DL,
1790 SDValue Val, uint32_t Offset,
1791 uint32_t Width) {
1792 // Transformation function, pack the offset and width of a BFE into
1793 // the format expected by the S_BFE_I32 / S_BFE_U32. In the second
1794 // source, bits [5:0] contain the offset and bits [22:16] the width.
1795 uint32_t PackedVal = Offset | (Width << 16);
1796 SDValue PackedConst = CurDAG->getTargetConstant(PackedVal, DL, MVT::i32);
1797
1798 return CurDAG->getMachineNode(Opcode, DL, MVT::i32, Val, PackedConst);
1799 }
1800
1801 void AMDGPUDAGToDAGISel::SelectS_BFEFromShifts(SDNode *N) {
1802 // "(a << b) srl c)" ---> "BFE_U32 a, (c-b), (32-c)
1803 // "(a << b) sra c)" ---> "BFE_I32 a, (c-b), (32-c)
1804 // Predicate: 0 < b <= c < 32
1805
1806 const SDValue &Shl = N->getOperand(0);
1807 ConstantSDNode *B = dyn_cast<ConstantSDNode>(Shl->getOperand(1));
1808 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
1809
1810 if (B && C) {
1811 uint32_t BVal = B->getZExtValue();
1812 uint32_t CVal = C->getZExtValue();
1813
1814 if (0 < BVal && BVal <= CVal && CVal < 32) {
1815 bool Signed = N->getOpcode() == ISD::SRA;
1816 unsigned Opcode = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
1817
1818 ReplaceNode(N, getS_BFE(Opcode, SDLoc(N), Shl.getOperand(0), CVal - BVal,
1819 32 - CVal));
1820 return;
1821 }
1822 }
1823 SelectCode(N);
1824 }
1825
1826 void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) {
1827 switch (N->getOpcode()) {
1828 case ISD::AND:
1829 if (N->getOperand(0).getOpcode() == ISD::SRL) {
1830 // "(a srl b) & mask" ---> "BFE_U32 a, b, popcount(mask)"
1831 // Predicate: isMask(mask)
1832 const SDValue &Srl = N->getOperand(0);
1833 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(Srl.getOperand(1));
1834 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
1835
1836 if (Shift && Mask) {
1837 uint32_t ShiftVal = Shift->getZExtValue();
1838 uint32_t MaskVal = Mask->getZExtValue();
1839
1840 if (isMask_32(MaskVal)) {
1841 uint32_t WidthVal = countPopulation(MaskVal);
1842
1843 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
1844 Srl.getOperand(0), ShiftVal, WidthVal));
1845 return;
1846 }
1847 }
1848 }
1849 break;
1850 case ISD::SRL:
1851 if (N->getOperand(0).getOpcode() == ISD::AND) {
1852 // "(a & mask) srl b)" ---> "BFE_U32 a, b, popcount(mask >> b)"
1853 // Predicate: isMask(mask >> b)
1854 const SDValue &And = N->getOperand(0);
1855 ConstantSDNode *Shift = dyn_cast<ConstantSDNode>(N->getOperand(1));
1856 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(And->getOperand(1));
1857
1858 if (Shift && Mask) {
1859 uint32_t ShiftVal = Shift->getZExtValue();
1860 uint32_t MaskVal = Mask->getZExtValue() >> ShiftVal;
1861
1862 if (isMask_32(MaskVal)) {
1863 uint32_t WidthVal = countPopulation(MaskVal);
1864
1865 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_U32, SDLoc(N),
1866 And.getOperand(0), ShiftVal, WidthVal));
1867 return;
1868 }
1869 }
1870 } else if (N->getOperand(0).getOpcode() == ISD::SHL) {
1871 SelectS_BFEFromShifts(N);
1872 return;
1873 }
1874 break;
1875 case ISD::SRA:
1876 if (N->getOperand(0).getOpcode() == ISD::SHL) {
1877 SelectS_BFEFromShifts(N);
1878 return;
1879 }
1880 break;
1881
1882 case ISD::SIGN_EXTEND_INREG: {
1883 // sext_inreg (srl x, 16), i8 -> bfe_i32 x, 16, 8
1884 SDValue Src = N->getOperand(0);
1885 if (Src.getOpcode() != ISD::SRL)
1886 break;
1887
1888 const ConstantSDNode *Amt = dyn_cast<ConstantSDNode>(Src.getOperand(1));
1889 if (!Amt)
1890 break;
1891
1892 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
1893 ReplaceNode(N, getS_BFE(AMDGPU::S_BFE_I32, SDLoc(N), Src.getOperand(0),
1894 Amt->getZExtValue(), Width));
1895 return;
1896 }
1897 }
1898
1899 SelectCode(N);
1900 }
1901
1902 bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const {
1903 assert(N->getOpcode() == ISD::BRCOND);
1904 if (!N->hasOneUse())
1905 return false;
1906
1907 SDValue Cond = N->getOperand(1);
1908 if (Cond.getOpcode() == ISD::CopyToReg)
1909 Cond = Cond.getOperand(2);
1910
1911 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
1912 return false;
1913
1914 MVT VT = Cond.getOperand(0).getSimpleValueType();
1915 if (VT == MVT::i32)
1916 return true;
1917
1918 if (VT == MVT::i64) {
1919 auto ST = static_cast<const GCNSubtarget *>(Subtarget);
1920
1921 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
1922 return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64();
1923 }
1924
1925 return false;
1926 }
1927
1928 void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) {
1929 SDValue Cond = N->getOperand(1);
1930
1931 if (Cond.isUndef()) {
1932 CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other,
1933 N->getOperand(2), N->getOperand(0));
1934 return;
1935 }
1936
1937 const GCNSubtarget *ST = static_cast<const GCNSubtarget *>(Subtarget);
1938 const SIRegisterInfo *TRI = ST->getRegisterInfo();
1939
1940 bool UseSCCBr = isCBranchSCC(N) && isUniformBr(N);
1941 unsigned BrOp = UseSCCBr ? AMDGPU::S_CBRANCH_SCC1 : AMDGPU::S_CBRANCH_VCCNZ;
1942 unsigned CondReg = UseSCCBr ? (unsigned)AMDGPU::SCC : TRI->getVCC();
1943 SDLoc SL(N);
1944
1945 if (!UseSCCBr) {
1946 // This is the case that we are selecting to S_CBRANCH_VCCNZ. We have not
1947 // analyzed what generates the vcc value, so we do not know whether vcc
1948 // bits for disabled lanes are 0. Thus we need to mask out bits for
1949 // disabled lanes.
1950 //
1951 // For the case that we select S_CBRANCH_SCC1 and it gets
1952 // changed to S_CBRANCH_VCCNZ in SIFixSGPRCopies, SIFixSGPRCopies calls
1953 // SIInstrInfo::moveToVALU which inserts the S_AND).
1954 //
1955 // We could add an analysis of what generates the vcc value here and omit
1956 // the S_AND when is unnecessary. But it would be better to add a separate
1957 // pass after SIFixSGPRCopies to do the unnecessary S_AND removal, so it
1958 // catches both cases.
1959 Cond = SDValue(CurDAG->getMachineNode(ST->isWave32() ? AMDGPU::S_AND_B32
1960 : AMDGPU::S_AND_B64,
1961 SL, MVT::i1,
1962 CurDAG->getRegister(ST->isWave32() ? AMDGPU::EXEC_LO
1963 : AMDGPU::EXEC,
1964 MVT::i1),
1965 Cond),
1966 0);
1967 }
1968
1969 SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, CondReg, Cond);
1970 CurDAG->SelectNodeTo(N, BrOp, MVT::Other,
1971 N->getOperand(2), // Basic Block
1972 VCC.getValue(0));
1973 }
1974
1975 void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
1976 MVT VT = N->getSimpleValueType(0);
1977 bool IsFMA = N->getOpcode() == ISD::FMA;
1978 if (VT != MVT::f32 || (!Subtarget->hasMadMixInsts() &&
1979 !Subtarget->hasFmaMixInsts()) ||
1980 ((IsFMA && Subtarget->hasMadMixInsts()) ||
1981 (!IsFMA && Subtarget->hasFmaMixInsts()))) {
1982 SelectCode(N);
1983 return;
1984 }
1985
1986 SDValue Src0 = N->getOperand(0);
1987 SDValue Src1 = N->getOperand(1);
1988 SDValue Src2 = N->getOperand(2);
1989 unsigned Src0Mods, Src1Mods, Src2Mods;
1990
1991 // Avoid using v_mad_mix_f32/v_fma_mix_f32 unless there is actually an operand
1992 // using the conversion from f16.
1993 bool Sel0 = SelectVOP3PMadMixModsImpl(Src0, Src0, Src0Mods);
1994 bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods);
1995 bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods);
1996
1997 assert((IsFMA || !Subtarget->hasFP32Denormals()) &&
1998 "fmad selected with denormals enabled");
1999 // TODO: We can select this with f32 denormals enabled if all the sources are
2000 // converted from f16 (in which case fmad isn't legal).
2001
2002 if (Sel0 || Sel1 || Sel2) {
2003 // For dummy operands.
2004 SDValue Zero = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2005 SDValue Ops[] = {
2006 CurDAG->getTargetConstant(Src0Mods, SDLoc(), MVT::i32), Src0,
2007 CurDAG->getTargetConstant(Src1Mods, SDLoc(), MVT::i32), Src1,
2008 CurDAG->getTargetConstant(Src2Mods, SDLoc(), MVT::i32), Src2,
2009 CurDAG->getTargetConstant(0, SDLoc(), MVT::i1),
2010 Zero, Zero
2011 };
2012
2013 CurDAG->SelectNodeTo(N,
2014 IsFMA ? AMDGPU::V_FMA_MIX_F32 : AMDGPU::V_MAD_MIX_F32,
2015 MVT::f32, Ops);
2016 } else {
2017 SelectCode(N);
2018 }
2019 }
2020
2021 // This is here because there isn't a way to use the generated sub0_sub1 as the
2022 // subreg index to EXTRACT_SUBREG in tablegen.
2023 void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
2024 MemSDNode *Mem = cast<MemSDNode>(N);
2025 unsigned AS = Mem->getAddressSpace();
2026 if (AS == AMDGPUAS::FLAT_ADDRESS) {
2027 SelectCode(N);
2028 return;
2029 }
2030
2031 MVT VT = N->getSimpleValueType(0);
2032 bool Is32 = (VT == MVT::i32);
2033 SDLoc SL(N);
2034
2035 MachineSDNode *CmpSwap = nullptr;
2036 if (Subtarget->hasAddr64()) {
2037 SDValue SRsrc, VAddr, SOffset, Offset, SLC;
2038
2039 if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset, SLC)) {
2040 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
2041 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
2042 SDValue CmpVal = Mem->getOperand(2);
2043
2044 // XXX - Do we care about glue operands?
2045
2046 SDValue Ops[] = {
2047 CmpVal, VAddr, SRsrc, SOffset, Offset, SLC, Mem->getChain()
2048 };
2049
2050 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2051 }
2052 }
2053
2054 if (!CmpSwap) {
2055 SDValue SRsrc, SOffset, Offset, SLC;
2056 if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset, SLC)) {
2057 unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
2058 AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
2059
2060 SDValue CmpVal = Mem->getOperand(2);
2061 SDValue Ops[] = {
2062 CmpVal, SRsrc, SOffset, Offset, SLC, Mem->getChain()
2063 };
2064
2065 CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2066 }
2067 }
2068
2069 if (!CmpSwap) {
2070 SelectCode(N);
2071 return;
2072 }
2073
2074 MachineMemOperand *MMO = Mem->getMemOperand();
2075 CurDAG->setNodeMemRefs(CmpSwap, {MMO});
2076
2077 unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
2078 SDValue Extract
2079 = CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
2080
2081 ReplaceUses(SDValue(N, 0), Extract);
2082 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
2083 CurDAG->RemoveDeadNode(N);
2084 }
2085
2086 void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
2087 // The address is assumed to be uniform, so if it ends up in a VGPR, it will
2088 // be copied to an SGPR with readfirstlane.
2089 unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
2090 AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
2091
2092 SDValue Chain = N->getOperand(0);
2093 SDValue Ptr = N->getOperand(2);
2094 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2095 MachineMemOperand *MMO = M->getMemOperand();
2096 bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
2097
2098 SDValue Offset;
2099 if (CurDAG->isBaseWithConstantOffset(Ptr)) {
2100 SDValue PtrBase = Ptr.getOperand(0);
2101 SDValue PtrOffset = Ptr.getOperand(1);
2102
2103 const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
2104 if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) {
2105 N = glueCopyToM0(N, PtrBase);
2106 Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
2107 }
2108 }
2109
2110 if (!Offset) {
2111 N = glueCopyToM0(N, Ptr);
2112 Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
2113 }
2114
2115 SDValue Ops[] = {
2116 Offset,
2117 CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
2118 Chain,
2119 N->getOperand(N->getNumOperands() - 1) // New glue
2120 };
2121
2122 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2123 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2124 }
2125
2126 static unsigned gwsIntrinToOpcode(unsigned IntrID) {
2127 switch (IntrID) {
2128 case Intrinsic::amdgcn_ds_gws_init:
2129 return AMDGPU::DS_GWS_INIT;
2130 case Intrinsic::amdgcn_ds_gws_barrier:
2131 return AMDGPU::DS_GWS_BARRIER;
2132 case Intrinsic::amdgcn_ds_gws_sema_v:
2133 return AMDGPU::DS_GWS_SEMA_V;
2134 case Intrinsic::amdgcn_ds_gws_sema_br:
2135 return AMDGPU::DS_GWS_SEMA_BR;
2136 case Intrinsic::amdgcn_ds_gws_sema_p:
2137 return AMDGPU::DS_GWS_SEMA_P;
2138 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2139 return AMDGPU::DS_GWS_SEMA_RELEASE_ALL;
2140 default:
2141 llvm_unreachable("not a gws intrinsic");
2142 }
2143 }
2144
2145 void AMDGPUDAGToDAGISel::SelectDS_GWS(SDNode *N, unsigned IntrID) {
2146 if (IntrID == Intrinsic::amdgcn_ds_gws_sema_release_all &&
2147 !Subtarget->hasGWSSemaReleaseAll()) {
2148 // Let this error.
2149 SelectCode(N);
2150 return;
2151 }
2152
2153 // Chain, intrinsic ID, vsrc, offset
2154 const bool HasVSrc = N->getNumOperands() == 4;
2155 assert(HasVSrc || N->getNumOperands() == 3);
2156
2157 SDLoc SL(N);
2158 SDValue BaseOffset = N->getOperand(HasVSrc ? 3 : 2);
2159 int ImmOffset = 0;
2160 MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
2161 MachineMemOperand *MMO = M->getMemOperand();
2162
2163 // Don't worry if the offset ends up in a VGPR. Only one lane will have
2164 // effect, so SIFixSGPRCopies will validly insert readfirstlane.
2165
2166 // The resource id offset is computed as (<isa opaque base> + M0[21:16] +
2167 // offset field) % 64. Some versions of the programming guide omit the m0
2168 // part, or claim it's from offset 0.
2169 if (ConstantSDNode *ConstOffset = dyn_cast<ConstantSDNode>(BaseOffset)) {
2170 // If we have a constant offset, try to use the 0 in m0 as the base.
2171 // TODO: Look into changing the default m0 initialization value. If the
2172 // default -1 only set the low 16-bits, we could leave it as-is and add 1 to
2173 // the immediate offset.
2174 glueCopyToM0(N, CurDAG->getTargetConstant(0, SL, MVT::i32));
2175 ImmOffset = ConstOffset->getZExtValue();
2176 } else {
2177 if (CurDAG->isBaseWithConstantOffset(BaseOffset)) {
2178 ImmOffset = BaseOffset.getConstantOperandVal(1);
2179 BaseOffset = BaseOffset.getOperand(0);
2180 }
2181
2182 // Prefer to do the shift in an SGPR since it should be possible to use m0
2183 // as the result directly. If it's already an SGPR, it will be eliminated
2184 // later.
2185 SDNode *SGPROffset
2186 = CurDAG->getMachineNode(AMDGPU::V_READFIRSTLANE_B32, SL, MVT::i32,
2187 BaseOffset);
2188 // Shift to offset in m0
2189 SDNode *M0Base
2190 = CurDAG->getMachineNode(AMDGPU::S_LSHL_B32, SL, MVT::i32,
2191 SDValue(SGPROffset, 0),
2192 CurDAG->getTargetConstant(16, SL, MVT::i32));
2193 glueCopyToM0(N, SDValue(M0Base, 0));
2194 }
2195
2196 SDValue Chain = N->getOperand(0);
2197 SDValue OffsetField = CurDAG->getTargetConstant(ImmOffset, SL, MVT::i32);
2198
2199 // TODO: Can this just be removed from the instruction?
2200 SDValue GDS = CurDAG->getTargetConstant(1, SL, MVT::i1);
2201
2202 const unsigned Opc = gwsIntrinToOpcode(IntrID);
2203 SmallVector<SDValue, 5> Ops;
2204 if (HasVSrc)
2205 Ops.push_back(N->getOperand(2));
2206 Ops.push_back(OffsetField);
2207 Ops.push_back(GDS);
2208 Ops.push_back(Chain);
2209
2210 SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2211 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
2212 }
2213
2214 void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
2215 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2216 switch (IntrID) {
2217 case Intrinsic::amdgcn_ds_append:
2218 case Intrinsic::amdgcn_ds_consume: {
2219 if (N->getValueType(0) != MVT::i32)
2220 break;
2221 SelectDSAppendConsume(N, IntrID);
2222 return;
2223 }
2224 }
2225
2226 SelectCode(N);
2227 }
2228
2229 void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
2230 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2231 unsigned Opcode;
2232 switch (IntrID) {
2233 case Intrinsic::amdgcn_wqm:
2234 Opcode = AMDGPU::WQM;
2235 break;
2236 case Intrinsic::amdgcn_softwqm:
2237 Opcode = AMDGPU::SOFT_WQM;
2238 break;
2239 case Intrinsic::amdgcn_wwm:
2240 Opcode = AMDGPU::WWM;
2241 break;
2242 default:
2243 SelectCode(N);
2244 return;
2245 }
2246
2247 SDValue Src = N->getOperand(1);
2248 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), {Src});
2249 }
2250
2251 void AMDGPUDAGToDAGISel::SelectINTRINSIC_VOID(SDNode *N) {
2252 unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2253 switch (IntrID) {
2254 case Intrinsic::amdgcn_ds_gws_init:
2255 case Intrinsic::amdgcn_ds_gws_barrier:
2256 case Intrinsic::amdgcn_ds_gws_sema_v:
2257 case Intrinsic::amdgcn_ds_gws_sema_br:
2258 case Intrinsic::amdgcn_ds_gws_sema_p:
2259 case Intrinsic::amdgcn_ds_gws_sema_release_all:
2260 SelectDS_GWS(N, IntrID);
2261 return;
2262 default:
2263 break;
2264 }
2265
2266 SelectCode(N);
2267 }
2268
2269 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
2270 unsigned &Mods) const {
2271 Mods = 0;
2272 Src = In;
2273
2274 if (Src.getOpcode() == ISD::FNEG) {
2275 Mods |= SISrcMods::NEG;
2276 Src = Src.getOperand(0);
2277 }
2278
2279 if (Src.getOpcode() == ISD::FABS) {
2280 Mods |= SISrcMods::ABS;
2281 Src = Src.getOperand(0);
2282 }
2283
2284 return true;
2285 }
2286
2287 bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src,
2288 SDValue &SrcMods) const {
2289 unsigned Mods;
2290 if (SelectVOP3ModsImpl(In, Src, Mods)) {
2291 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2292 return true;
2293 }
2294
2295 return false;
2296 }
2297
2298 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_NNaN(SDValue In, SDValue &Src,
2299 SDValue &SrcMods) const {
2300 SelectVOP3Mods(In, Src, SrcMods);
2301 return isNoNanSrc(Src);
2302 }
2303
2304 bool AMDGPUDAGToDAGISel::SelectVOP3Mods_f32(SDValue In, SDValue &Src,
2305 SDValue &SrcMods) const {
2306 if (In.getValueType() == MVT::f32)
2307 return SelectVOP3Mods(In, Src, SrcMods);
2308 Src = In;
2309 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);;
2310 return true;
2311 }
2312
2313 bool AMDGPUDAGToDAGISel::SelectVOP3NoMods(SDValue In, SDValue &Src) const {
2314 if (In.getOpcode() == ISD::FABS || In.getOpcode() == ISD::FNEG)
2315 return false;
2316
2317 Src = In;
2318 return true;
2319 }
2320
2321 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src,
2322 SDValue &SrcMods, SDValue &Clamp,
2323 SDValue &Omod) const {
2324 SDLoc DL(In);
2325 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2326 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2327
2328 return SelectVOP3Mods(In, Src, SrcMods);
2329 }
2330
2331 bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src,
2332 SDValue &SrcMods,
2333 SDValue &Clamp,
2334 SDValue &Omod) const {
2335 Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2336 return SelectVOP3Mods(In, Src, SrcMods);
2337 }
2338
2339 bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src,
2340 SDValue &Clamp, SDValue &Omod) const {
2341 Src = In;
2342
2343 SDLoc DL(In);
2344 Clamp = CurDAG->getTargetConstant(0, DL, MVT::i1);
2345 Omod = CurDAG->getTargetConstant(0, DL, MVT::i1);
2346
2347 return true;
2348 }
2349
2350 bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
2351 SDValue &SrcMods) const {
2352 unsigned Mods = 0;
2353 Src = In;
2354
2355 if (Src.getOpcode() == ISD::FNEG) {
2356 Mods ^= (SISrcMods::NEG | SISrcMods::NEG_HI);
2357 Src = Src.getOperand(0);
2358 }
2359
2360 if (Src.getOpcode() == ISD::BUILD_VECTOR) {
2361 unsigned VecMods = Mods;
2362
2363 SDValue Lo = stripBitcast(Src.getOperand(0));
2364 SDValue Hi = stripBitcast(Src.getOperand(1));
2365
2366 if (Lo.getOpcode() == ISD::FNEG) {
2367 Lo = stripBitcast(Lo.getOperand(0));
2368 Mods ^= SISrcMods::NEG;
2369 }
2370
2371 if (Hi.getOpcode() == ISD::FNEG) {
2372 Hi = stripBitcast(Hi.getOperand(0));
2373 Mods ^= SISrcMods::NEG_HI;
2374 }
2375
2376 if (isExtractHiElt(Lo, Lo))
2377 Mods |= SISrcMods::OP_SEL_0;
2378
2379 if (isExtractHiElt(Hi, Hi))
2380 Mods |= SISrcMods::OP_SEL_1;
2381
2382 Lo = stripExtractLoElt(Lo);
2383 Hi = stripExtractLoElt(Hi);
2384
2385 if (Lo == Hi && !isInlineImmediate(Lo.getNode())) {
2386 // Really a scalar input. Just select from the low half of the register to
2387 // avoid packing.
2388
2389 Src = Lo;
2390 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2391 return true;
2392 }
2393
2394 Mods = VecMods;
2395 }
2396
2397 // Packed instructions do not have abs modifiers.
2398 Mods |= SISrcMods::OP_SEL_1;
2399
2400 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2401 return true;
2402 }
2403
2404 bool AMDGPUDAGToDAGISel::SelectVOP3PMods0(SDValue In, SDValue &Src,
2405 SDValue &SrcMods,
2406 SDValue &Clamp) const {
2407 SDLoc SL(In);
2408
2409 // FIXME: Handle clamp and op_sel
2410 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
2411
2412 return SelectVOP3PMods(In, Src, SrcMods);
2413 }
2414
2415 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel(SDValue In, SDValue &Src,
2416 SDValue &SrcMods) const {
2417 Src = In;
2418 // FIXME: Handle op_sel
2419 SrcMods = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32);
2420 return true;
2421 }
2422
2423 bool AMDGPUDAGToDAGISel::SelectVOP3OpSel0(SDValue In, SDValue &Src,
2424 SDValue &SrcMods,
2425 SDValue &Clamp) const {
2426 SDLoc SL(In);
2427
2428 // FIXME: Handle clamp
2429 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
2430
2431 return SelectVOP3OpSel(In, Src, SrcMods);
2432 }
2433
2434 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods(SDValue In, SDValue &Src,
2435 SDValue &SrcMods) const {
2436 // FIXME: Handle op_sel
2437 return SelectVOP3Mods(In, Src, SrcMods);
2438 }
2439
2440 bool AMDGPUDAGToDAGISel::SelectVOP3OpSelMods0(SDValue In, SDValue &Src,
2441 SDValue &SrcMods,
2442 SDValue &Clamp) const {
2443 SDLoc SL(In);
2444
2445 // FIXME: Handle clamp
2446 Clamp = CurDAG->getTargetConstant(0, SL, MVT::i32);
2447
2448 return SelectVOP3OpSelMods(In, Src, SrcMods);
2449 }
2450
2451 // The return value is not whether the match is possible (which it always is),
2452 // but whether or not it a conversion is really used.
2453 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
2454 unsigned &Mods) const {
2455 Mods = 0;
2456 SelectVOP3ModsImpl(In, Src, Mods);
2457
2458 if (Src.getOpcode() == ISD::FP_EXTEND) {
2459 Src = Src.getOperand(0);
2460 assert(Src.getValueType() == MVT::f16);
2461 Src = stripBitcast(Src);
2462
2463 // Be careful about folding modifiers if we already have an abs. fneg is
2464 // applied last, so we don't want to apply an earlier fneg.
2465 if ((Mods & SISrcMods::ABS) == 0) {
2466 unsigned ModsTmp;
2467 SelectVOP3ModsImpl(Src, Src, ModsTmp);
2468
2469 if ((ModsTmp & SISrcMods::NEG) != 0)
2470 Mods ^= SISrcMods::NEG;
2471
2472 if ((ModsTmp & SISrcMods::ABS) != 0)
2473 Mods |= SISrcMods::ABS;
2474 }
2475
2476 // op_sel/op_sel_hi decide the source type and source.
2477 // If the source's op_sel_hi is set, it indicates to do a conversion from fp16.
2478 // If the sources's op_sel is set, it picks the high half of the source
2479 // register.
2480
2481 Mods |= SISrcMods::OP_SEL_1;
2482 if (isExtractHiElt(Src, Src)) {
2483 Mods |= SISrcMods::OP_SEL_0;
2484
2485 // TODO: Should we try to look for neg/abs here?
2486 }
2487
2488 return true;
2489 }
2490
2491 return false;
2492 }
2493
2494 bool AMDGPUDAGToDAGISel::SelectVOP3PMadMixMods(SDValue In, SDValue &Src,
2495 SDValue &SrcMods) const {
2496 unsigned Mods = 0;
2497 SelectVOP3PMadMixModsImpl(In, Src, Mods);
2498 SrcMods = CurDAG->getTargetConstant(Mods, SDLoc(In), MVT::i32);
2499 return true;
2500 }
2501
2502 SDValue AMDGPUDAGToDAGISel::getHi16Elt(SDValue In) const {
2503 if (In.isUndef())
2504 return CurDAG->getUNDEF(MVT::i32);
2505
2506 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(In)) {
2507 SDLoc SL(In);
2508 return CurDAG->getConstant(C->getZExtValue() << 16, SL, MVT::i32);
2509 }
2510
2511 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(In)) {
2512 SDLoc SL(In);
2513 return CurDAG->getConstant(
2514 C->getValueAPF().bitcastToAPInt().getZExtValue() << 16, SL, MVT::i32);
2515 }
2516
2517 SDValue Src;
2518 if (isExtractHiElt(In, Src))
2519 return Src;
2520
2521 return SDValue();
2522 }
2523
2524 bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
2525 assert(CurDAG->getTarget().getTargetTriple().getArch() == Triple::amdgcn);
2526
2527 const SIRegisterInfo *SIRI =
2528 static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo());
2529 const SIInstrInfo * SII =
2530 static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
2531
2532 unsigned Limit = 0;
2533 bool AllUsesAcceptSReg = true;
2534 for (SDNode::use_iterator U = N->use_begin(), E = SDNode::use_end();
2535 Limit < 10 && U != E; ++U, ++Limit) {
2536 const TargetRegisterClass *RC = getOperandRegClass(*U, U.getOperandNo());
2537
2538 // If the register class is unknown, it could be an unknown
2539 // register class that needs to be an SGPR, e.g. an inline asm
2540 // constraint
2541 if (!RC || SIRI->isSGPRClass(RC))
2542 return false;
2543
2544 if (RC != &AMDGPU::VS_32RegClass) {
2545 AllUsesAcceptSReg = false;
2546 SDNode * User = *U;
2547 if (User->isMachineOpcode()) {
2548 unsigned Opc = User->getMachineOpcode();
2549 MCInstrDesc Desc = SII->get(Opc);
2550 if (Desc.isCommutable()) {
2551 unsigned OpIdx = Desc.getNumDefs() + U.getOperandNo();
2552 unsigned CommuteIdx1 = TargetInstrInfo::CommuteAnyOperandIndex;
2553 if (SII->findCommutedOpIndices(Desc, OpIdx, CommuteIdx1)) {
2554 unsigned CommutedOpNo = CommuteIdx1 - Desc.getNumDefs();
2555 const TargetRegisterClass *CommutedRC = getOperandRegClass(*U, CommutedOpNo);
2556 if (CommutedRC == &AMDGPU::VS_32RegClass)
2557 AllUsesAcceptSReg = true;
2558 }
2559 }
2560 }
2561 // If "AllUsesAcceptSReg == false" so far we haven't suceeded
2562 // commuting current user. This means have at least one use
2563 // that strictly require VGPR. Thus, we will not attempt to commute
2564 // other user instructions.
2565 if (!AllUsesAcceptSReg)
2566 break;
2567 }
2568 }
2569 return !AllUsesAcceptSReg && (Limit < 10);
2570 }
2571
2572 bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode * N) const {
2573 auto Ld = cast<LoadSDNode>(N);
2574
2575 return Ld->getAlignment() >= 4 &&
2576 (
2577 (
2578 (
2579 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2580 Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT
2581 )
2582 &&
2583 !N->isDivergent()
2584 )
2585 ||
2586 (
2587 Subtarget->getScalarizeGlobalBehavior() &&
2588 Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
2589 !Ld->isVolatile() &&
2590 !N->isDivergent() &&
2591 static_cast<const SITargetLowering *>(
2592 getTargetLowering())->isMemOpHasNoClobberedMemOperand(N)
2593 )
2594 );
2595 }
2596
2597 void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
2598 const AMDGPUTargetLowering& Lowering =
2599 *static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
2600 bool IsModified = false;
2601 do {
2602 IsModified = false;
2603
2604 // Go over all selected nodes and try to fold them a bit more
2605 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_begin();
2606 while (Position != CurDAG->allnodes_end()) {
2607 SDNode *Node = &*Position++;
2608 MachineSDNode *MachineNode = dyn_cast<MachineSDNode>(Node);
2609 if (!MachineNode)
2610 continue;
2611
2612 SDNode *ResNode = Lowering.PostISelFolding(MachineNode, *CurDAG);
2613 if (ResNode != Node) {
2614 if (ResNode)
2615 ReplaceUses(Node, ResNode);
2616 IsModified = true;
2617 }
2618 }
2619 CurDAG->RemoveDeadNodes();
2620 } while (IsModified);
2621 }
2622
2623 bool R600DAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
2624 Subtarget = &MF.getSubtarget<R600Subtarget>();
2625 return SelectionDAGISel::runOnMachineFunction(MF);
2626 }
2627
2628 bool R600DAGToDAGISel::isConstantLoad(const MemSDNode *N, int CbId) const {
2629 if (!N->readMem())
2630 return false;
2631 if (CbId == -1)
2632 return N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
2633 N->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
2634
2635 return N->getAddressSpace() == AMDGPUAS::CONSTANT_BUFFER_0 + CbId;
2636 }
2637
2638 bool R600DAGToDAGISel::SelectGlobalValueConstantOffset(SDValue Addr,
2639 SDValue& IntPtr) {
2640 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Addr)) {
2641 IntPtr = CurDAG->getIntPtrConstant(Cst->getZExtValue() / 4, SDLoc(Addr),
2642 true);
2643 return true;
2644 }
2645 return false;
2646 }
2647
2648 bool R600DAGToDAGISel::SelectGlobalValueVariableOffset(SDValue Addr,
2649 SDValue& BaseReg, SDValue &Offset) {
2650 if (!isa<ConstantSDNode>(Addr)) {
2651 BaseReg = Addr;
2652 Offset = CurDAG->getIntPtrConstant(0, SDLoc(Addr), true);
2653 return true;
2654 }
2655 return false;
2656 }
2657
2658 void R600DAGToDAGISel::Select(SDNode *N) {
2659 unsigned int Opc = N->getOpcode();
2660 if (N->isMachineOpcode()) {
2661 N->setNodeId(-1);
2662 return; // Already selected.
2663 }
2664
2665 switch (Opc) {
2666 default: break;
2667 case AMDGPUISD::BUILD_VERTICAL_VECTOR:
2668 case ISD::SCALAR_TO_VECTOR:
2669 case ISD::BUILD_VECTOR: {
2670 EVT VT = N->getValueType(0);
2671 unsigned NumVectorElts = VT.getVectorNumElements();
2672 unsigned RegClassID;
2673 // BUILD_VECTOR was lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
2674 // that adds a 128 bits reg copy when going through TwoAddressInstructions
2675 // pass. We want to avoid 128 bits copies as much as possible because they
2676 // can't be bundled by our scheduler.
2677 switch(NumVectorElts) {
2678 case 2: RegClassID = R600::R600_Reg64RegClassID; break;
2679 case 4:
2680 if (Opc == AMDGPUISD::BUILD_VERTICAL_VECTOR)
2681 RegClassID = R600::R600_Reg128VerticalRegClassID;
2682 else
2683 RegClassID = R600::R600_Reg128RegClassID;
2684 break;
2685 default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
2686 }
2687 SelectBuildVector(N, RegClassID);
2688 return;
2689 }
2690 }
2691
2692 SelectCode(N);
2693 }
2694
2695 bool R600DAGToDAGISel::SelectADDRIndirect(SDValue Addr, SDValue &Base,
2696 SDValue &Offset) {
2697 ConstantSDNode *C;
2698 SDLoc DL(Addr);
2699
2700 if ((C = dyn_cast<ConstantSDNode>(Addr))) {
2701 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
2702 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
2703 } else if ((Addr.getOpcode() == AMDGPUISD::DWORDADDR) &&
2704 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(0)))) {
2705 Base = CurDAG->getRegister(R600::INDIRECT_BASE_ADDR, MVT::i32);
2706 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
2707 } else if ((Addr.getOpcode() == ISD::ADD || Addr.getOpcode() == ISD::OR) &&
2708 (C = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))) {
2709 Base = Addr.getOperand(0);
2710 Offset = CurDAG->getTargetConstant(C->getZExtValue(), DL, MVT::i32);
2711 } else {
2712 Base = Addr;
2713 Offset = CurDAG->getTargetConstant(0, DL, MVT::i32);
2714 }
2715
2716 return true;
2717 }
2718
2719 bool R600DAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base,
2720 SDValue &Offset) {
2721 ConstantSDNode *IMMOffset;
2722
2723 if (Addr.getOpcode() == ISD::ADD
2724 && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1)))
2725 && isInt<16>(IMMOffset->getZExtValue())) {
2726
2727 Base = Addr.getOperand(0);
2728 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
2729 MVT::i32);
2730 return true;
2731 // If the pointer address is constant, we can move it to the offset field.
2732 } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr))
2733 && isInt<16>(IMMOffset->getZExtValue())) {
2734 Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(),
2735 SDLoc(CurDAG->getEntryNode()),
2736 R600::ZERO, MVT::i32);
2737 Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), SDLoc(Addr),
2738 MVT::i32);
2739 return true;
2740 }
2741
2742 // Default case, no offset
2743 Base = Addr;
2744 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i32);
2745 return true;
2746 }
The low level virtual machine